diff --git a/Dockerfile b/Dockerfile index 6faaf180b99..79b8ba93b30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,7 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache && chown -R superset:superset ./* \ && rm -rf /var/lib/apt/lists/* -COPY --chown=superset:superset setup.py MANIFEST.in README.md ./ +COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./ # setup.py uses the version information in package.json COPY --chown=superset:superset superset-frontend/package.json superset-frontend/ COPY --chown=superset:superset requirements/base.txt requirements/ diff --git a/docs/docs/contributing/local-backend.mdx b/docs/docs/contributing/local-backend.mdx index ce62808d0c4..ea34c85ec53 100644 --- a/docs/docs/contributing/local-backend.mdx +++ b/docs/docs/contributing/local-backend.mdx @@ -70,7 +70,10 @@ If you have made changes to the FAB-managed templates, which are not built the s #### Dependencies -If you add a new requirement or update an existing requirement (per the `install_requires` section in `setup.py`) you must recompile (freeze) the Python dependencies to ensure that for CI, testing, etc. the build is deterministic. This can be achieved via, +If you add a new requirement or update an existing requirement (per the +`requirements` section in `pyproject.toml`) you must recompile (freeze) the +Python dependencies to ensure that for CI, testing, etc. the build is deterministic. +This can be achieved via, ```bash $ python3 -m venv venv diff --git a/docs/docs/contributing/pull-request-guidelines.mdx b/docs/docs/contributing/pull-request-guidelines.mdx index 4e2f823a979..acc4dc0287e 100644 --- a/docs/docs/contributing/pull-request-guidelines.mdx +++ b/docs/docs/contributing/pull-request-guidelines.mdx @@ -54,7 +54,9 @@ Finally, never submit a PR that will put master branch in broken state. If the P - Recommended capture tools ([Kap](https://getkap.co/), [LICEcap](https://www.cockos.com/licecap/), [Skitch](https://download.cnet.com/Skitch/3000-13455_4-189876.html)) - If no screenshot is provided, the committers will mark the PR with `need:screenshot` label and will not review until screenshot is provided. - **Dependencies:** Be careful about adding new dependency and avoid unnecessary dependencies. - - For Python, include it in `setup.py` denoting any specific restrictions and in `requirements.txt` pinned to a specific version which ensures that the application build is deterministic. + - For Python, include it in `pyproject.toml` denoting any specific restrictions and + in `requirements.txt` pinned to a specific version which ensures that the application + build is deterministic. - For TypeScript/JavaScript, include new libraries in `package.json` - **Tests:** The pull request should include tests, either as doctests, unit tests, or both. Make sure to resolve all errors and test failures. See [Testing](#testing) for how to run tests. - **Documentation:** If the pull request adds functionality, the docs should be updated as part of the same PR. diff --git a/docs/docs/databases/installing-database-drivers.mdx b/docs/docs/databases/installing-database-drivers.mdx index fd8ad841667..50305db1adc 100644 --- a/docs/docs/databases/installing-database-drivers.mdx +++ b/docs/docs/databases/installing-database-drivers.mdx @@ -20,7 +20,9 @@ which is part of the Python standard library. You’ll need to install the required packages for the database you want to use as your metadata database as well as the packages needed to connect to the databases you want to access through Superset. -Some of the recommended packages are shown below. Please refer to [setup.py](https://github.com/apache/superset/blob/master/setup.py) for the versions that are compatible with Superset. +Some of the recommended packages are shown below. Please refer to +[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml) for the versions that +are compatible with Superset. | Database | PyPI package | Connection String | | --------------------------------------------------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | diff --git a/docs/docs/installation/installing-superset-from-pypi.mdx b/docs/docs/installation/installing-superset-from-pypi.mdx index dc721cec963..710f6c0a22e 100644 --- a/docs/docs/installation/installing-superset-from-pypi.mdx +++ b/docs/docs/installation/installing-superset-from-pypi.mdx @@ -66,7 +66,11 @@ We don't recommend using the system installed Python. Instead, first install the brew install readline pkg-config libffi openssl mysql postgresql@14 ``` -You should install a recent version of Python. Refer to the [setup.py](https://github.com/apache/superset/blob/master/setup.py) file for a list of Python versions officially supported by Superset. We'd recommend using a Python version manager like [pyenv](https://github.com/pyenv/pyenv) (and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)). +You should install a recent version of Python. Refer to the +[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml) file for a list of Python +versions officially supported by Superset. We'd recommend using a Python version manager +like [pyenv](https://github.com/pyenv/pyenv) +(and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)). :::tip To identify the Python version used by the official docker image, see the [Dockerfile](https://github.com/apache/superset/blob/master/Dockerfile). Additional docker images published for newer versions of Python can be found in [this file](https://github.com/apache/superset/blob/master/scripts/build_docker.py). diff --git a/pyproject.toml b/pyproject.toml index 7a77fcb9dbf..da7dbcfed77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,9 @@ build-backend = "setuptools.build_meta" name = "apache-superset" description = "A modern, enterprise-ready business intelligence web application" readme = "README.md" -dynamic = ["version", "scripts", "entry-points", "license", "requires-python"] +dynamic = ["version", "scripts", "entry-points"] +requires-python = "~=3.9" +license = { file="LICENSE.txt" } authors = [ { name = "Apache Software Foundation", email = "dev@superset.apache.org" }, ] @@ -197,3 +199,196 @@ development = [ [project.urls] homepage = "https://superset.apache.org/" documentation = "https://superset.apache.org/docs/intro" + + +[tool.isort] +combine_as_imports = true +include_trailing_comma = true +line_length = 88 +known_first_party = "superset" +known_third_party = "alembic, apispec, backoff, celery, click, colorama, cron_descriptor, croniter, cryptography, dateutil, deprecation, flask, flask_appbuilder, flask_babel, flask_caching, flask_compress, flask_jwt_extended, flask_login, flask_migrate, flask_sqlalchemy, flask_talisman, flask_testing, flask_wtf, freezegun, geohash, geopy, holidays, humanize, isodate, jinja2, jwt, markdown, markupsafe, marshmallow, msgpack, nh3, numpy, pandas, parameterized, parsedatetime, pgsanity, pkg_resources, polyline, prison, progress, pyarrow, sqlalchemy_bigquery, pyhive, pyparsing, pytest, pytest_mock, pytz, redis, requests, selenium, setuptools, shillelagh, simplejson, slack, sqlalchemy, sqlalchemy_utils, sqlparse, typing_extensions, urllib3, werkzeug, wtforms, wtforms_json, yaml" +multi_line_output = 3 +order_by_type = false + +[tool.mypy] +check_untyped_defs = true +disallow_any_generics = true +disallow_untyped_calls = true +disallow_untyped_defs = true +ignore_missing_imports = true +no_implicit_optional = true +warn_unused_ignores = true + +[[tool.mypy.overrides]] +module = "superset.migrations.versions.*" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "tests.*" +check_untyped_defs = false +disallow_untyped_calls = false +disallow_untyped_defs = false + +[tool.tox] +legacy_tox_ini = """ +# Remember to start celery workers to run celery tests, e.g. +# celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 +[testenv] +basepython = python3.10 +ignore_basepython_conflict = true +commands = + superset db upgrade + superset init + # use -s to be able to use break pointers. + # no args or tests/* can be passed as an argument to run all tests + pytest -s {posargs} +deps = + -rrequirements/development.txt +setenv = + PYTHONPATH = {toxinidir} + SUPERSET_TESTENV = true + SUPERSET_CONFIG = tests.integration_tests.superset_test_config + SUPERSET_HOME = {envtmpdir} + mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 + postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test + sqlite: + SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db + SUPERSET__SQLALCHEMY_EXAMPLES_URI = sqlite:////{envtmpdir}/examples.db + mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 + # docker run -p 8080:8080 --name presto starburstdata/presto + mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default + # based on https://github.com/big-data-europe/docker-hadoop + # clone the repo & run docker compose up -d to test locally + mysql-hive: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 + mysql-hive: SUPERSET__SQLALCHEMY_EXAMPLES_URI = hive://localhost:10000/default + # make sure that directory is accessible by docker + hive: UPLOAD_FOLDER = /tmp/.superset/app/static/uploads/ +usedevelop = true +allowlist_externals = + npm + pkill + +[testenv:cypress] +setenv = + PYTHONPATH = {toxinidir} + SUPERSET_TESTENV = true + SUPERSET_CONFIG = tests.integration_tests.superset_test_config + SUPERSET_HOME = {envtmpdir} +commands = + npm install -g npm@'>=6.5.0' + pip install -e {toxinidir}/ + {toxinidir}/superset-frontend/cypress_build.sh +commands_post = + pkill -if "python {envbindir}/flask" + +[testenv:cypress-dashboard] +setenv = + PYTHONPATH = {toxinidir} + SUPERSET_TESTENV = true + SUPERSET_CONFIG = tests.integration_tests.superset_test_config + SUPERSET_HOME = {envtmpdir} +commands = + npm install -g npm@'>=6.5.0' + pip install -e {toxinidir}/ + {toxinidir}/superset-frontend/cypress_build.sh dashboard +commands_post = + pkill -if "python {envbindir}/flask" + +[testenv:cypress-explore] +setenv = + PYTHONPATH = {toxinidir} + SUPERSET_TESTENV = true + SUPERSET_CONFIG = tests.integration_tests.superset_test_config + SUPERSET_HOME = {envtmpdir} +commands = + npm install -g npm@'>=6.5.0' + pip install -e {toxinidir}/ + {toxinidir}/superset-frontend/cypress_build.sh explore +commands_post = + pkill -if "python {envbindir}/flask" + +[testenv:cypress-sqllab] +setenv = + PYTHONPATH = {toxinidir} + SUPERSET_TESTENV = true + SUPERSET_CONFIG = tests.integration_tests.superset_test_config + SUPERSET_HOME = {envtmpdir} +commands = + npm install -g npm@'>=6.5.0' + pip install -e {toxinidir}/ + {toxinidir}/superset-frontend/cypress_build.sh sqllab +commands_post = + pkill -if "python {envbindir}/flask" + +[testenv:cypress-sqllab-backend-persist] +setenv = + PYTHONPATH = {toxinidir} + SUPERSET_TESTENV = true + SUPERSET_CONFIG = tests.integration_tests.superset_test_config + SUPERSET_HOME = {envtmpdir} +commands = + npm install -g npm@'>=6.5.0' + pip install -e {toxinidir}/ + {toxinidir}/superset-frontend/cypress_build.sh sqllab +commands_post = + pkill -if "python {envbindir}/flask" + +[testenv:eslint] +changedir = {toxinidir}/superset-frontend +commands = + npm run lint +deps = + +[testenv:fossa] +commands = + {toxinidir}/scripts/fossa.sh +deps = +passenv = * + +[testenv:javascript] +commands = + npm install -g npm@'>=6.5.0' + {toxinidir}/superset-frontend/js_build.sh +deps = + +[testenv:license-check] +commands = + {toxinidir}/scripts/check_license.sh +passenv = * +whitelist_externals = + {toxinidir}/scripts/check_license.sh +deps = + +[testenv:pre-commit] +commands = + pre-commit run --all-files +deps = + -rrequirements/development.txt +skip_install = true + +[testenv:pylint] +commands = + pylint superset +deps = + -rrequirements/development.txt + +[testenv:thumbnails] +setenv = + SUPERSET_CONFIG = tests.integration_tests.superset_test_config_thumbnails +deps = + -rrequirements/development.txt + +[tox] +envlist = + cypress-dashboard + cypress-explore + cypress-sqllab + cypress-sqllab-backend-persist + eslint + fossa + javascript + license-check + pre-commit + pylint +skipsdist = true +""" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 0d7c0b655fd..00000000000 --- a/setup.cfg +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -[metadata] -name = Superset -summary = a data exploration platform -description_file = README.md -author = Apache Superset Dev -author_email = dev@superset.apache.org -license = Apache License, Version 2.0 - -[files] -packages = superset - -[isort] -combine_as_imports = true -include_trailing_comma = true -line_length = 88 -known_first_party = superset -known_third_party =alembic,apispec,backoff,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,nh3,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,shillelagh,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml -multi_line_output = 3 -order_by_type = false - -[mypy] -check_untyped_defs = true -disallow_any_generics = true -disallow_untyped_calls = true -disallow_untyped_defs = true -ignore_missing_imports = true -no_implicit_optional = true -warn_unused_ignores = true - -[mypy-superset.migrations.versions.*] -ignore_errors = true - -[mypy-tests.*] -check_untyped_defs = false -disallow_untyped_calls = false -disallow_untyped_defs = false diff --git a/setup.py b/setup.py index 47502c08239..4a7247a3a98 100644 --- a/setup.py +++ b/setup.py @@ -23,12 +23,10 @@ from setuptools import find_packages, setup BASE_DIR = os.path.abspath(os.path.dirname(__file__)) PACKAGE_JSON = os.path.join(BASE_DIR, "superset-frontend", "package.json") + with open(PACKAGE_JSON) as package_file: version_string = json.load(package_file)["version"] -with open("README.md", encoding="utf-8") as f: - long_description = f.read() - def get_git_sha() -> str: try: @@ -54,10 +52,6 @@ with open(VERSION_INFO_FILE, "w") as version_file: version_string = version_string.replace("-dev", ".dev0") setup( - name="apache-superset", - description="A modern, enterprise-ready business intelligence web application", - long_description=long_description, - long_description_content_type="text/markdown", version=version_string, packages=find_packages(), include_package_data=True, @@ -75,6 +69,5 @@ setup( "superset=superset.extensions.metadb:SupersetShillelaghAdapter" ], }, - python_requires="~=3.9", download_url="https://www.apache.org/dist/superset/" + version_string, ) diff --git a/superset/utils/retries.py b/superset/utils/retries.py index b8db1aa8444..2d1036ebf96 100644 --- a/superset/utils/retries.py +++ b/superset/utils/retries.py @@ -30,7 +30,7 @@ def retry_call( # pylint: disable=too-many-arguments giveup_log_level: int = logging.WARNING, fargs: Optional[list[Any]] = None, fkwargs: Optional[dict[str, Any]] = None, - **kwargs: Any + **kwargs: Any, ) -> Any: """ Retry a given call. diff --git a/tests/integration_tests/reports/utils.py b/tests/integration_tests/reports/utils.py index 8b860892ed2..84db93aa574 100644 --- a/tests/integration_tests/reports/utils.py +++ b/tests/integration_tests/reports/utils.py @@ -185,7 +185,7 @@ def create_dashboard_report(dashboard, extra, **kwargs): extra={ "dashboard": extra, }, - **kwargs + **kwargs, ) error = None diff --git a/tests/integration_tests/utils/encrypt_tests.py b/tests/integration_tests/utils/encrypt_tests.py index 45fd291ee85..33a742cbdcc 100644 --- a/tests/integration_tests/utils/encrypt_tests.py +++ b/tests/integration_tests/utils/encrypt_tests.py @@ -30,7 +30,7 @@ class CustomEncFieldAdapter(AbstractEncryptedFieldAdapter): self, app_config: Optional[dict[str, Any]], *args: list[Any], - **kwargs: Optional[dict[str, Any]] + **kwargs: Optional[dict[str, Any]], ) -> TypeDecorator: if app_config: return StringEncryptedType(*args, app_config["SECRET_KEY"], **kwargs) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 98b5e55fcee..00000000000 --- a/tox.ini +++ /dev/null @@ -1,177 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Remember to start celery workers to run celery tests, e.g. -# celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 -[testenv] -basepython = python3.10 -ignore_basepython_conflict = true -commands = - superset db upgrade - superset init - # use -s to be able to use break pointers. - # no args or tests/* can be passed as an argument to run all tests - pytest -s {posargs} -deps = - -rrequirements/development.txt -setenv = - PYTHONPATH = {toxinidir} - SUPERSET_TESTENV = true - SUPERSET_CONFIG = tests.integration_tests.superset_test_config - SUPERSET_HOME = {envtmpdir} - mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 - postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test - sqlite: - SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db - SUPERSET__SQLALCHEMY_EXAMPLES_URI = sqlite:////{envtmpdir}/examples.db - mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 - # docker run -p 8080:8080 --name presto starburstdata/presto - mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default - # based on https://github.com/big-data-europe/docker-hadoop - # clone the repo & run docker compose up -d to test locally - mysql-hive: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 - mysql-hive: SUPERSET__SQLALCHEMY_EXAMPLES_URI = hive://localhost:10000/default - # make sure that directory is accessible by docker - hive: UPLOAD_FOLDER = /tmp/.superset/app/static/uploads/ -usedevelop = true -allowlist_externals = - npm - pkill - -[testenv:cypress] -setenv = - PYTHONPATH = {toxinidir} - SUPERSET_TESTENV = true - SUPERSET_CONFIG = tests.integration_tests.superset_test_config - SUPERSET_HOME = {envtmpdir} -commands = - npm install -g npm@'>=6.5.0' - pip install -e {toxinidir}/ - {toxinidir}/superset-frontend/cypress_build.sh -commands_post = - pkill -if "python {envbindir}/flask" - -[testenv:cypress-dashboard] -setenv = - PYTHONPATH = {toxinidir} - SUPERSET_TESTENV = true - SUPERSET_CONFIG = tests.integration_tests.superset_test_config - SUPERSET_HOME = {envtmpdir} -commands = - npm install -g npm@'>=6.5.0' - pip install -e {toxinidir}/ - {toxinidir}/superset-frontend/cypress_build.sh dashboard -commands_post = - pkill -if "python {envbindir}/flask" - -[testenv:cypress-explore] -setenv = - PYTHONPATH = {toxinidir} - SUPERSET_TESTENV = true - SUPERSET_CONFIG = tests.integration_tests.superset_test_config - SUPERSET_HOME = {envtmpdir} -commands = - npm install -g npm@'>=6.5.0' - pip install -e {toxinidir}/ - {toxinidir}/superset-frontend/cypress_build.sh explore -commands_post = - pkill -if "python {envbindir}/flask" - -[testenv:cypress-sqllab] -setenv = - PYTHONPATH = {toxinidir} - SUPERSET_TESTENV = true - SUPERSET_CONFIG = tests.integration_tests.superset_test_config - SUPERSET_HOME = {envtmpdir} -commands = - npm install -g npm@'>=6.5.0' - pip install -e {toxinidir}/ - {toxinidir}/superset-frontend/cypress_build.sh sqllab -commands_post = - pkill -if "python {envbindir}/flask" - -[testenv:cypress-sqllab-backend-persist] -setenv = - PYTHONPATH = {toxinidir} - SUPERSET_TESTENV = true - SUPERSET_CONFIG = tests.integration_tests.superset_test_config - SUPERSET_HOME = {envtmpdir} -commands = - npm install -g npm@'>=6.5.0' - pip install -e {toxinidir}/ - {toxinidir}/superset-frontend/cypress_build.sh sqllab -commands_post = - pkill -if "python {envbindir}/flask" - -[testenv:eslint] -changedir = {toxinidir}/superset-frontend -commands = - npm run lint -deps = - -[testenv:fossa] -commands = - {toxinidir}/scripts/fossa.sh -deps = -passenv = * - -[testenv:javascript] -commands = - npm install -g npm@'>=6.5.0' - {toxinidir}/superset-frontend/js_build.sh -deps = - -[testenv:license-check] -commands = - {toxinidir}/scripts/check_license.sh -passenv = * -whitelist_externals = - {toxinidir}/scripts/check_license.sh -deps = - -[testenv:pre-commit] -commands = - pre-commit run --all-files -deps = - -rrequirements/development.txt -skip_install = true - -[testenv:pylint] -commands = - pylint superset -deps = - -rrequirements/development.txt - -[testenv:thumbnails] -setenv = - SUPERSET_CONFIG = tests.integration_tests.superset_test_config_thumbnails -deps = - -rrequirements/development.txt - -[tox] -envlist = - cypress-dashboard - cypress-explore - cypress-sqllab - cypress-sqllab-backend-persist - eslint - fossa - javascript - license-check - pre-commit - pylint -skipsdist = true