Flask App factory PR #1 (#8418)

* First cut at app factory

* Setting things back to master

* Working with new FLASK_APP

* Still need to refactor Celery

* CLI mostly working

* Working on unit tests

* Moving cli stuff around a bit

* Removing get in config

* Defaulting test config

* Adding flask-testing

* flask-testing casing

* resultsbackend property bug

* Fixing up cli

* Quick fix for KV api

* Working on save slice

* Fixed core_tests

* Fixed utils_tests

* Most tests working - still need to dig into remaining app_context issue in tests

* All tests passing locally - need to update code comments

* Fixing dashboard tests again

* Blacking

* Sorting imports

* linting

* removing envvar mangling

* blacking

* Fixing unit tests

* isorting

* licensing

* fixing mysql tests

* fixing cypress?

* fixing .flaskenv

* fixing test app_ctx

* fixing cypress

* moving manifest processor around

* moving results backend manager around

* Cleaning up __init__ a bit more

* Addressing PR comments

* Addressing PR comments

* Blacking

* Fixes for running celery worker

* Tuning isort

* Blacking

superset/cli.py

@@ -25,27 +25,28 @@ import click
 import yaml
 from colorama import Fore, Style
 from flask import g
+from flask.cli import FlaskGroup, with_appcontext
 from flask_appbuilder import Model
 from pathlib2 import Path
 
-from superset import app, appbuilder, db, examples, security_manager
-from superset.common.tags import add_favorites, add_owners, add_types
-from superset.utils import core as utils, dashboard_import_export, dict_import_export
-
-config = app.config
-celery_app = utils.get_celery_app(config)
+from superset import app, appbuilder, security_manager
+from superset.app import create_app
+from superset.extensions import celery_app, db
+from superset.utils import core as utils
 
 
-def create_app(script_info=None):
-    return app
+@click.group(cls=FlaskGroup, create_app=create_app)
+@with_appcontext
+def superset():
+    """This is a management script for the Superset application."""
+
+    @app.shell_context_processor
+    def make_shell_context():
+        return dict(app=app, db=db)
 
 
-@app.shell_context_processor
-def make_shell_context():
-    return dict(app=app, db=db)
-
-
-@app.cli.command()
+@superset.command()
+@with_appcontext
 def init():
     """Inits the Superset application"""
     utils.get_example_database()
@@ -53,7 +54,8 @@ def init():
     security_manager.sync_role_definitions()
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option("--verbose", "-v", is_flag=True, help="Show extra information")
 def version(verbose):
     """Prints the current version number"""
@@ -62,7 +64,7 @@ def version(verbose):
         Fore.YELLOW
         + "Superset "
         + Fore.CYAN
-        + "{version}".format(version=config["VERSION_STRING"])
+        + "{version}".format(version=app.config["VERSION_STRING"])
     )
     print(Fore.BLUE + "-=" * 15)
     if verbose:
@@ -77,6 +79,8 @@ def load_examples_run(load_test_data, only_metadata=False, force=False):
         examples_db = utils.get_example_database()
         print(f"Loading examples metadata and related data into {examples_db}")
 
+    from superset import examples
+
     examples.load_css_templates()
 
     print("Loading energy related dataset")
@@ -129,7 +133,8 @@ def load_examples_run(load_test_data, only_metadata=False, force=False):
     examples.load_tabbed_dashboard(only_metadata)
 
 
-@app.cli.command()
+@with_appcontext
+@superset.command()
 @click.option("--load-test-data", "-t", is_flag=True, help="Load additional test data")
 @click.option(
     "--only-metadata", "-m", is_flag=True, help="Only load metadata, skip actual data"
@@ -142,7 +147,8 @@ def load_examples(load_test_data, only_metadata=False, force=False):
     load_examples_run(load_test_data, only_metadata, force)
 
 
-@app.cli.command()
+@with_appcontext
+@superset.command()
 @click.option("--database_name", "-d", help="Database name to change")
 @click.option("--uri", "-u", help="Database URI to change")
 def set_database_uri(database_name, uri):
@@ -150,7 +156,8 @@ def set_database_uri(database_name, uri):
     utils.get_or_create_db(database_name, uri)
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "--datasource",
     "-d",
@@ -180,7 +187,8 @@ def refresh_druid(datasource, merge):
     session.commit()
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "--path",
     "-p",
@@ -202,6 +210,8 @@ def refresh_druid(datasource, merge):
 )
 def import_dashboards(path, recursive, username):
     """Import dashboards from JSON"""
+    from superset.utils import dashboard_import_export
+
     p = Path(path)
     files = []
     if p.is_file():
@@ -222,7 +232,8 @@ def import_dashboards(path, recursive, username):
             logging.error(e)
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "--dashboard-file", "-f", default=None, help="Specify the the file to export to"
 )
@@ -231,6 +242,8 @@ def import_dashboards(path, recursive, username):
 )
 def export_dashboards(print_stdout, dashboard_file):
     """Export dashboards to JSON"""
+    from superset.utils import dashboard_import_export
+
     data = dashboard_import_export.export_dashboards(db.session)
     if print_stdout or not dashboard_file:
         print(data)
@@ -240,7 +253,8 @@ def export_dashboards(print_stdout, dashboard_file):
             data_stream.write(data)
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "--path",
     "-p",
@@ -265,6 +279,8 @@ def export_dashboards(print_stdout, dashboard_file):
 )
 def import_datasources(path, sync, recursive):
     """Import datasources from YAML"""
+    from superset.utils import dict_import_export
+
     sync_array = sync.split(",")
     p = Path(path)
     files = []
@@ -288,7 +304,8 @@ def import_datasources(path, sync, recursive):
             logging.error(e)
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "--datasource-file", "-f", default=None, help="Specify the the file to export to"
 )
@@ -313,6 +330,8 @@ def export_datasources(
     print_stdout, datasource_file, back_references, include_defaults
 ):
     """Export datasources to YAML"""
+    from superset.utils import dict_import_export
+
     data = dict_import_export.export_to_dict(
         session=db.session,
         recursive=True,
@@ -327,7 +346,8 @@ def export_datasources(
             yaml.safe_dump(data, data_stream, default_flow_style=False)
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "--back-references",
     "-b",
@@ -337,11 +357,14 @@ def export_datasources(
 )
 def export_datasource_schema(back_references):
     """Export datasource YAML schema to stdout"""
+    from superset.utils import dict_import_export
+
     data = dict_import_export.export_schema_to_dict(back_references=back_references)
     yaml.safe_dump(data, stdout, default_flow_style=False)
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 def update_datasources_cache():
     """Refresh sqllab datasources cache"""
     from superset.models.core import Database
@@ -360,7 +383,8 @@ def update_datasources_cache():
                 print("{}".format(str(e)))
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "--workers", "-w", type=int, help="Number of celery server workers to fire up"
 )
@@ -372,14 +396,17 @@ def worker(workers):
     )
     if workers:
         celery_app.conf.update(CELERYD_CONCURRENCY=workers)
-    elif config["SUPERSET_CELERY_WORKERS"]:
-        celery_app.conf.update(CELERYD_CONCURRENCY=config["SUPERSET_CELERY_WORKERS"])
+    elif app.config["SUPERSET_CELERY_WORKERS"]:
+        celery_app.conf.update(
+            CELERYD_CONCURRENCY=app.config["SUPERSET_CELERY_WORKERS"]
+        )
 
     worker = celery_app.Worker(optimization="fair")
     worker.start()
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 @click.option(
     "-p", "--port", default="5555", help="Port on which to start the Flower process"
 )
@@ -409,7 +436,8 @@ def flower(port, address):
     Popen(cmd, shell=True).wait()
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 def load_test_users():
     """
     Loads admin, alpha, and gamma user for testing purposes
@@ -426,7 +454,7 @@ def load_test_users_run():
 
     Syncs permissions for those users/roles
     """
-    if config["TESTING"]:
+    if app.config["TESTING"]:
 
         sm = security_manager
 
@@ -463,11 +491,15 @@ def load_test_users_run():
         sm.get_session.commit()
 
 
-@app.cli.command()
+@superset.command()
+@with_appcontext
 def sync_tags():
     """Rebuilds special tags (owner, type, favorited by)."""
     # pylint: disable=no-member
     metadata = Model.metadata
+
+    from superset.common.tags import add_favorites, add_owners, add_types
+
     add_types(db.engine, metadata)
     add_owners(db.engine, metadata)
     add_favorites(db.engine, metadata)