[utils.py] gathering/refactoring into a "utils/" folder (#6095)

* [utils] gathering/refactoring into a "utils/" folder

Moving current utils.py into utils/core.py and moving other *util*
modules under this new "utils/" as well.

Following steps include eroding at "utils/core.py" and breaking it down
into smaller modules.

* Improve tests

* Make loading examples in scope for tests

* Remove test class attrs examples_loaded and requires_examples

superset/data/__init__.py

@@ -7,15 +7,16 @@ import os
 import random
 import textwrap
 
-import pandas as pd
-from sqlalchemy import BigInteger, Date, DateTime, Float, String, Text
 import geohash
+import pandas as pd
 import polyline
+from sqlalchemy import BigInteger, Date, DateTime, Float, String, Text
 
-from superset import app, db, utils
+from superset import app, db
 from superset.connectors.connector_registry import ConnectorRegistry
 from superset.connectors.sqla.models import TableColumn
 from superset.models import core as models
+from superset.utils.core import get_or_create_main_db, readfile
 
 # Shortcuts
 DB = models.Database
@@ -26,7 +27,7 @@ TBL = ConnectorRegistry.sources['table']
 
 config = app.config
 
-DATA_FOLDER = os.path.join(config.get("BASE_DIR"), 'data')
+DATA_FOLDER = os.path.join(config.get('BASE_DIR'), 'data')
 
 misc_dash_slices = set()  # slices assembled in a "Misc Chart" dashboard
 
@@ -38,7 +39,8 @@ def update_slice_ids(layout_dict, slices):
     ]
     sorted_charts = sorted(charts, key=lambda k: k['meta']['chartId'])
     for i, chart_component in enumerate(sorted_charts):
-        chart_component['meta']['chartId'] = int(slices[i].id)
+        if i < len(slices):
+            chart_component['meta']['chartId'] = int(slices[i].id)
 
 
 def merge_slice(slc):
@@ -77,7 +79,7 @@ def load_energy():
     if not tbl:
         tbl = TBL(table_name=tbl_name)
     tbl.description = "Energy consumption"
-    tbl.database = utils.get_or_create_main_db()
+    tbl.database = get_or_create_main_db()
     db.session.merge(tbl)
     db.session.commit()
     tbl.fetch_metadata()
@@ -183,9 +185,9 @@ def load_world_bank_health_n_pop():
     tbl = db.session.query(TBL).filter_by(table_name=tbl_name).first()
     if not tbl:
         tbl = TBL(table_name=tbl_name)
-    tbl.description = utils.readfile(os.path.join(DATA_FOLDER, 'countries.md'))
+    tbl.description = readfile(os.path.join(DATA_FOLDER, 'countries.md'))
     tbl.main_dttm_col = 'year'
-    tbl.database = utils.get_or_create_main_db()
+    tbl.database = get_or_create_main_db()
     tbl.filter_select_enabled = True
     db.session.merge(tbl)
     db.session.commit()
@@ -723,7 +725,7 @@ def load_birth_names():
     if not obj:
         obj = TBL(table_name='birth_names')
     obj.main_dttm_col = 'ds'
-    obj.database = utils.get_or_create_main_db()
+    obj.database = get_or_create_main_db()
     obj.filter_select_enabled = True
 
     if not any(col.column_name == 'num_california' for col in obj.columns):
@@ -1256,7 +1258,7 @@ def load_unicode_test_data():
     if not obj:
         obj = TBL(table_name='unicode_test')
     obj.main_dttm_col = 'dttm'
-    obj.database = utils.get_or_create_main_db()
+    obj.database = get_or_create_main_db()
     db.session.merge(obj)
     db.session.commit()
     obj.fetch_metadata()
@@ -1369,7 +1371,7 @@ def load_random_time_series_data():
     if not obj:
         obj = TBL(table_name='random_time_series')
     obj.main_dttm_col = 'ds'
-    obj.database = utils.get_or_create_main_db()
+    obj.database = get_or_create_main_db()
     db.session.merge(obj)
     db.session.commit()
     obj.fetch_metadata()
@@ -1432,7 +1434,7 @@ def load_country_map_data():
     if not obj:
         obj = TBL(table_name='birth_france_by_region')
     obj.main_dttm_col = 'dttm'
-    obj.database = utils.get_or_create_main_db()
+    obj.database = get_or_create_main_db()
     db.session.merge(obj)
     db.session.commit()
     obj.fetch_metadata()
@@ -1507,7 +1509,7 @@ def load_long_lat_data():
     if not obj:
         obj = TBL(table_name='long_lat')
     obj.main_dttm_col = 'datetime'
-    obj.database = utils.get_or_create_main_db()
+    obj.database = get_or_create_main_db()
     db.session.merge(obj)
     db.session.commit()
     obj.fetch_metadata()
@@ -1568,7 +1570,7 @@ def load_multiformat_time_series_data():
     if not obj:
         obj = TBL(table_name='multiformat_time_series')
     obj.main_dttm_col = 'ds'
-    obj.database = utils.get_or_create_main_db()
+    obj.database = get_or_create_main_db()
     dttm_and_expr_dict = {
         'ds': [None, None],
         'ds2': [None, None],
@@ -2391,7 +2393,7 @@ def load_flights():
     if not tbl:
         tbl = TBL(table_name=tbl_name)
     tbl.description = "Random set of flights in the US"
-    tbl.database = utils.get_or_create_main_db()
+    tbl.database = get_or_create_main_db()
     db.session.merge(tbl)
     db.session.commit()
     tbl.fetch_metadata()
@@ -2422,7 +2424,7 @@ def load_paris_iris_geojson():
     if not tbl:
         tbl = TBL(table_name=tbl_name)
     tbl.description = "Map of Paris"
-    tbl.database = utils.get_or_create_main_db()
+    tbl.database = get_or_create_main_db()
     db.session.merge(tbl)
     db.session.commit()
     tbl.fetch_metadata()
@@ -2452,7 +2454,7 @@ def load_sf_population_polygons():
     if not tbl:
         tbl = TBL(table_name=tbl_name)
     tbl.description = "Population density of San Francisco"
-    tbl.database = utils.get_or_create_main_db()
+    tbl.database = get_or_create_main_db()
     db.session.merge(tbl)
     db.session.commit()
     tbl.fetch_metadata()
@@ -2482,7 +2484,7 @@ def load_bart_lines():
     if not tbl:
         tbl = TBL(table_name=tbl_name)
     tbl.description = "BART lines"
-    tbl.database = utils.get_or_create_main_db()
+    tbl.database = get_or_create_main_db()
     db.session.merge(tbl)
     db.session.commit()
     tbl.fetch_metadata()