Fix for BigQuery connection checks and CSV uploads (#8511)

* Fix for BigQuery connection checks and CSV uploads

* Don't assume encrypted_extra will be populated

* Fix undefined method error

* Refactor to avoid circular import strangeness
This commit is contained in:
Will Barrett
2019-11-19 14:50:47 -08:00
committed by Maxime Beauchemin
parent 3b97ae3b9d
commit d70e0fc359
6 changed files with 25 additions and 12 deletions

View File

@@ -182,6 +182,7 @@ class BigQueryEngineSpec(BaseEngineSpec):
"""
try:
import pandas_gbq
from google.oauth2 import service_account
except ImportError:
raise Exception(
"Could not import the library `pandas_gbq`, which is "
@@ -191,10 +192,17 @@ class BigQueryEngineSpec(BaseEngineSpec):
if not ("name" in kwargs and "schema" in kwargs):
raise Exception("name and schema need to be defined in kwargs")
gbq_kwargs = {}
gbq_kwargs["project_id"] = kwargs["con"].engine.url.host
gbq_kwargs["destination_table"] = f"{kwargs.pop('schema')}.{kwargs.pop('name')}"
# add credentials if they are set on the SQLAlchemy Dialect:
creds = kwargs["con"].dialect.credentials_info
if creds:
credentials = service_account.Credentials.from_service_account_info(creds)
gbq_kwargs["credentials"] = credentials
# Only pass through supported kwargs
supported_kwarg_keys = {"if_exists"}
for key in supported_kwarg_keys: