diff --git a/requirements.txt b/requirements.txt index b79d2a7c1c0..fae62165dba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -80,6 +80,7 @@ retry==0.9.2 # via apache-superset (setup.py) selenium==3.141.0 # via apache-superset (setup.py) simplejson==3.17.0 # via apache-superset (setup.py) six==1.14.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json +slackclient==2.6.2 # via apache-superset (setup.py) sqlalchemy-utils==0.36.4 # via apache-superset (setup.py), flask-appbuilder sqlalchemy==1.3.16 # via alembic, apache-superset (setup.py), flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils sqlparse==0.3.1 # via apache-superset (setup.py) diff --git a/setup.py b/setup.py index 16b0828afc4..7ebdb02ac76 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,7 @@ setup( "retry>=0.9.2", "selenium>=3.141.0", "simplejson>=3.15.0", + "slackclient>=2.6.2", "sqlalchemy>=1.3.16, <2.0", # Breaking change in sqlalchemy-utils==0.36.6, upgrading will probably # require a migration on EncryptedType columns. For more information, see diff --git a/superset/config.py b/superset/config.py index 6da24b46345..84d11a603af 100644 --- a/superset/config.py +++ b/superset/config.py @@ -737,6 +737,10 @@ ENABLE_FLASK_COMPRESS = True # Enable / disable scheduled email reports ENABLE_SCHEDULED_EMAIL_REPORTS = False +# Slack API token for the superset reports +SLACK_API_TOKEN = None +SLACK_PROXY = None + # If enabled, certail features are run in debug mode # Current list: # * Emails are sent using dry-run mode (logging only) diff --git a/superset/migrations/versions/743a117f0d98_add_slack_to_the_schedule.py b/superset/migrations/versions/743a117f0d98_add_slack_to_the_schedule.py new file mode 100644 index 00000000000..2e72a01cbcf --- /dev/null +++ b/superset/migrations/versions/743a117f0d98_add_slack_to_the_schedule.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Add slack to the schedule + +Revision ID: 743a117f0d98 +Revises: 620241d1153f +Create Date: 2020-05-13 21:01:26.163478 + +""" + +# revision identifiers, used by Alembic. +revision = "743a117f0d98" +down_revision = "620241d1153f" + +import sqlalchemy as sa +from alembic import op + + +def upgrade(): + op.add_column( + "dashboard_email_schedules", + sa.Column("slack_channel", sa.Text(), nullable=True), + ) + op.add_column( + "slice_email_schedules", sa.Column("slack_channel", sa.Text(), nullable=True) + ) + + +def downgrade(): + op.drop_column("dashboard_email_schedules", "slack_channel") + op.drop_column("slice_email_schedules", "slack_channel") diff --git a/superset/models/schedules.py b/superset/models/schedules.py index 0eb31a5d76b..f66a23768b3 100644 --- a/superset/models/schedules.py +++ b/superset/models/schedules.py @@ -67,6 +67,7 @@ class EmailSchedule: ) recipients = Column(Text) + slack_channel = Column(Text) deliver_as_group = Column(Boolean, default=False) delivery_type = Column(Enum(EmailDeliveryType)) diff --git a/superset/tasks/schedules.py b/superset/tasks/schedules.py index 3e6c1ddc779..3930bebafd3 100644 --- a/superset/tasks/schedules.py +++ b/superset/tasks/schedules.py @@ -41,15 +41,15 @@ from werkzeug.http import parse_cookie # Superset framework imports from superset import app, db, security_manager from superset.extensions import celery_app +from superset.models.dashboard import Dashboard from superset.models.schedules import ( - DashboardEmailSchedule, EmailDeliveryType, - EmailSchedule, get_scheduler_model, ScheduleType, SliceEmailReportFormat, - SliceEmailSchedule, ) +from superset.models.slice import Slice +from superset.tasks.slack_util import deliver_slack_msg from superset.utils.core import get_email_address_list, send_email_smtp if TYPE_CHECKING: @@ -66,47 +66,71 @@ EMAIL_PAGE_RENDER_WAIT = config["EMAIL_PAGE_RENDER_WAIT"] WEBDRIVER_BASEURL = config["WEBDRIVER_BASEURL"] WEBDRIVER_BASEURL_USER_FRIENDLY = config["WEBDRIVER_BASEURL_USER_FRIENDLY"] -EmailContent = namedtuple("EmailContent", ["body", "data", "images"]) +ReportContent = namedtuple( + "EmailContent", + [ + "body", # email body + "data", # attachments + "images", # embedded images for the email + "slack_message", # html not supported, only markdown + # attachments for the slack message, embedding not supported + "slack_attachment", + ], +) -def _get_recipients( - schedule: Union[DashboardEmailSchedule, SliceEmailSchedule] +def _get_email_to_and_bcc( + recipients: str, deliver_as_group: bool ) -> Iterator[Tuple[str, str]]: bcc = config["EMAIL_REPORT_BCC_ADDRESS"] - if schedule.deliver_as_group: - to = schedule.recipients + if deliver_as_group: + to = recipients yield (to, bcc) else: - for to in get_email_address_list(schedule.recipients): + for to in get_email_address_list(recipients): yield (to, bcc) -def _deliver_email( - schedule: Union[DashboardEmailSchedule, SliceEmailSchedule], +# TODO(bkyryliuk): move email functionality into a separate module. +def _deliver_email( # pylint: disable=too-many-arguments + recipients: str, + deliver_as_group: bool, subject: str, - email: EmailContent, + body: str, + data: Optional[Dict[str, Any]], + images: Optional[Dict[str, str]], ) -> None: - for (to, bcc) in _get_recipients(schedule): + for (to, bcc) in _get_email_to_and_bcc(recipients, deliver_as_group): send_email_smtp( to, subject, - email.body, + body, config, - data=email.data, - images=email.images, + data=data, + images=images, bcc=bcc, mime_subtype="related", dryrun=config["SCHEDULED_EMAIL_DEBUG_MODE"], ) -def _generate_mail_content( - schedule: EmailSchedule, screenshot: bytes, name: str, url: str -) -> EmailContent: +def _generate_report_content( + delivery_type: EmailDeliveryType, screenshot: bytes, name: str, url: str +) -> ReportContent: data: Optional[Dict[str, Any]] - if schedule.delivery_type == EmailDeliveryType.attachment: + # how to: https://api.slack.com/reference/surfaces/formatting + slack_message = __( + """ + *%(name)s*\n + <%(url)s|Explore in Superset> + """, + name=name, + url=url, + ) + + if delivery_type == EmailDeliveryType.attachment: images = None data = {"screenshot.png": screenshot} body = __( @@ -114,7 +138,7 @@ def _generate_mail_content( name=name, url=url, ) - elif schedule.delivery_type == EmailDeliveryType.inline: + elif delivery_type == EmailDeliveryType.inline: # Get the domain from the 'From' address .. # and make a message id without the < > in the ends domain = parseaddr(config["SMTP_MAIL_FROM"])[1].split("@")[1] @@ -132,7 +156,7 @@ def _generate_mail_content( msgid=msgid, ) - return EmailContent(body, data, images) + return ReportContent(body, data, images, slack_message, screenshot) def _get_auth_cookies() -> List["TypeConversionDict[Any, Any]"]: @@ -223,11 +247,18 @@ def destroy_webdriver( pass -def deliver_dashboard(schedule: DashboardEmailSchedule) -> None: +def deliver_dashboard( + dashboard_id: int, + recipients: Optional[str], + slack_channel: Optional[str], + delivery_type: EmailDeliveryType, + deliver_as_group: bool, +) -> None: + """ Given a schedule, delivery the dashboard as an email report """ - dashboard = schedule.dashboard + dashboard = db.session.query(Dashboard).filter_by(id=dashboard_id).one() dashboard_url = _get_url_path( "Superset.dashboard", dashboard_id_or_slug=dashboard.id @@ -260,8 +291,11 @@ def deliver_dashboard(schedule: DashboardEmailSchedule) -> None: destroy_webdriver(driver) # Generate the email body and attachments - email = _generate_mail_content( - schedule, screenshot, dashboard.dashboard_title, dashboard_url_user_friendly + report_content = _generate_report_content( + delivery_type, + screenshot, + dashboard.dashboard_title, + dashboard_url_user_friendly, ) subject = __( @@ -270,12 +304,25 @@ def deliver_dashboard(schedule: DashboardEmailSchedule) -> None: title=dashboard.dashboard_title, ) - _deliver_email(schedule, subject, email) + if recipients: + _deliver_email( + recipients, + deliver_as_group, + subject, + report_content.body, + report_content.data, + report_content.images, + ) + if slack_channel: + deliver_slack_msg( + slack_channel, + subject, + report_content.slack_message, + report_content.slack_attachment, + ) -def _get_slice_data(schedule: SliceEmailSchedule) -> EmailContent: - slc = schedule.slice - +def _get_slice_data(slc: Slice, delivery_type: EmailDeliveryType) -> ReportContent: slice_url = _get_url_path( "Superset.explore_json", csv="true", form_data=json.dumps({"slice_id": slc.id}) ) @@ -299,7 +346,7 @@ def _get_slice_data(schedule: SliceEmailSchedule) -> EmailContent: content = response.read() rows = [r.split(b",") for r in content.splitlines()] - if schedule.delivery_type == EmailDeliveryType.inline: + if delivery_type == EmailDeliveryType.inline: data = None # Parse the csv file and generate HTML @@ -313,7 +360,7 @@ def _get_slice_data(schedule: SliceEmailSchedule) -> EmailContent: link=slice_url_user_friendly, ) - elif schedule.delivery_type == EmailDeliveryType.attachment: + elif delivery_type == EmailDeliveryType.attachment: data = {__("%(name)s.csv", name=slc.slice_name): content} body = __( 'Explore in Superset
', @@ -321,12 +368,22 @@ def _get_slice_data(schedule: SliceEmailSchedule) -> EmailContent: url=slice_url_user_friendly, ) - return EmailContent(body, data, None) + # how to: https://api.slack.com/reference/surfaces/formatting + slack_message = __( + """ + *%(slice_name)s*\n + <%(slice_url_user_friendly)s|Explore in Superset> + """, + slice_name=slc.slice_name, + slice_url_user_friendly=slice_url_user_friendly, + ) + + return ReportContent(body, data, None, slack_message, content) -def _get_slice_visualization(schedule: SliceEmailSchedule) -> EmailContent: - slc = schedule.slice - +def _get_slice_visualization( + slc: Slice, delivery_type: EmailDeliveryType +) -> ReportContent: # Create a driver, fetch the page, wait for the page to render driver = create_webdriver() window = config["WEBDRIVER_WINDOW"]["slice"] @@ -359,29 +416,53 @@ def _get_slice_visualization(schedule: SliceEmailSchedule) -> EmailContent: destroy_webdriver(driver) # Generate the email body and attachments - return _generate_mail_content( - schedule, screenshot, slc.slice_name, slice_url_user_friendly + return _generate_report_content( + delivery_type, screenshot, slc.slice_name, slice_url_user_friendly ) -def deliver_slice(schedule: Union[DashboardEmailSchedule, SliceEmailSchedule]) -> None: +def deliver_slice( # pylint: disable=too-many-arguments + slice_id: int, + recipients: Optional[str], + slack_channel: Optional[str], + delivery_type: EmailDeliveryType, + email_format: SliceEmailReportFormat, + deliver_as_group: bool, +) -> None: """ Given a schedule, delivery the slice as an email report """ - if schedule.email_format == SliceEmailReportFormat.data: - email = _get_slice_data(schedule) - elif schedule.email_format == SliceEmailReportFormat.visualization: - email = _get_slice_visualization(schedule) + slc = db.session.query(Slice).filter_by(id=slice_id).one() + + if email_format == SliceEmailReportFormat.data: + report_content = _get_slice_data(slc, delivery_type) + elif email_format == SliceEmailReportFormat.visualization: + report_content = _get_slice_visualization(slc, delivery_type) else: raise RuntimeError("Unknown email report format") subject = __( "%(prefix)s %(title)s", prefix=config["EMAIL_REPORTS_SUBJECT_PREFIX"], - title=schedule.slice.slice_name, + title=slc.slice_name, ) - _deliver_email(schedule, subject, email) + if recipients: + _deliver_email( + recipients, + deliver_as_group, + subject, + report_content.body, + report_content.data, + report_content.images, + ) + if slack_channel: + deliver_slack_msg( + slack_channel, + subject, + report_content.slack_message, + report_content.slack_attachment, + ) @celery_app.task( @@ -394,6 +475,7 @@ def schedule_email_report( # pylint: disable=unused-argument report_type: ScheduleType, schedule_id: int, recipients: Optional[str] = None, + slack_channel: Optional[str] = None, ) -> None: model_cls = get_scheduler_model(report_type) schedule = db.create_scoped_session().query(model_cls).get(schedule_id) @@ -403,15 +485,29 @@ def schedule_email_report( # pylint: disable=unused-argument logger.info("Ignoring deactivated schedule") return - # TODO: Detach the schedule object from the db session - if recipients is not None: - schedule.id = schedule_id - schedule.recipients = recipients + recipients = recipients or schedule.recipients + slack_channel = slack_channel or schedule.slack_channel + logger.info( + f"Starting report for slack: {slack_channel} and recipients: {recipients}." + ) if report_type == ScheduleType.dashboard: - deliver_dashboard(schedule) + deliver_dashboard( + schedule.dashboard_id, + recipients, + slack_channel, + schedule.delivery_type, + schedule.deliver_as_group, + ) elif report_type == ScheduleType.slice: - deliver_slice(schedule) + deliver_slice( + schedule.slice_id, + recipients, + slack_channel, + schedule.delivery_type, + schedule.email_format, + schedule.deliver_as_group, + ) else: raise RuntimeError("Unknown report type") diff --git a/superset/tasks/slack_util.py b/superset/tasks/slack_util.py new file mode 100644 index 00000000000..018308d2beb --- /dev/null +++ b/superset/tasks/slack_util.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import logging +from io import IOBase +from typing import cast, Union + +from retry.api import retry +from slack import WebClient +from slack.errors import SlackApiError +from slack.web.slack_response import SlackResponse + +from superset import app + +# Globals +config = app.config # type: ignore +logger = logging.getLogger("tasks.slack_util") + + +@retry(SlackApiError, delay=10, backoff=2, tries=5) +def deliver_slack_msg( + slack_channel: str, subject: str, body: str, file: Union[str, IOBase] +) -> None: + client = WebClient(token=config["SLACK_API_TOKEN"], proxy=config["SLACK_PROXY"]) + # files_upload returns SlackResponse as we run it in sync mode. + response = cast( + SlackResponse, + client.files_upload( + channels=slack_channel, file=file, initial_comment=body, title=subject + ), + ) + logger.info(f"Sent the report to the slack {slack_channel}") + assert response["file"], str(response) # the uploaded file diff --git a/superset/views/schedules.py b/superset/views/schedules.py index de09c3141cc..d98c339b60e 100644 --- a/superset/views/schedules.py +++ b/superset/views/schedules.py @@ -90,6 +90,11 @@ class EmailScheduleView( description="List of recipients to send test email to. " "If empty, we send it to the original recipients", ), + "test_slack_channel": StringField( + "Test Slack Channel", + default=None, + description="A slack channel to send a test message to.", + ), } edit_form_extra_fields = add_form_extra_fields @@ -99,8 +104,16 @@ class EmailScheduleView( test_email_recipients = form.test_email_recipients.data.strip() else: test_email_recipients = None + + test_slack_channel = ( + form.test_slack_channel.data.strip() + if form.test_slack_channel.data + else None + ) + self._extra_data["test_email"] = form.test_email.data self._extra_data["test_email_recipients"] = test_email_recipients + self._extra_data["test_slack_channel"] = test_slack_channel def pre_add(self, item: "EmailScheduleView") -> None: try: @@ -120,8 +133,9 @@ class EmailScheduleView( # Schedule a test mail if the user requested for it. if self._extra_data["test_email"]: recipients = self._extra_data["test_email_recipients"] or item.recipients + slack_channel = self._extra_data["test_slack_channel"] or item.slack_channel args = (self.schedule_type, item.id) - kwargs = dict(recipients=recipients) + kwargs = dict(recipients=recipients, slack_channel=slack_channel) schedule_email_report.apply_async(args=args, kwargs=kwargs) # Notify the user that schedule changes will be activate only in the @@ -187,10 +201,12 @@ class DashboardEmailScheduleView( "active", "crontab", "recipients", + "slack_channel", "deliver_as_group", "delivery_type", "test_email", "test_email_recipients", + "test_slack_channel", ] edit_columns = add_columns @@ -211,6 +227,7 @@ class DashboardEmailScheduleView( "active": _("Active"), "crontab": _("Crontab"), "recipients": _("Recipients"), + "slack_channel": _("Slack Channel"), "deliver_as_group": _("Deliver As Group"), "delivery_type": _("Delivery Type"), } @@ -245,11 +262,13 @@ class SliceEmailScheduleView(EmailScheduleView): # pylint: disable=too-many-anc "active", "crontab", "recipients", + "slack_channel", "deliver_as_group", "delivery_type", "email_format", "test_email", "test_email_recipients", + "test_slack_channel", ] edit_columns = add_columns @@ -271,6 +290,7 @@ class SliceEmailScheduleView(EmailScheduleView): # pylint: disable=too-many-anc "active": _("Active"), "crontab": _("Crontab"), "recipients": _("Recipients"), + "slack_channel": _("Slack Channel"), "deliver_as_group": _("Deliver As Group"), "delivery_type": _("Delivery Type"), "email_format": _("Email Format"), diff --git a/tests/schedules_test.py b/tests/schedules_test.py index 7528fadb088..348ee862ad3 100644 --- a/tests/schedules_test.py +++ b/tests/schedules_test.py @@ -72,6 +72,7 @@ class SchedulesTestCase(SupersetTestCase): slice_schedule.slice_id = slce.id slice_schedule.user_id = 1 slice_schedule.email_format = SliceEmailReportFormat.data + slice_schedule.slack_channel = "#test_channel" db.session.add(slice_schedule) db.session.commit() @@ -190,7 +191,14 @@ class SchedulesTestCase(SupersetTestCase): .all()[0] ) - deliver_dashboard(schedule) + deliver_dashboard( + schedule.dashboard_id, + schedule.recipients, + schedule.slack_channel, + schedule.delivery_type, + schedule.deliver_as_group, + ) + mtime.sleep.assert_called_once() driver.screenshot.assert_not_called() send_email_smtp.assert_called_once() @@ -220,7 +228,14 @@ class SchedulesTestCase(SupersetTestCase): ) schedule.delivery_type = EmailDeliveryType.attachment - deliver_dashboard(schedule) + + deliver_dashboard( + schedule.dashboard_id, + schedule.recipients, + schedule.slack_channel, + schedule.delivery_type, + schedule.deliver_as_group, + ) mtime.sleep.assert_called_once() driver.screenshot.assert_not_called() @@ -256,7 +271,14 @@ class SchedulesTestCase(SupersetTestCase): .all()[0] ) - deliver_dashboard(schedule) + deliver_dashboard( + schedule.dashboard_id, + schedule.recipients, + schedule.slack_channel, + schedule.delivery_type, + schedule.deliver_as_group, + ) + mtime.sleep.assert_called_once() driver.screenshot.assert_called_once() send_email_smtp.assert_called_once() @@ -294,17 +316,27 @@ class SchedulesTestCase(SupersetTestCase): # Set a bcc email address app.config["EMAIL_REPORT_BCC_ADDRESS"] = self.BCC - deliver_dashboard(schedule) + deliver_dashboard( + schedule.dashboard_id, + schedule.recipients, + schedule.slack_channel, + schedule.delivery_type, + schedule.deliver_as_group, + ) + mtime.sleep.assert_called_once() driver.screenshot.assert_not_called() self.assertEqual(send_email_smtp.call_count, 2) self.assertEqual(send_email_smtp.call_args[1]["bcc"], self.BCC) + @patch("superset.tasks.slack_util.WebClient.files_upload") @patch("superset.tasks.schedules.firefox.webdriver.WebDriver") @patch("superset.tasks.schedules.send_email_smtp") @patch("superset.tasks.schedules.time") - def test_deliver_slice_inline_image(self, mtime, send_email_smtp, driver_class): + def test_deliver_slice_inline_image( + self, mtime, send_email_smtp, driver_class, files_upload + ): element = Mock() driver = Mock() mtime.sleep.return_value = None @@ -325,7 +357,14 @@ class SchedulesTestCase(SupersetTestCase): schedule.email_format = SliceEmailReportFormat.visualization schedule.delivery_format = EmailDeliveryType.inline - deliver_slice(schedule) + deliver_slice( + schedule.slice_id, + schedule.recipients, + schedule.slack_channel, + schedule.delivery_type, + schedule.email_format, + schedule.deliver_as_group, + ) mtime.sleep.assert_called_once() driver.screenshot.assert_not_called() send_email_smtp.assert_called_once() @@ -335,10 +374,23 @@ class SchedulesTestCase(SupersetTestCase): element.screenshot_as_png, ) + self.assertEqual( + files_upload.call_args[1], + { + "channels": "#test_channel", + "file": element.screenshot_as_png, + "initial_comment": "\n *Participants*\n\n