diff --git a/.env.example b/.env.example index f6804fff4..ab12d9077 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,15 @@ SECRET_KEY_BASE=secret-value # Get it here: https://synthfinance.com/ SYNTH_API_KEY= +# Optional: Twelve Data API Key for exchange rates + stock prices +# (you can also set this in your self-hosted settings page) +# Get it here: https://twelvedata.com/ +TWELVE_DATA_API_KEY= + +# Optional: Twelve Data provider is the default for exchange rates and securities. +EXCHANGE_RATE_PROVIDER=twelve_data +SECURITIES_PROVIDER=twelve_data + # Custom port config # For users who have other applications listening at 3000, this allows them to set a value puma will listen to. PORT=3000 diff --git a/.env.local.example b/.env.local.example index d393f6234..88ce74ded 100644 --- a/.env.local.example +++ b/.env.local.example @@ -3,3 +3,4 @@ SELF_HOSTED=false # Enable Synth market data (careful, this will use your API credits) SYNTH_API_KEY=yourapikeyhere +TWELVE_DATA_API_KEY=yourapikeyhere diff --git a/Gemfile b/Gemfile index 33520c13e..2b08768a7 100644 --- a/Gemfile +++ b/Gemfile @@ -52,6 +52,7 @@ gem "ostruct" gem "bcrypt", "~> 3.1" gem "jwt" gem "jbuilder" +gem "countries" # OAuth & API Security gem "doorkeeper" diff --git a/Gemfile.lock b/Gemfile.lock index c393f5da1..d88d4703b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -140,6 +140,8 @@ GEM climate_control (1.2.0) concurrent-ruby (1.3.5) connection_pool (2.5.3) + countries (8.0.3) + unaccent (~> 0.3) crack (1.0.0) bigdecimal rexml @@ -576,6 +578,7 @@ GEM railties (>= 7.1.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + unaccent (0.4.0) unicode (0.4.4.5) unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) @@ -629,6 +632,7 @@ DEPENDENCIES brakeman capybara climate_control + countries csv debug derailed_benchmarks diff --git a/app/controllers/settings/hostings_controller.rb b/app/controllers/settings/hostings_controller.rb index 6eea6eccf..6702bbb94 100644 --- a/app/controllers/settings/hostings_controller.rb +++ b/app/controllers/settings/hostings_controller.rb @@ -8,6 +8,9 @@ class Settings::HostingsController < ApplicationController def show synth_provider = Provider::Registry.get_provider(:synth) @synth_usage = synth_provider&.usage + + twelve_data_provider = Provider::Registry.get_provider(:twelve_data) + @twelve_data_usage = twelve_data_provider&.usage end def update @@ -23,6 +26,10 @@ class Settings::HostingsController < ApplicationController Setting.synth_api_key = hosting_params[:synth_api_key] end + if hosting_params.key?(:twelve_data_api_key) + Setting.twelve_data_api_key = hosting_params[:twelve_data_api_key] + end + redirect_to settings_hosting_path, notice: t(".success") rescue ActiveRecord::RecordInvalid => error flash.now[:alert] = t(".failure") @@ -36,7 +43,7 @@ class Settings::HostingsController < ApplicationController private def hosting_params - params.require(:setting).permit(:require_invite_for_signup, :require_email_confirmation, :synth_api_key) + params.require(:setting).permit(:require_invite_for_signup, :require_email_confirmation, :synth_api_key, :twelve_data_api_key) end def ensure_admin diff --git a/app/models/exchange_rate/provided.rb b/app/models/exchange_rate/provided.rb index defee4210..4235bba83 100644 --- a/app/models/exchange_rate/provided.rb +++ b/app/models/exchange_rate/provided.rb @@ -3,8 +3,9 @@ module ExchangeRate::Provided class_methods do def provider + provider = ENV["EXCHANGE_RATE_PROVIDER"] || "twelve_data" registry = Provider::Registry.for_concept(:exchange_rates) - registry.get_provider(:synth) + registry.get_provider(provider.to_sym) end def find_or_fetch_rate(from:, to:, date: Date.current, cache: true) diff --git a/app/models/provider/registry.rb b/app/models/provider/registry.rb index c2d37db00..9d6de82f3 100644 --- a/app/models/provider/registry.rb +++ b/app/models/provider/registry.rb @@ -40,6 +40,14 @@ class Provider::Registry Provider::Synth.new(api_key) end + def twelve_data + api_key = ENV.fetch("TWELVE_DATA_API_KEY", Setting.twelve_data_api_key) + + return nil unless api_key.present? + + Provider::TwelveData.new(api_key) + end + def plaid_us config = Rails.application.config.plaid @@ -92,9 +100,9 @@ class Provider::Registry def available_providers case concept when :exchange_rates - %i[synth] + %i[synth twelve_data] when :securities - %i[synth] + %i[synth twelve_data] when :llm %i[openai] else diff --git a/app/models/provider/twelve_data.rb b/app/models/provider/twelve_data.rb new file mode 100644 index 000000000..ccb97ff64 --- /dev/null +++ b/app/models/provider/twelve_data.rb @@ -0,0 +1,195 @@ +class Provider::TwelveData < Provider + include ExchangeRateConcept, SecurityConcept + + # Subclass so errors caught in this provider are raised as Provider::TwelveData::Error + Error = Class.new(Provider::Error) + InvalidExchangeRateError = Class.new(Error) + InvalidSecurityPriceError = Class.new(Error) + + def initialize(api_key) + @api_key = api_key + end + + def healthy? + with_provider_response do + response = client.get("#{base_url}/api_usage") + JSON.parse(response.body).dig("plan_category").present? + end + end + + def usage + with_provider_response do + response = client.get("#{base_url}/api_usage") + + parsed = JSON.parse(response.body) + + limit = parsed.dig("plan_daily_limit") + used = parsed.dig("daily_usage") + remaining = limit - used + + UsageData.new( + used: used, + limit: limit, + utilization: used / limit * 100, + plan: parsed.dig("plan_category"), + ) + end + end + + # ================================ + # Exchange Rates + # ================================ + + def fetch_exchange_rate(from:, to:, date:) + with_provider_response do + response = client.get("#{base_url}/exchange_rate") do |req| + req.params["symbol"] = "#{from}/#{to}" + req.params["date"] = date.to_s + end + + rate = JSON.parse(response.body).dig("rate") + + Rate.new(date: date.to_date, from:, to:, rate: rate) + end + end + + def fetch_exchange_rates(from:, to:, start_date:, end_date:) + with_provider_response do + response = client.get("#{base_url}/time_series") do |req| + req.params["symbol"] = "#{from}/#{to}" + req.params["start_date"] = start_date.to_s + req.params["end_date"] = end_date.to_s + req.params["interval"] = "1day" + end + + data = JSON.parse(response.body).dig("values") + data.map do |resp| + rate = resp.dig("close") + date = resp.dig("datetime") + if rate.nil? + Rails.logger.warn("#{self.class.name} returned invalid rate data for pair from: #{from} to: #{to} on: #{date}. Rate data: #{rate.inspect}") + next + end + + Rate.new(date: date.to_date, from:, to:, rate:) + end.compact + end + end + + # ================================ + # Securities + # ================================ + + def search_securities(symbol, country_code: nil, exchange_operating_mic: nil) + with_provider_response do + response = client.get("#{base_url}/symbol_search") do |req| + req.params["symbol"] = symbol + req.params["outputsize"] = 25 + end + + parsed = JSON.parse(response.body) + + parsed.dig("data").map do |security| + country = ISO3166::Country.find_country_by_any_name(security.dig("country")) + + Security.new( + symbol: security.dig("symbol"), + name: security.dig("instrument_name"), + logo_url: nil, + exchange_operating_mic: security.dig("mic_code"), + country_code: country ? country.alpha2 : nil + ) + end + end + end + + def fetch_security_info(symbol:, exchange_operating_mic:) + with_provider_response do + response = client.get("#{base_url}/profile") do |req| + req.params["symbol"] = symbol + req.params["mic_code"] = exchange_operating_mic + end + + profile = JSON.parse(response.body) + + response = client.get("#{base_url}/logo") do |req| + req.params["symbol"] = symbol + req.params["mic_code"] = exchange_operating_mic + end + + logo = JSON.parse(response.body) + + SecurityInfo.new( + symbol: symbol, + name: profile.dig("name"), + links: profile.dig("website"), + logo_url: logo.dig("url"), + description: profile.dig("description"), + kind: profile.dig("type"), + exchange_operating_mic: exchange_operating_mic + ) + end + end + + def fetch_security_price(symbol:, exchange_operating_mic: nil, date:) + with_provider_response do + historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date) + + raise ProviderError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.empty? + + historical_data.data.first + end + end + + def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:) + with_provider_response do + response = client.get("#{base_url}/time_series") do |req| + req.params["symbol"] = symbol + req.params["mic_code"] = exchange_operating_mic + req.params["start_date"] = start_date.to_s + req.params["end_date"] = end_date.to_s + req.params["interval"] = "1day" + end + + parsed = JSON.parse(response.body) + parsed.dig("values").map do |resp| + price = resp.dig("close") + date = resp.dig("datetime") + if price.nil? + Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date}. Price data: #{price.inspect}") + next + end + + Price.new( + symbol: symbol, + date: date.to_date, + price: price, + currency: parsed.dig("currency"), + exchange_operating_mic: exchange_operating_mic + ) + end.compact + end + end + + private + attr_reader :api_key + + def base_url + ENV["TWELVE_DATA_URL"] || "https://api.twelvedata.com" + end + + def client + @client ||= Faraday.new(url: base_url) do |faraday| + faraday.request(:retry, { + max: 2, + interval: 0.05, + interval_randomness: 0.5, + backoff_factor: 2 + }) + + faraday.request :json + faraday.response :raise_error + faraday.headers["Authorization"] = "apikey #{api_key}" + end + end +end diff --git a/app/models/security/provided.rb b/app/models/security/provided.rb index 6f490a574..4ff84425e 100644 --- a/app/models/security/provided.rb +++ b/app/models/security/provided.rb @@ -5,8 +5,9 @@ module Security::Provided class_methods do def provider + provider = ENV["SECURITIES_PROVIDER"] || "twelve_data" registry = Provider::Registry.for_concept(:securities) - registry.get_provider(:synth) + registry.get_provider(provider.to_sym) end def search_provider(symbol, country_code: nil, exchange_operating_mic: nil) diff --git a/app/models/setting.rb b/app/models/setting.rb index 5f44284ad..522ddb7c8 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -3,6 +3,7 @@ class Setting < RailsSettings::Base cache_prefix { "v1" } field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"] + field :twelve_data_api_key, type: :string, default: ENV["TWELVE_DATA_API_KEY"] field :openai_access_token, type: :string, default: ENV["OPENAI_ACCESS_TOKEN"] field :require_invite_for_signup, type: :boolean, default: false diff --git a/app/views/securities/_combobox_security.turbo_stream.erb b/app/views/securities/_combobox_security.turbo_stream.erb index ef2fd8ba0..979551c0d 100644 --- a/app/views/securities/_combobox_security.turbo_stream.erb +++ b/app/views/securities/_combobox_security.turbo_stream.erb @@ -1,5 +1,7 @@
- <%= image_tag(combobox_security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %> + <% if combobox_security.logo_url.present? %> + <%= image_tag(combobox_security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %> + <% end %>
diff --git a/app/views/settings/_section.html.erb b/app/views/settings/_section.html.erb index 01ed0e956..5a8c9633a 100644 --- a/app/views/settings/_section.html.erb +++ b/app/views/settings/_section.html.erb @@ -6,7 +6,7 @@

<%= subtitle %>

<% end %>
-
+
<%= content %>
diff --git a/app/views/settings/hostings/_twelve_data_settings.html.erb b/app/views/settings/hostings/_twelve_data_settings.html.erb new file mode 100644 index 000000000..1bf4862c9 --- /dev/null +++ b/app/views/settings/hostings/_twelve_data_settings.html.erb @@ -0,0 +1,49 @@ +
+
+

<%= t(".title") %>

+ <% if ENV["TWELVE_DATA_API_KEY"].present? %> +

You have successfully configured your Twelve Data API key through the TWELVE_DATA_API_KEY environment variable.

+ <% else %> +

<%= t(".description") %>

+ <% end %> +
+ + <%= styled_form_with model: Setting.new, + url: settings_hosting_path, + method: :patch, + data: { + controller: "auto-submit-form", + "auto-submit-form-trigger-event-value": "blur" + } do |form| %> + <%= form.text_field :twelve_data_api_key, + label: t(".label"), + type: "password", + placeholder: t(".placeholder"), + value: ENV.fetch("TWELVE_DATA_API_KEY", Setting.twelve_data_api_key), + disabled: ENV["TWELVE_DATA_API_KEY"].present?, + container_class: @twelve_data_usage.present? && !@twelve_data_usage.success? ? "border-red-500" : "", + data: { "auto-submit-form-target": "auto" } %> + <% end %> + + <% if @twelve_data_usage.present? && @twelve_data_usage.success? %> +
+
+

+ <%= t(".api_calls_used", + used: number_with_delimiter(@twelve_data_usage.data.used), + limit: number_with_delimiter(@twelve_data_usage.data.limit), + percentage: number_to_percentage(@twelve_data_usage.data.utilization, precision: 1)) %> +

+
+
+
+
+
+

+ <%= t(".plan", plan: @twelve_data_usage.data.plan) %> +

+
+
+ <% end %> +
diff --git a/app/views/settings/hostings/show.html.erb b/app/views/settings/hostings/show.html.erb index 8f944323a..0b2a19164 100644 --- a/app/views/settings/hostings/show.html.erb +++ b/app/views/settings/hostings/show.html.erb @@ -4,6 +4,9 @@
<%= render "settings/hostings/synth_settings" %>
+
+ <%= render "settings/hostings/twelve_data_settings" %> +
<% end %> <%= settings_section title: t(".invites") do %> diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index 7377c492a..c93647646 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -28,6 +28,13 @@ en: placeholder: Enter your API key here plan: "%{plan} plan" title: Synth Settings + twelve_data_settings: + api_calls_used: "%{used} / %{limit} API daily calls used (%{percentage})" + description: Input the API key provided by Twelve Data + label: API Key + placeholder: Enter your API key here + plan: "%{plan} plan" + title: Twelve Data Settings update: failure: Invalid setting value success: Settings updated diff --git a/test/controllers/settings/hostings_controller_test.rb b/test/controllers/settings/hostings_controller_test.rb index 4e10e901a..ef672d3d7 100644 --- a/test/controllers/settings/hostings_controller_test.rb +++ b/test/controllers/settings/hostings_controller_test.rb @@ -9,6 +9,7 @@ class Settings::HostingsControllerTest < ActionDispatch::IntegrationTest @provider = mock Provider::Registry.stubs(:get_provider).with(:synth).returns(@provider) + Provider::Registry.stubs(:get_provider).with(:twelve_data).returns(@provider) @usage_response = provider_success_response( OpenStruct.new( used: 10, @@ -31,6 +32,7 @@ class Settings::HostingsControllerTest < ActionDispatch::IntegrationTest test "should get edit when self hosting is enabled" do @provider.expects(:usage).returns(@usage_response) + @provider.expects(:usage).returns(@usage_response) with_self_hosting do get settings_hosting_url diff --git a/test/models/account/market_data_importer_test.rb b/test/models/account/market_data_importer_test.rb index 74c42d36a..005411655 100644 --- a/test/models/account/market_data_importer_test.rb +++ b/test/models/account/market_data_importer_test.rb @@ -18,7 +18,7 @@ class Account::MarketDataImporterTest < ActiveSupport::TestCase @provider = mock("provider") Provider::Registry.any_instance .stubs(:get_provider) - .with(:synth) + .with(:twelve_data) .returns(@provider) end diff --git a/test/models/market_data_importer_test.rb b/test/models/market_data_importer_test.rb index b39bf0ad1..b5d6854ef 100644 --- a/test/models/market_data_importer_test.rb +++ b/test/models/market_data_importer_test.rb @@ -17,7 +17,7 @@ class MarketDataImporterTest < ActiveSupport::TestCase @provider = mock("provider") Provider::Registry.any_instance .stubs(:get_provider) - .with(:synth) + .with(:twelve_data) .returns(@provider) end diff --git a/test/system/settings_test.rb b/test/system/settings_test.rb index da9abeab4..028aad18d 100644 --- a/test/system/settings_test.rb +++ b/test/system/settings_test.rb @@ -34,6 +34,7 @@ class SettingsTest < ApplicationSystemTestCase test "can update self hosting settings" do Rails.application.config.app_mode.stubs(:self_hosted?).returns(true) Provider::Registry.stubs(:get_provider).with(:synth).returns(nil) + Provider::Registry.stubs(:get_provider).with(:twelve_data).returns(nil) open_settings_from_sidebar assert_selector "li", text: "Self hosting" click_link "Self hosting"