From 96c893ec189a413dd6e9e593ee6fa56b83c3c1c4 Mon Sep 17 00:00:00 2001 From: Michal Tajchert Date: Mon, 11 May 2026 23:09:21 +0200 Subject: [PATCH 01/31] Mobile: custom proxy headers + small login UX fixes (#1748) * Add mobile custom proxy headers * Clear login placeholders on focus Email/password fields ship with example values pre-filled. Tapping the field now clears the placeholder so users don't have to delete it manually. Skips clearing if the user has already edited the value. * Push Configuration as a route from Sign in Opening Configuration from the Sign in screen now uses Navigator.push instead of toggling a state flag, so Android back returns to Sign in instead of quitting the app. Saving the URL auto-pops the route. * Address PR review on custom proxy headers - Test Connection no longer leaves global ApiConfig headers mutated; unsaved edits are restored in a finally block after the probe. - _loadSavedUrl / _loadCustomHeaders wrap storage reads in try/catch and always finish initialization with sensible defaults. - Sanitization is now a single CustomProxyHeader.sanitize() reused by ApiConfig.setCustomProxyHeaders and CustomProxyHeadersService. - Brief comment on redactedValue explaining the length-obscuring design. * Harden custom proxy header validation and load path - validateValue now rejects ASCII control characters (CR/LF/tab/etc.) to prevent header-injection via crafted values. - loadHeaders moves the secure-storage read inside the try block so platform exceptions are caught the same way JSON parse errors are. --- mobile/lib/main.dart | 10 +- mobile/lib/models/custom_proxy_header.dart | 90 +++++++++++ mobile/lib/screens/backend_config_screen.dart | 83 ++++++++-- mobile/lib/screens/login_screen.dart | 42 ++++- mobile/lib/screens/settings_screen.dart | 116 ++++++++++++++ mobile/lib/services/api_config.dart | 48 +++++- mobile/lib/services/auth_service.dart | 31 +--- .../custom_proxy_headers_service.dart | 47 ++++++ .../widgets/custom_proxy_headers_editor.dart | 145 ++++++++++++++++++ .../test/models/custom_proxy_header_test.dart | 47 ++++++ .../services/api_config_headers_test.dart | 78 ++++++++++ .../custom_proxy_headers_service_test.dart | 47 ++++++ 12 files changed, 734 insertions(+), 50 deletions(-) create mode 100644 mobile/lib/models/custom_proxy_header.dart create mode 100644 mobile/lib/services/custom_proxy_headers_service.dart create mode 100644 mobile/lib/widgets/custom_proxy_headers_editor.dart create mode 100644 mobile/test/models/custom_proxy_header_test.dart create mode 100644 mobile/test/services/api_config_headers_test.dart create mode 100644 mobile/test/services/custom_proxy_headers_service_test.dart diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 8bb8457a0..bc1b4afc6 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -257,12 +257,6 @@ class _AppWrapperState extends State with WidgetsBindingObserver { }); } - void _goToBackendConfig() { - setState(() { - _hasBackendUrl = false; - }); - } - @override Widget build(BuildContext context) { if (_isCheckingConfig) { @@ -314,9 +308,7 @@ class _AppWrapperState extends State with WidgetsBindingObserver { return const SsoOnboardingScreen(); } - return LoginScreen( - onGoToSettings: _goToBackendConfig, - ); + return const LoginScreen(); }, ); } diff --git a/mobile/lib/models/custom_proxy_header.dart b/mobile/lib/models/custom_proxy_header.dart new file mode 100644 index 000000000..900bd2486 --- /dev/null +++ b/mobile/lib/models/custom_proxy_header.dart @@ -0,0 +1,90 @@ +class CustomProxyHeader { + static final RegExp _headerNamePattern = RegExp(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$"); + // Reject ASCII control bytes in values to block CR/LF header injection. + static final RegExp _headerValueControlChars = RegExp(r'[\x00-\x1F\x7F]'); + static const Set _reservedNames = { + 'accept', + 'authorization', + 'content-type', + 'x-api-key', + }; + + final String name; + final String value; + + CustomProxyHeader({ + required String name, + required String value, + }) : name = name.trim(), + value = value.trim(); + + factory CustomProxyHeader.fromJson(Map json) { + return CustomProxyHeader( + name: json['name'] as String? ?? '', + value: json['value'] as String? ?? '', + ); + } + + Map toJson() => { + 'name': name, + 'value': value, + }; + + String get normalizedName => name.toLowerCase(); + + // Length is intentionally obscured: short values get a fixed 4-bullet mask + // and longer values get a fixed 6-bullet prefix + last 4 chars. Keeping the + // last 4 lets users sanity-check what they entered without leaking length. + String get redactedValue { + if (value.isEmpty) return ''; + if (value.length <= 4) return '••••'; + return '••••••${value.substring(value.length - 4)}'; + } + + /// Drops headers with empty/invalid name or value, then dedupes by + /// case-insensitive name (last write wins). Single source of truth used by + /// both `ApiConfig.setCustomProxyHeaders` and the persistence service. + static List sanitize(List headers) { + final byName = {}; + for (final header in headers) { + if (!header.isComplete) continue; + if (validateName(header.name) != null) continue; + if (validateValue(header.value) != null) continue; + byName[header.normalizedName] = header; + } + return byName.values.toList(growable: false); + } + + bool get isComplete => name.isNotEmpty && value.isNotEmpty; + + static String? validateName(String value) { + final trimmed = value.trim(); + if (trimmed.isEmpty) return 'Header name is required'; + if (!_headerNamePattern.hasMatch(trimmed)) { + return 'Use a valid HTTP header name'; + } + if (_reservedNames.contains(trimmed.toLowerCase())) { + return 'This header is managed by the app'; + } + return null; + } + + static String? validateValue(String value) { + if (value.trim().isEmpty) return 'Header value is required'; + if (_headerValueControlChars.hasMatch(value)) { + return 'Header value contains control characters'; + } + return null; + } + + @override + bool operator ==(Object other) { + return identical(this, other) || + other is CustomProxyHeader && + name == other.name && + value == other.value; + } + + @override + int get hashCode => Object.hash(name, value); +} diff --git a/mobile/lib/screens/backend_config_screen.dart b/mobile/lib/screens/backend_config_screen.dart index edc50d774..b5cbf5519 100644 --- a/mobile/lib/screens/backend_config_screen.dart +++ b/mobile/lib/screens/backend_config_screen.dart @@ -1,7 +1,10 @@ import 'package:flutter/material.dart'; import 'package:shared_preferences/shared_preferences.dart'; import 'package:http/http.dart' as http; +import '../models/custom_proxy_header.dart'; import '../services/api_config.dart'; +import '../services/custom_proxy_headers_service.dart'; +import '../widgets/custom_proxy_headers_editor.dart'; class BackendConfigScreen extends StatefulWidget { final VoidCallback? onConfigSaved; @@ -17,8 +20,10 @@ class _BackendConfigScreenState extends State { final _urlController = TextEditingController(); bool _isLoading = false; bool _isTesting = false; + bool _hasLoadedConfig = false; String? _errorMessage; String? _successMessage; + List _customHeaders = []; @override void initState() { @@ -33,16 +38,27 @@ class _BackendConfigScreenState extends State { } Future _loadSavedUrl() async { - final prefs = await SharedPreferences.getInstance(); - final savedUrl = prefs.getString('backend_url'); - final urlToShow = (savedUrl != null && savedUrl.isNotEmpty) - ? savedUrl - : ApiConfig.baseUrl; - - if (mounted) { - setState(() { - _urlController.text = urlToShow; - }); + String urlToShow = ApiConfig.baseUrl; + List headers = const []; + try { + final prefs = await SharedPreferences.getInstance(); + final savedUrl = prefs.getString('backend_url'); + headers = await CustomProxyHeadersService.instance.loadHeaders(); + if (savedUrl != null && savedUrl.isNotEmpty) { + urlToShow = savedUrl; + } + } catch (e, stack) { + // Swallow storage failures so the screen still becomes interactive with + // sensible defaults; the user can re-enter and re-save. + debugPrint('BackendConfigScreen: failed to load saved config: $e\n$stack'); + } finally { + if (mounted) { + setState(() { + _urlController.text = urlToShow; + _customHeaders = headers; + _hasLoadedConfig = true; + }); + } } } @@ -55,6 +71,7 @@ class _BackendConfigScreenState extends State { _successMessage = null; }); + final previousHeaders = ApiConfig.customProxyHeaders; try { // Normalize base URL by removing trailing slashes final normalizedUrl = _urlController.text.trim().replaceAll( @@ -62,10 +79,14 @@ class _BackendConfigScreenState extends State { '', ); + // Apply the unsaved edits only for the duration of this probe so the + // test reflects what the user is about to save. Restored in `finally`. + ApiConfig.setCustomProxyHeaders(_customHeaders); + // Check /sessions/new page to verify it's a Sure backend final sessionsUrl = Uri.parse('$normalizedUrl/sessions/new'); final sessionsResponse = await http - .get(sessionsUrl, headers: {'Accept': 'text/html'}) + .get(sessionsUrl, headers: ApiConfig.htmlHeaders()) .timeout( const Duration(seconds: 10), onTimeout: () { @@ -98,6 +119,7 @@ class _BackendConfigScreenState extends State { }); } } finally { + ApiConfig.setCustomProxyHeaders(previousHeaders); if (mounted) { setState(() { _isTesting = false; @@ -125,6 +147,10 @@ class _BackendConfigScreenState extends State { final prefs = await SharedPreferences.getInstance(); await prefs.setString('backend_url', normalizedUrl); + // Save custom proxy headers + await CustomProxyHeadersService.instance.saveHeaders(_customHeaders); + ApiConfig.setCustomProxyHeaders(_customHeaders); + // Update ApiConfig ApiConfig.setBaseUrl(normalizedUrl); @@ -334,6 +360,41 @@ class _BackendConfigScreenState extends State { validator: _validateUrl, onFieldSubmitted: (_) => _saveAndContinue(), ), + const SizedBox(height: 24), + ExpansionTile( + tilePadding: EdgeInsets.zero, + leading: const Icon(Icons.http_outlined), + title: const Text('Custom proxy headers'), + subtitle: Text( + _customHeaders.isEmpty + ? 'Optional headers for a reverse proxy or auth gateway' + : '${_customHeaders.length} configured', + ), + children: [ + const SizedBox(height: 8), + if (_hasLoadedConfig) + CustomProxyHeadersEditor( + initialHeaders: _customHeaders, + onChanged: (headers) { + setState(() => _customHeaders = headers); + }, + ) + else + const Center( + child: Padding( + padding: EdgeInsets.all(16), + child: CircularProgressIndicator(), + ), + ), + const SizedBox(height: 8), + Text( + 'Headers are sent by the app with API requests. External browser SSO pages may not receive them.', + style: Theme.of(context).textTheme.bodySmall?.copyWith( + color: colorScheme.onSurfaceVariant, + ), + ), + ], + ), const SizedBox(height: 16), // Test Connection Button diff --git a/mobile/lib/screens/login_screen.dart b/mobile/lib/screens/login_screen.dart index 3c97f457e..efff02d6f 100644 --- a/mobile/lib/screens/login_screen.dart +++ b/mobile/lib/screens/login_screen.dart @@ -5,21 +5,40 @@ import 'package:url_launcher/url_launcher.dart'; import 'package:flutter_svg/flutter_svg.dart'; import '../providers/auth_provider.dart'; import '../services/api_config.dart'; +import 'backend_config_screen.dart'; class LoginScreen extends StatefulWidget { final VoidCallback? onGoToSettings; const LoginScreen({super.key, this.onGoToSettings}); + void _openSettings(BuildContext context) { + if (onGoToSettings != null) { + onGoToSettings!(); + return; + } + Navigator.of(context).push( + MaterialPageRoute( + builder: (routeContext) => BackendConfigScreen( + onConfigSaved: () => Navigator.of(routeContext).pop(), + ), + ), + ); + } + @override State createState() => _LoginScreenState(); } class _LoginScreenState extends State { final _formKey = GlobalKey(); - final _emailController = TextEditingController(text: 'user@example.com'); - final _passwordController = TextEditingController(text: 'Password1!'); + static const _emailPlaceholder = 'user@example.com'; + static const _passwordPlaceholder = 'Password1!'; + final _emailController = TextEditingController(text: _emailPlaceholder); + final _passwordController = TextEditingController(text: _passwordPlaceholder); final _otpController = TextEditingController(); + final _emailFocus = FocusNode(); + final _passwordFocus = FocusNode(); bool _obscurePassword = true; late final TapGestureRecognizer _signUpTapRecognizer; @@ -27,6 +46,17 @@ class _LoginScreenState extends State { void initState() { super.initState(); _signUpTapRecognizer = TapGestureRecognizer()..onTap = _openSignUpPage; + _emailFocus.addListener(() => _clearPlaceholderOnFocus( + _emailFocus, _emailController, _emailPlaceholder)); + _passwordFocus.addListener(() => _clearPlaceholderOnFocus( + _passwordFocus, _passwordController, _passwordPlaceholder)); + } + + void _clearPlaceholderOnFocus( + FocusNode node, TextEditingController controller, String placeholder) { + if (node.hasFocus && controller.text == placeholder) { + controller.clear(); + } } @override @@ -35,6 +65,8 @@ class _LoginScreenState extends State { _emailController.dispose(); _passwordController.dispose(); _otpController.dispose(); + _emailFocus.dispose(); + _passwordFocus.dispose(); super.dispose(); } @@ -261,6 +293,7 @@ class _LoginScreenState extends State { // Email Field TextFormField( controller: _emailController, + focusNode: _emailFocus, keyboardType: TextInputType.emailAddress, autocorrect: false, textInputAction: TextInputAction.next, @@ -291,6 +324,7 @@ class _LoginScreenState extends State { // Password Field TextFormField( controller: _passwordController, + focusNode: _passwordFocus, obscureText: _obscurePassword, textInputAction: showOtp ? TextInputAction.next @@ -442,7 +476,7 @@ class _LoginScreenState extends State { // Backend URL info InkWell( - onTap: widget.onGoToSettings, + onTap: () => widget._openSettings(context), borderRadius: BorderRadius.circular(8), child: Container( padding: const EdgeInsets.all(12), @@ -494,7 +528,7 @@ class _LoginScreenState extends State { child: IconButton( icon: const Icon(Icons.settings_outlined), tooltip: 'Backend Settings', - onPressed: widget.onGoToSettings, + onPressed: () => widget._openSettings(context), ), ), ], diff --git a/mobile/lib/screens/settings_screen.dart b/mobile/lib/screens/settings_screen.dart index 1806d7b5e..676926153 100644 --- a/mobile/lib/screens/settings_screen.dart +++ b/mobile/lib/screens/settings_screen.dart @@ -11,6 +11,10 @@ import '../services/biometric_service.dart'; import '../services/preferences_service.dart'; import '../services/user_service.dart'; import 'log_viewer_screen.dart'; +import '../models/custom_proxy_header.dart'; +import '../services/api_config.dart'; +import '../services/custom_proxy_headers_service.dart'; +import '../widgets/custom_proxy_headers_editor.dart'; class SettingsScreen extends StatefulWidget { const SettingsScreen({super.key}); @@ -27,6 +31,7 @@ class _SettingsScreenState extends State { bool _biometricSupported = false; bool _biometricEnabled = false; bool _isTogglingBiometric = false; + List _customHeaders = []; @override void initState() { @@ -34,6 +39,7 @@ class _SettingsScreenState extends State { _loadPreferences(); _loadAppVersion(); _loadBiometricState(); + _loadCustomHeaders(); } Future _loadBiometricState() async { @@ -93,6 +99,18 @@ class _SettingsScreenState extends State { } } + Future _loadCustomHeaders() async { + try { + final headers = await CustomProxyHeadersService.instance.loadHeaders(); + if (mounted) { + setState(() => _customHeaders = headers); + } + } catch (e, stack) { + debugPrint('SettingsScreen: failed to load custom headers: $e\n$stack'); + // Keep the existing _customHeaders state so the screen remains usable. + } + } + Future _handleClearLocalData(BuildContext context) async { final confirmed = await showDialog( context: context, @@ -318,6 +336,79 @@ class _SettingsScreenState extends State { } } + Future _showCustomHeadersDialog() async { + final formKey = GlobalKey(); + final latestHeaders = await CustomProxyHeadersService.instance.loadHeaders(); + if (!mounted) return; + + setState(() => _customHeaders = latestHeaders); + var draftHeaders = List.from(latestHeaders); + + final saved = await showDialog( + context: context, + builder: (context) { + return AlertDialog( + title: const Text('Custom proxy headers'), + content: SingleChildScrollView( + child: Form( + key: formKey, + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + CustomProxyHeadersEditor( + initialHeaders: draftHeaders, + onChanged: (headers) => draftHeaders = headers, + ), + const SizedBox(height: 12), + Text( + 'Headers are sent by the app with API requests. External browser SSO pages may not receive them.', + style: Theme.of(context).textTheme.bodySmall?.copyWith( + color: Theme.of(context).colorScheme.onSurfaceVariant, + ), + ), + ], + ), + ), + ), + actions: [ + TextButton( + onPressed: () => Navigator.pop(context, false), + child: const Text('Cancel'), + ), + ElevatedButton( + onPressed: () { + if (formKey.currentState?.validate() != true) return; + Navigator.pop(context, true); + }, + child: const Text('Save'), + ), + ], + ); + }, + ); + + if (saved != true) return; + + try { + await CustomProxyHeadersService.instance.saveHeaders(draftHeaders); + ApiConfig.setCustomProxyHeaders(draftHeaders); + if (!mounted) return; + setState(() => _customHeaders = draftHeaders); + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar(content: Text('Custom proxy headers saved')), + ); + } catch (e) { + if (!mounted) return; + ScaffoldMessenger.of(context).showSnackBar( + SnackBar( + content: Text('Failed to save custom proxy headers: $e'), + backgroundColor: Theme.of(context).colorScheme.error, + ), + ); + } + } + @override Widget build(BuildContext context) { final colorScheme = Theme.of(context).colorScheme; @@ -482,6 +573,31 @@ class _SettingsScreenState extends State { const Divider(), + const Padding( + padding: EdgeInsets.fromLTRB(16, 16, 16, 8), + child: Text( + 'Connection', + style: TextStyle( + fontSize: 14, + fontWeight: FontWeight.bold, + color: Colors.grey, + ), + ), + ), + + ListTile( + leading: const Icon(Icons.http_outlined), + title: const Text('Custom proxy headers'), + subtitle: Text( + _customHeaders.isEmpty + ? 'Optional headers for a reverse proxy or auth gateway' + : '${_customHeaders.length} configured', + ), + onTap: _showCustomHeadersDialog, + ), + + const Divider(), + // Data Management Section const Padding( padding: EdgeInsets.fromLTRB(16, 16, 16, 8), diff --git a/mobile/lib/services/api_config.dart b/mobile/lib/services/api_config.dart index 441618a7c..a012786d0 100644 --- a/mobile/lib/services/api_config.dart +++ b/mobile/lib/services/api_config.dart @@ -1,5 +1,8 @@ import 'package:shared_preferences/shared_preferences.dart'; +import '../models/custom_proxy_header.dart'; +import 'custom_proxy_headers_service.dart'; + class ApiConfig { // Base URL for the API - can be changed to point to different environments // For local development, use: http://10.0.2.2:3000 (Android emulator) @@ -32,14 +35,53 @@ class ApiConfig { _apiKeyValue = null; } + // Custom proxy headers + static List _customProxyHeaders = []; + + static List get customProxyHeaders => + List.unmodifiable(_customProxyHeaders); + + static void setCustomProxyHeaders(List headers) { + _customProxyHeaders = CustomProxyHeader.sanitize(headers); + } + + static Map get customProxyHeaderMap { + return { + for (final header in _customProxyHeaders) header.name: header.value, + }; + } + + static Map jsonHeaders() { + return { + ...customProxyHeaderMap, + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }; + } + + static Map htmlHeaders() { + return { + ...customProxyHeaderMap, + 'Accept': 'text/html', + }; + } + /// Returns the correct auth headers based on the current auth mode. /// In API key mode, uses X-Api-Key header. /// In token mode, uses Authorization: Bearer header. static Map getAuthHeaders(String token) { if (_isApiKeyAuth && _apiKeyValue != null) { - return {'X-Api-Key': _apiKeyValue!, 'Accept': 'application/json'}; + return { + ...customProxyHeaderMap, + 'X-Api-Key': _apiKeyValue!, + 'Accept': 'application/json', + }; } - return {'Authorization': 'Bearer $token', 'Accept': 'application/json'}; + return { + ...customProxyHeaderMap, + 'Authorization': 'Bearer $token', + 'Accept': 'application/json', + }; } /// Initialize the API configuration by loading the backend URL from storage @@ -51,6 +93,7 @@ class ApiConfig { if (savedUrl != null && savedUrl.isNotEmpty) { _baseUrl = savedUrl; + _customProxyHeaders = await CustomProxyHeadersService.instance.loadHeaders(); return true; } @@ -58,6 +101,7 @@ class ApiConfig { // go straight to login while still letting users override it later. _baseUrl = _defaultBaseUrl; await prefs.setString(_backendUrlKey, _defaultBaseUrl); + _customProxyHeaders = await CustomProxyHeadersService.instance.loadHeaders(); return true; } catch (e) { // If initialization fails, keep the default URL diff --git a/mobile/lib/services/auth_service.dart b/mobile/lib/services/auth_service.dart index e31c34554..e39ae6254 100644 --- a/mobile/lib/services/auth_service.dart +++ b/mobile/lib/services/auth_service.dart @@ -36,10 +36,7 @@ class AuthService { final response = await http.post( url, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, + headers: ApiConfig.jsonHeaders(), body: jsonEncode(body), ).timeout(const Duration(seconds: 30)); @@ -145,10 +142,7 @@ class AuthService { final response = await http.post( url, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, + headers: ApiConfig.jsonHeaders(), body: jsonEncode(body), ).timeout(const Duration(seconds: 30)); @@ -227,10 +221,7 @@ class AuthService { final response = await http.post( url, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, + headers: ApiConfig.jsonHeaders(), body: jsonEncode({ 'refresh_token': refreshToken, 'device': deviceInfo, @@ -301,6 +292,7 @@ class AuthService { final response = await http.get( url, headers: { + ...ApiConfig.customProxyHeaderMap, 'X-Api-Key': apiKey, 'Accept': 'application/json', }, @@ -398,10 +390,7 @@ class AuthService { final url = Uri.parse('${ApiConfig.baseUrl}/api/v1/auth/sso_exchange'); final response = await http.post( url, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, + headers: ApiConfig.jsonHeaders(), body: jsonEncode({'code': code}), ).timeout(const Duration(seconds: 30)); @@ -463,10 +452,7 @@ class AuthService { final url = Uri.parse('${ApiConfig.baseUrl}/api/v1/auth/sso_link'); final response = await http.post( url, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, + headers: ApiConfig.jsonHeaders(), body: jsonEncode({ 'linking_code': linkingCode, 'email': email, @@ -523,10 +509,7 @@ class AuthService { final response = await http.post( url, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, + headers: ApiConfig.jsonHeaders(), body: jsonEncode(body), ).timeout(const Duration(seconds: 30)); diff --git a/mobile/lib/services/custom_proxy_headers_service.dart b/mobile/lib/services/custom_proxy_headers_service.dart new file mode 100644 index 000000000..26fd7f37b --- /dev/null +++ b/mobile/lib/services/custom_proxy_headers_service.dart @@ -0,0 +1,47 @@ +import 'dart:convert'; + +import 'package:flutter_secure_storage/flutter_secure_storage.dart'; + +import '../models/custom_proxy_header.dart'; + +class CustomProxyHeadersService { + static const String storageKey = 'custom_proxy_headers'; + + static CustomProxyHeadersService? _instance; + + CustomProxyHeadersService._(); + + static CustomProxyHeadersService get instance { + _instance ??= CustomProxyHeadersService._(); + return _instance!; + } + + Future> loadHeaders() async { + const storage = FlutterSecureStorage(); + try { + final raw = await storage.read(key: storageKey); + if (raw == null || raw.isEmpty) return []; + + final decoded = jsonDecode(raw); + if (decoded is! List) return []; + + return CustomProxyHeader.sanitize( + decoded + .whereType() + .map((item) => CustomProxyHeader.fromJson(Map.from(item))) + .toList(), + ); + } catch (_) { + return []; + } + } + + Future saveHeaders(List headers) async { + const storage = FlutterSecureStorage(); + final sanitized = CustomProxyHeader.sanitize(headers); + await storage.write( + key: storageKey, + value: jsonEncode(sanitized.map((header) => header.toJson()).toList()), + ); + } +} diff --git a/mobile/lib/widgets/custom_proxy_headers_editor.dart b/mobile/lib/widgets/custom_proxy_headers_editor.dart new file mode 100644 index 000000000..29dcc95ed --- /dev/null +++ b/mobile/lib/widgets/custom_proxy_headers_editor.dart @@ -0,0 +1,145 @@ +import 'package:flutter/material.dart'; + +import '../models/custom_proxy_header.dart'; + +class CustomProxyHeadersEditor extends StatefulWidget { + final List initialHeaders; + final ValueChanged> onChanged; + + const CustomProxyHeadersEditor({ + super.key, + required this.initialHeaders, + required this.onChanged, + }); + + @override + State createState() => _CustomProxyHeadersEditorState(); +} + +class _CustomProxyHeadersEditorState extends State { + late List<_HeaderDraft> _drafts; + + @override + void initState() { + super.initState(); + _drafts = widget.initialHeaders + .map((header) => _HeaderDraft(name: header.name, value: header.value)) + .toList(); + } + + void _notifyChanged() { + widget.onChanged( + _drafts + .map((draft) => CustomProxyHeader(name: draft.name.text, value: draft.value.text)) + .where((header) => header.isComplete) + .toList(), + ); + } + + void _addHeader() { + setState(() => _drafts.add(_HeaderDraft())); + } + + void _removeHeader(int index) { + setState(() { + final draft = _drafts.removeAt(index); + draft.dispose(); + }); + _notifyChanged(); + } + + @override + void dispose() { + for (final draft in _drafts) { + draft.dispose(); + } + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + for (var index = 0; index < _drafts.length; index++) ...[ + _HeaderRow( + draft: _drafts[index], + onChanged: _notifyChanged, + onRemove: () => _removeHeader(index), + ), + const SizedBox(height: 12), + ], + OutlinedButton.icon( + onPressed: _addHeader, + icon: const Icon(Icons.add), + label: const Text('Add header'), + ), + ], + ); + } +} + +class _HeaderRow extends StatelessWidget { + final _HeaderDraft draft; + final VoidCallback onChanged; + final VoidCallback onRemove; + + const _HeaderRow({ + required this.draft, + required this.onChanged, + required this.onRemove, + }); + + @override + Widget build(BuildContext context) { + return Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Expanded( + child: Column( + children: [ + TextFormField( + controller: draft.name, + decoration: const InputDecoration( + labelText: 'Header name', + hintText: 'X-Auth-Token', + ), + validator: (value) => CustomProxyHeader.validateName(value ?? ''), + onChanged: (_) => onChanged(), + ), + const SizedBox(height: 8), + TextFormField( + controller: draft.value, + decoration: const InputDecoration( + labelText: 'Header value', + ), + obscureText: true, + validator: (value) => CustomProxyHeader.validateValue(value ?? ''), + onChanged: (_) => onChanged(), + ), + ], + ), + ), + IconButton( + tooltip: 'Remove header', + icon: const Icon(Icons.delete_outline), + onPressed: onRemove, + ), + ], + ); + } +} + +class _HeaderDraft { + final TextEditingController name; + final TextEditingController value; + + _HeaderDraft({String name = '', String value = ''}) + : name = TextEditingController(text: name), + value = TextEditingController(text: value); + + void dispose() { + name.dispose(); + value.dispose(); + } +} diff --git a/mobile/test/models/custom_proxy_header_test.dart b/mobile/test/models/custom_proxy_header_test.dart new file mode 100644 index 000000000..6943f4fca --- /dev/null +++ b/mobile/test/models/custom_proxy_header_test.dart @@ -0,0 +1,47 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:sure_mobile/models/custom_proxy_header.dart'; + +void main() { + group('CustomProxyHeader', () { + test('serializes trimmed header name and value', () { + final header = CustomProxyHeader(name: ' X-Auth-Id ', value: ' abc '); + + expect(header.name, 'X-Auth-Id'); + expect(header.value, 'abc'); + expect(header.toJson(), { + 'name': 'X-Auth-Id', + 'value': 'abc', + }); + expect(CustomProxyHeader.fromJson(header.toJson()), header); + }); + + test('rejects empty, malformed, and reserved names', () { + expect(CustomProxyHeader.validateName(''), isNotNull); + expect(CustomProxyHeader.validateName('Bad Header'), isNotNull); + expect(CustomProxyHeader.validateName('Bad:Header'), isNotNull); + expect(CustomProxyHeader.validateName('Authorization'), isNotNull); + expect(CustomProxyHeader.validateName('X-Api-Key'), isNotNull); + expect(CustomProxyHeader.validateName('Accept'), isNotNull); + expect(CustomProxyHeader.validateName('Content-Type'), isNotNull); + }); + + test('allows custom header names with hyphens', () { + expect(CustomProxyHeader.validateName('X-Auth-Id'), isNull); + expect(CustomProxyHeader.validateName('X-Auth-Secret'), isNull); + }); + + test('rejects values containing control characters (header injection)', () { + expect(CustomProxyHeader.validateValue('abc\r\nInjected: 1'), isNotNull); + expect(CustomProxyHeader.validateValue('abc\tdef'), isNotNull); + expect(CustomProxyHeader.validateValue('abc\x7Fdef'), isNotNull); + expect(CustomProxyHeader.validateValue('plain value with spaces'), isNull); + }); + + test('redacts values for display', () { + expect( + CustomProxyHeader(name: 'X-Auth-Secret', value: '1234567890').redactedValue, + '••••••7890', + ); + }); + }); +} diff --git a/mobile/test/services/api_config_headers_test.dart b/mobile/test/services/api_config_headers_test.dart new file mode 100644 index 000000000..cde001b22 --- /dev/null +++ b/mobile/test/services/api_config_headers_test.dart @@ -0,0 +1,78 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sure_mobile/models/custom_proxy_header.dart'; +import 'package:sure_mobile/services/api_config.dart'; + +void main() { + setUp(() async { + SharedPreferences.setMockInitialValues({}); + ApiConfig.clearApiKeyAuth(); + ApiConfig.setBaseUrl(ApiConfig.defaultBaseUrl); + ApiConfig.setCustomProxyHeaders([]); + }); + + test('adds custom proxy headers to token auth headers', () { + ApiConfig.setCustomProxyHeaders([ + CustomProxyHeader(name: 'X-Auth-Id', value: 'id'), + CustomProxyHeader(name: 'X-Auth-Secret', value: 'secret'), + ]); + + expect(ApiConfig.getAuthHeaders('token'), { + 'X-Auth-Id': 'id', + 'X-Auth-Secret': 'secret', + 'Authorization': 'Bearer token', + 'Accept': 'application/json', + }); + }); + + test('adds custom proxy headers to unauthenticated json headers', () { + ApiConfig.setCustomProxyHeaders([ + CustomProxyHeader(name: 'X-Mobile-Bypass', value: 'pass'), + ]); + + expect(ApiConfig.jsonHeaders(), { + 'X-Mobile-Bypass': 'pass', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }); + }); + + test('drops headers with reserved names', () { + ApiConfig.setCustomProxyHeaders([ + CustomProxyHeader(name: 'Accept', value: 'text/plain'), + CustomProxyHeader(name: 'Authorization', value: 'should-be-dropped'), + CustomProxyHeader(name: 'X-Api-Key', value: 'should-be-dropped-too'), + CustomProxyHeader(name: 'Content-Type', value: 'application/xml'), + CustomProxyHeader(name: 'X-Auth-Id', value: 'id'), + ]); + + final result = ApiConfig.customProxyHeaders; + expect(result.length, 1); + expect(result.first.name, 'X-Auth-Id'); + }); + + test('deduplicates headers by normalized name keeping the last value', () { + ApiConfig.setCustomProxyHeaders([ + CustomProxyHeader(name: 'X-Auth-Id', value: 'first'), + CustomProxyHeader(name: 'x-auth-id', value: 'second'), + CustomProxyHeader(name: 'X-Auth-Id', value: 'third'), + ]); + + final result = ApiConfig.customProxyHeaders; + expect(result.length, 1); + expect(result.first.name, 'X-Auth-Id'); + expect(result.first.value, 'third'); + }); + + test('app managed headers win over custom headers', () { + ApiConfig.setCustomProxyHeaders([ + CustomProxyHeader(name: 'Accept', value: 'text/plain'), + CustomProxyHeader(name: 'X-Auth-Id', value: 'id'), + ]); + + expect(ApiConfig.htmlHeaders(), { + 'X-Auth-Id': 'id', + 'Accept': 'text/html', + }); + }); +} diff --git a/mobile/test/services/custom_proxy_headers_service_test.dart b/mobile/test/services/custom_proxy_headers_service_test.dart new file mode 100644 index 000000000..f64f26823 --- /dev/null +++ b/mobile/test/services/custom_proxy_headers_service_test.dart @@ -0,0 +1,47 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_secure_storage/flutter_secure_storage.dart'; +import 'package:sure_mobile/models/custom_proxy_header.dart'; +import 'package:sure_mobile/services/custom_proxy_headers_service.dart'; + +void main() { + setUp(() { + FlutterSecureStorage.setMockInitialValues({}); + }); + + test('saves and loads custom proxy headers', () async { + final service = CustomProxyHeadersService.instance; + final headers = [ + CustomProxyHeader(name: 'X-Auth-Id', value: 'id'), + CustomProxyHeader(name: 'X-Auth-Secret', value: 'secret'), + ]; + + await service.saveHeaders(headers); + + expect(await service.loadHeaders(), headers); + }); + + test('drops incomplete and duplicate headers, keeping the last value', () async { + final service = CustomProxyHeadersService.instance; + + await service.saveHeaders([ + CustomProxyHeader(name: 'X-Auth-Id', value: 'old'), + CustomProxyHeader(name: '', value: 'ignored'), + CustomProxyHeader(name: 'X-Auth-Id', value: 'new'), + CustomProxyHeader(name: 'X-Empty', value: ''), + ]); + + expect(await service.loadHeaders(), [ + CustomProxyHeader(name: 'X-Auth-Id', value: 'new'), + ]); + }); + + test('returns an empty list for invalid stored json', () async { + const storage = FlutterSecureStorage(); + await storage.write( + key: CustomProxyHeadersService.storageKey, + value: 'not json', + ); + + expect(await CustomProxyHeadersService.instance.loadHeaders(), isEmpty); + }); +} From 974f55e2d5858d0ef5d6ddaa7e93984a4131a5fd Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 14:13:24 -0700 Subject: [PATCH 02/31] feat(api): add transaction idempotency keys (#1729) * feat(api): add transaction idempotency keys * fix(api): validate transaction idempotency source * fix(api): tighten transaction idempotency params --- .../api/v1/transactions_controller.rb | 77 +++++- .../transactions/_transaction.json.jbuilder | 2 + docs/api/openapi.yaml | 20 ++ spec/requests/api/v1/transactions_spec.rb | 34 +++ spec/swagger_helper.rb | 2 + .../api/v1/transactions_controller_test.rb | 239 ++++++++++++++++++ 6 files changed, 371 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/v1/transactions_controller.rb b/app/controllers/api/v1/transactions_controller.rb index c3538c79e..5dd65aba3 100644 --- a/app/controllers/api/v1/transactions_controller.rb +++ b/app/controllers/api/v1/transactions_controller.rb @@ -69,7 +69,7 @@ class Api::V1::TransactionsController < Api::V1::BaseController family = current_resource_owner.family # Validate account_id is present - unless transaction_params[:account_id].present? + unless account_id_param.present? render json: { error: "validation_failed", message: "Account ID is required", @@ -78,7 +78,21 @@ class Api::V1::TransactionsController < Api::V1::BaseController return end - account = family.accounts.writable_by(current_resource_owner).find(transaction_params[:account_id]) + if idempotency_source_param.present? && idempotency_external_id.blank? + render json: { + error: "validation_failed", + message: "Source requires external_id", + errors: [ "Source requires external_id" ] + }, status: :unprocessable_entity + return + end + + account = family.accounts.writable_by(current_resource_owner).find(account_id_param) + + if idempotency_key_requested? && (existing_entry = existing_idempotent_entry(account)) + return render_existing_idempotent_entry(existing_entry) + end + @entry = account.entries.new(entry_params_for_create) if @entry.save @@ -96,6 +110,12 @@ class Api::V1::TransactionsController < Api::V1::BaseController }, status: :unprocessable_entity end + rescue ActiveRecord::RecordNotUnique + if idempotency_key_requested? && account && (existing_entry = existing_idempotent_entry(account)) + render_existing_idempotent_entry(existing_entry) + else + raise + end rescue => e Rails.logger.error "TransactionsController#create error: #{e.message}" Rails.logger.error e.backtrace.join("\n") @@ -282,11 +302,15 @@ end def transaction_params params.require(:transaction).permit( - :account_id, :date, :amount, :name, :description, :notes, :currency, + :date, :amount, :name, :description, :notes, :currency, :category_id, :merchant_id, :nature, tag_ids: [] ) end + def account_id_param + params.dig(:transaction, :account_id).presence + end + def entry_params_for_create entry_params = { name: transaction_params[:name] || transaction_params[:description], @@ -301,6 +325,10 @@ end tag_ids: transaction_params[:tag_ids] || [] } } + if idempotency_key_requested? + entry_params[:external_id] = idempotency_external_id + entry_params[:source] = idempotency_source + end entry_params.compact end @@ -339,6 +367,49 @@ end params.dig(:transaction, :nature).present? end + def idempotency_key_requested? + idempotency_external_id.present? + end + + def idempotency_external_id + idempotency_param_value(:external_id) + end + + def idempotency_source + idempotency_source_param.presence || "api" + end + + def idempotency_source_param + idempotency_param_value(:source) + end + + def idempotency_param_value(key) + value = params.dig(:transaction, key) + value.to_s.presence if value.is_a?(String) || value.is_a?(Numeric) + end + + def existing_idempotent_entry(account) + account.entries.find_by( + external_id: idempotency_external_id, + source: idempotency_source + ) + end + + def render_existing_idempotent_entry(entry) + unless entry.entryable.is_a?(Transaction) + render json: { + error: "validation_failed", + message: "External ID already exists for a non-transaction entry", + errors: [ "External ID already exists for a non-transaction entry" ] + }, status: :unprocessable_entity + return + end + + @entry = entry + @transaction = entry.transaction + render :show, status: :ok + end + def calculate_signed_amount amount = transaction_params[:amount].to_f nature = transaction_params[:nature] diff --git a/app/views/api/v1/transactions/_transaction.json.jbuilder b/app/views/api/v1/transactions/_transaction.json.jbuilder index 9f3a47a98..488ecbe5b 100644 --- a/app/views/api/v1/transactions/_transaction.json.jbuilder +++ b/app/views/api/v1/transactions/_transaction.json.jbuilder @@ -17,6 +17,8 @@ json.signed_amount_cents(transaction.entry.classification == "income" ? amount_c json.currency transaction.entry.currency json.name transaction.entry.name json.notes transaction.entry.notes +json.external_id transaction.entry.external_id +json.source transaction.entry.source json.classification transaction.entry.classification # Account information diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9b2f867d9..9db36b8f7 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1435,6 +1435,12 @@ components: notes: type: string nullable: true + external_id: + type: string + nullable: true + source: + type: string + nullable: true classification: type: string account: @@ -6256,6 +6262,12 @@ paths: application/json: schema: "$ref": "#/components/schemas/Transaction" + '200': + description: transaction already exists for external idempotency key + content: + application/json: + schema: + "$ref": "#/components/schemas/Transaction" '422': description: validation error - missing required fields content: @@ -6310,6 +6322,14 @@ paths: - inflow - outflow description: Transaction nature (determines sign) + external_id: + type: string + description: Optional external idempotency key scoped to account + and source + source: + type: string + description: Optional source namespace for external_id. Requires + external_id and defaults to api when external_id is provided tag_ids: type: array items: diff --git a/spec/requests/api/v1/transactions_spec.rb b/spec/requests/api/v1/transactions_spec.rb index a4eab7590..d57a40b6a 100644 --- a/spec/requests/api/v1/transactions_spec.rb +++ b/spec/requests/api/v1/transactions_spec.rb @@ -173,6 +173,8 @@ RSpec.describe 'API V1 Transactions', type: :request do category_id: { type: :string, format: :uuid, description: 'Category ID' }, merchant_id: { type: :string, format: :uuid, description: 'Merchant ID' }, nature: { type: :string, enum: %w[income expense inflow outflow], description: 'Transaction nature (determines sign)' }, + external_id: { type: :string, description: 'Optional external idempotency key scoped to account and source' }, + source: { type: :string, description: 'Optional source namespace for external_id. Requires external_id and defaults to api when external_id is provided' }, tag_ids: { type: :array, items: { type: :string, format: :uuid }, description: 'Array of tag IDs' } }, required: %w[account_id date amount name] @@ -201,6 +203,38 @@ RSpec.describe 'API V1 Transactions', type: :request do run_test! end + response '200', 'transaction already exists for external idempotency key' do + schema '$ref' => '#/components/schemas/Transaction' + + let(:body) do + { + transaction: { + account_id: account.id, + date: Date.current.to_s, + amount: 50.00, + name: 'Test purchase', + nature: 'expense', + external_id: 'docs-import-transaction-1', + source: 'external_import' + } + } + end + + before do + account.entries.create!( + name: 'Test purchase', + date: Date.current, + amount: 50.00, + currency: 'USD', + external_id: 'docs-import-transaction-1', + source: 'external_import', + entryable: Transaction.new + ) + end + + run_test! + end + response '422', 'validation error - missing account_id' do schema '$ref' => '#/components/schemas/ErrorResponse' diff --git a/spec/swagger_helper.rb b/spec/swagger_helper.rb index d8260c27e..0692d2aad 100644 --- a/spec/swagger_helper.rb +++ b/spec/swagger_helper.rb @@ -788,6 +788,8 @@ RSpec.configure do |config| currency: { type: :string }, name: { type: :string }, notes: { type: :string, nullable: true }, + external_id: { type: :string, nullable: true }, + source: { type: :string, nullable: true }, classification: { type: :string }, account: { '$ref' => '#/components/schemas/Account' }, category: { '$ref' => '#/components/schemas/Category', nullable: true }, diff --git a/test/controllers/api/v1/transactions_controller_test.rb b/test/controllers/api/v1/transactions_controller_test.rb index 8ddb0ebf7..002c80c30 100644 --- a/test/controllers/api/v1/transactions_controller_test.rb +++ b/test/controllers/api/v1/transactions_controller_test.rb @@ -179,6 +179,220 @@ class Api::V1::TransactionsControllerTest < ActionDispatch::IntegrationTest assert_equal @account.id, response_data["account"]["id"] end + test "should create transaction with external idempotency key" do + transaction_params = { + transaction: { + account_id: @account.id, + name: "Imported Transaction", + amount: 25.00, + date: Date.current, + currency: "USD", + nature: "expense", + external_id: "import-txn-1", + source: "external_import" + } + } + + assert_difference("@account.entries.count", 1) do + post api_v1_transactions_url, + params: transaction_params, + headers: api_headers(@api_key) + end + + assert_response :created + response_data = JSON.parse(response.body) + assert_equal "import-txn-1", response_data["external_id"] + assert_equal "external_import", response_data["source"] + + entry = @account.entries.find_by!(external_id: "import-txn-1", source: "external_import") + assert_equal response_data["id"], entry.transaction.id + end + + test "should use default source when external_id provided without source" do + transaction_params = { + transaction: { + account_id: @account.id, + name: "Imported Transaction", + amount: 25.00, + date: Date.current, + currency: "USD", + nature: "expense", + external_id: "default-source-test" + } + } + + assert_difference("@account.entries.count", 1) do + post api_v1_transactions_url, + params: transaction_params, + headers: api_headers(@api_key) + end + + assert_response :created + response_data = JSON.parse(response.body) + entry = @account.entries.find_by!(external_id: "default-source-test") + assert_equal "api", entry.source + assert_equal "api", response_data["source"] + + assert_no_difference("@account.entries.count") do + post api_v1_transactions_url, + params: transaction_params.deep_merge(transaction: { name: "Changed Name" }), + headers: api_headers(@api_key) + end + + assert_response :ok + end + + test "should reject source without external idempotency key" do + transaction_params = { + transaction: { + account_id: @account.id, + name: "Imported Transaction", + amount: 25.00, + date: Date.current, + currency: "USD", + nature: "expense", + source: "external_import" + } + } + + assert_no_difference("@account.entries.count") do + post api_v1_transactions_url, + params: transaction_params, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + response_data = JSON.parse(response.body) + assert_equal "validation_failed", response_data["error"] + assert_equal "Source requires external_id", response_data["message"] + assert_equal [ "Source requires external_id" ], response_data["errors"] + end + + test "should return existing transaction for duplicate external idempotency key" do + transaction_params = { + transaction: { + account_id: @account.id, + name: "Imported Transaction", + amount: 25.00, + date: Date.current, + currency: "USD", + nature: "expense", + external_id: "import-txn-2", + source: "external_import" + } + } + + post api_v1_transactions_url, + params: transaction_params, + headers: api_headers(@api_key) + assert_response :created + created_data = JSON.parse(response.body) + + assert_no_difference("@account.entries.count") do + post api_v1_transactions_url, + params: transaction_params.deep_merge(transaction: { name: "Changed Name" }), + headers: api_headers(@api_key) + end + + assert_response :ok + response_data = JSON.parse(response.body) + assert_equal created_data["id"], response_data["id"] + assert_equal "Imported Transaction", response_data["name"] + end + + test "should scope external idempotency keys to account" do + other_account = @family.accounts.create!( + name: "Other API Account", + accountable: Depository.new, + balance: 0, + currency: "USD" + ) + transaction_params = { + transaction: { + name: "Imported Transaction", + amount: 25.00, + date: Date.current, + currency: "USD", + nature: "expense", + external_id: "shared-import-txn", + source: "external_import" + } + } + + assert_difference("Entry.count", 2) do + post api_v1_transactions_url, + params: transaction_params.deep_merge(transaction: { account_id: @account.id }), + headers: api_headers(@api_key) + assert_response :created + + post api_v1_transactions_url, + params: transaction_params.deep_merge(transaction: { account_id: other_account.id }), + headers: api_headers(@api_key) + assert_response :created + end + end + + test "should scope external idempotency keys to source" do + transaction_params = { + transaction: { + account_id: @account.id, + name: "Imported Transaction", + amount: 25.00, + date: Date.current, + currency: "USD", + nature: "expense", + external_id: "shared-source-txn", + source: "external_import" + } + } + + assert_difference("Entry.count", 2) do + post api_v1_transactions_url, + params: transaction_params, + headers: api_headers(@api_key) + assert_response :created + + post api_v1_transactions_url, + params: transaction_params.deep_merge(transaction: { source: "other_import" }), + headers: api_headers(@api_key) + assert_response :created + end + + @account.entries.find_by!(external_id: "shared-source-txn", source: "external_import") + @account.entries.find_by!(external_id: "shared-source-txn", source: "other_import") + end + + test "should reject external idempotency key collision with non-transaction entry" do + @account.entries.create!( + name: "Existing valuation", + amount: 100, + currency: "USD", + date: Date.current, + external_id: "import-non-transaction", + source: "external_import", + entryable: Valuation.new + ) + + post api_v1_transactions_url, + params: { + transaction: { + account_id: @account.id, + name: "Imported Transaction", + amount: 25.00, + date: Date.current - 1.day, + currency: "USD", + nature: "expense", + external_id: "import-non-transaction", + source: "external_import" + } + }, + headers: api_headers(@api_key) + + assert_response :unprocessable_entity + response_data = JSON.parse(response.body) + assert_equal "validation_failed", response_data["error"] + end + test "should reject create with read-only API key" do transaction_params = { transaction: { @@ -209,6 +423,31 @@ class Api::V1::TransactionsControllerTest < ActionDispatch::IntegrationTest assert_response :unprocessable_entity end + test "should reject invalid date on create" do + transaction_params = { + transaction: { + account_id: @account.id, + name: "Invalid Date Transaction", + amount: 25.00, + date: "not-a-date", + currency: "USD", + nature: "expense" + } + } + + assert_no_difference("@account.entries.count") do + post api_v1_transactions_url, + params: transaction_params, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + response_data = JSON.parse(response.body) + assert_equal "validation_failed", response_data["error"] + assert_equal "Transaction could not be created", response_data["message"] + assert response_data["errors"].any? { |error| error.match?(/Date/) } + end + test "should reject create without API key" do post api_v1_transactions_url, params: { transaction: { name: "Test" } } assert_response :unauthorized From f50c151e21d1db2a5808f0698e794b544dfff0f4 Mon Sep 17 00:00:00 2001 From: Guillem Arias Fauste Date: Mon, 11 May 2026 23:29:05 +0200 Subject: [PATCH 03/31] fix(design-system): DS::Alert alignment, accessibility, and hierarchy polish (#1734) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(design-system): align DS::Alert icon with title The icon was rendered at size 'sm' (w-4 h-4) and started at the very top of the flex row (items-start without an offset), which optically sat above the title's cap when the title was present and slightly above the message baseline when it wasn't. The hand-rolled alerts this PR replaced used 'w-5 h-5 mt-0.5' for exactly this reason — restore the same combination in the component: - size: sm -> md (w-4/h-4 -> w-5/h-5). - class adds mt-0.5 so the icon's vertical center lines up with the bold title's cap-height (and with the body baseline in the title-less case). No API change. Visual fix only. Refs #1731 * fix(design-system): split DS::Alert into title-row + indented body Replaces the items-start + margin-fudge approach with a two-row layout that doesn't depend on icon-bounding-box vs text-cap-height arithmetic: - Title case: icon and bold title share a flex row with items-center, so the icon's vertical centre lines up with the title's line. Body (block content or message) renders below in a separate row, padded by pl-8 (= icon md width + gap-3) so it indents under the title text rather than under the icon. - Block-only case (no title, no message — used by the alpha_vantage rate-limit alert): keeps the items-start fallback with a small mt-0.5 on the icon so the cap of the first paragraph still sits near the icon centre. - Single-line message case: items-center between icon and message, no fudge needed. container_classes loses its 'flex items-start gap-3' base since the outer div is no longer the flex container. Each branch declares its own flex/items-* combination. Refs #1731 * fix(design-system): a11y semantics + visual polish on DS::Alert Builds on the title-row restructure with the items the design / a11y review surfaced: - live: keyword (default :none, accepts :status / :polite and :alert / :assertive) maps to role="status" or role="alert" on the outer div. Static, page-baked alerts (the migrated callsites in #1731) keep the default :none and stay role-less. Dynamic surfaces (flash, validation summaries appearing after a Turbo update) opt into the live role they need. - aria-labelledby on the outer div pointing at the title

so AT picks the title as the alert's accessible name when one is set. - Variant prefix in the title / message via an sr-only span. Screen reader hears 'Warning: …', 'Error: …', etc.; sighted users see no change. Variant labels live under ds.alert.variants.* in config/locales/views/components/en.yml. - Body text inside titled alerts now defaults to text-secondary instead of text-primary, so hierarchy reads on weight + colour rather than weight alone (Refactoring UI: hierarchy needs both). Single-line message and block-only fallback keep text-primary since there is no second tier. - Icon size goes back from md (20px) to sm (16px) — proportionally closer to text-sm body — and the items-center branches grow -mt-0.5 to compensate for the cap-centre vs line-centre offset that flex's items-center alone can't bridge. - Title weight bumped from font-medium (500) to font-semibold (600) for clearer prominence against the now-softer body. No API breakage: existing callers passing only message:/title:/variant: keep working. The new live: arg defaults to the correct value for the static migration sites. Refs #1731 * fix(design-system): drop aria-labelledby when alert has no role; revert body to text-primary Two corrections after numerical contrast analysis and CodeRabbit feedback: 1. aria-labelledby was being emitted on every titled alert, but the default live: :none leaves the outer

with no role. ARIA spec only honours the labelling relationship on elements with a host role, so on a generic
the attribute is invalid and accessibility validators flag it. Now only emitted when aria_role is set (live: :status or :alert). Static, page-baked callsites stay role-less and label-less; dynamic callers that opt into a live role get the proper accessible-name relationship. 2. text-secondary on bg-{variant}/10 in light mode lands at ~4.07-4.25:1 contrast — below WCAG AA's 4.5:1 for normal text. Reverting the body wrapper to text-primary brings it back to AAA (~15:1). Loses some of the Refactoring UI body-vs-title colour hierarchy; the title's font-semibold weight + larger optical mass against an otherwise plain body still reads as hierarchy. Single-line message and block-only fallback already used text-primary, so this just unifies the three branches. The remaining contrast gap — text-success (green-600) icon on bg-success/10 light surface at 2.77:1 — is documented in the PR description; fixing it cleanly needs a token-level bump (--color-success: green-600 -> green-700 in light mode) which is out of scope for this PR. Refs #1731 * fix(settings/providers): use DS::Alert title:+message: instead of inline content_tag Three callsites added in #1710 passed block-level markup (`

`/`

`) through `message:` via `safe_join + content_tag`. The post-#1731 alert template wraps `message:` in a `

`, which makes nesting a `

` or `

` invalid HTML — browsers auto-close the outer paragraph and the indented body row collapses. Each of the three is semantically a title + body pair, so swap them to the proper `title:` + `message:` API. No new strings — the i18n keys (`*.no_withdraw_title` / `_body`, `encryption_error.title` / `.message`) already split that way; the inline assembly was the artefact. The encryption-error block loses an explicit `

` wrapper around the title; DS::Alert's title is a `

`. The visual hierarchy and sr-only variant prefix are unchanged. Worth tracking heading semantics as a follow-up against DS::Alert (a `heading_level:` arg) rather than bringing back the manual markup. * fix(design-system): make :destructive variant alias explicit in DS::Alert locale Add `destructive: Error` to `ds.alert.variants` and drop the implicit `:destructive -> :error` aliasing in `DS::Alert#variant_label`. Both the locale file and the component now self-document the variant set; lookup is direct, no conditional needed. Per @jjmata review on #1734. --- app/components/DS/alert.html.erb | 48 ++++++++++++++----- app/components/DS/alert.rb | 33 +++++++++++-- .../providers/_binance_panel.html.erb | 6 +-- .../providers/_enable_banking_panel.html.erb | 6 +-- app/views/settings/providers/show.html.erb | 6 +-- config/locales/views/components/en.yml | 8 ++++ 6 files changed, 79 insertions(+), 28 deletions(-) diff --git a/app/components/DS/alert.html.erb b/app/components/DS/alert.html.erb index 419a4607c..efda08221 100644 --- a/app/components/DS/alert.html.erb +++ b/app/components/DS/alert.html.erb @@ -1,15 +1,37 @@ -

- <%= helpers.icon icon_name, size: "sm", color: icon_color, class: "shrink-0 mt-0.5" %> +<%= tag.div(class: container_classes, role: aria_role, "aria-labelledby": (aria_role && title.present?) ? title_id : nil) do %> + <% if title.present? %> +
+ <%= helpers.icon icon_name, size: "sm", color: icon_color, class: "shrink-0 -mt-0.5" %> +

+ <%= variant_label %>: + <%= title %> +

+
-
- <% if title.present? %> -

<%= title %>

+ <% if content.present? || message.present? %> +
+ <% if content.present? %> + <%= content %> + <% else %> + <%= message %> + <% end %> +
<% end %> - - <% if content.present? %> - <%= content %> - <% elsif message.present? %> - <%= message %> - <% end %> -
-
+ <% elsif content.present? %> +
+ <%= helpers.icon icon_name, size: "sm", color: icon_color, class: "shrink-0" %> +
+ <%= variant_label %>: + <%= content %> +
+
+ <% elsif message.present? %> +
+ <%= helpers.icon icon_name, size: "sm", color: icon_color, class: "shrink-0 -mt-0.5" %> +

+ <%= variant_label %>: + <%= message %> +

+
+ <% end %> +<% end %> diff --git a/app/components/DS/alert.rb b/app/components/DS/alert.rb index dc7025eb7..050beaf66 100644 --- a/app/components/DS/alert.rb +++ b/app/components/DS/alert.rb @@ -1,22 +1,34 @@ class DS::Alert < DesignSystemComponent VARIANTS = %i[info success warning error destructive].freeze + LIVE_MODES = %i[none status alert].freeze - def initialize(message: nil, title: nil, variant: :info) + def initialize(message: nil, title: nil, variant: :info, live: :none) @message = message @title = title @variant = normalize_variant(variant) + @live = normalize_live(live) end private - attr_reader :message, :title, :variant + attr_reader :message, :title, :variant, :live def normalize_variant(raw) sym = raw.respond_to?(:to_sym) ? raw.to_sym : nil VARIANTS.include?(sym) ? sym : :info end + def normalize_live(raw) + sym = raw.respond_to?(:to_sym) ? raw.to_sym : nil + case sym + when :polite then :status + when :assertive then :alert + when *LIVE_MODES then sym + else :none + end + end + def container_classes - base_classes = "flex items-start gap-3 p-4 rounded-lg border" + base_classes = "p-4 rounded-lg border" variant_classes = case variant when :info @@ -57,4 +69,19 @@ class DS::Alert < DesignSystemComponent "info" end end + + def aria_role + case live + when :status then "status" + when :alert then "alert" + end + end + + def variant_label + I18n.t("ds.alert.variants.#{variant}") + end + + def title_id + @title_id ||= "DS-alert-title-#{SecureRandom.hex(4)}" + end end diff --git a/app/views/settings/providers/_binance_panel.html.erb b/app/views/settings/providers/_binance_panel.html.erb index f93eeea47..49b8cd0d4 100644 --- a/app/views/settings/providers/_binance_panel.html.erb +++ b/app/views/settings/providers/_binance_panel.html.erb @@ -3,10 +3,8 @@ <%= render DS::Alert.new( variant: :warning, - message: safe_join([ - content_tag(:p, t("settings.providers.binance_panel.no_withdraw_title"), class: "font-medium"), - content_tag(:p, t("settings.providers.binance_panel.no_withdraw_body"), class: "mt-1") - ]) + title: t("settings.providers.binance_panel.no_withdraw_title"), + message: t("settings.providers.binance_panel.no_withdraw_body") ) %>
diff --git a/app/views/settings/providers/_enable_banking_panel.html.erb b/app/views/settings/providers/_enable_banking_panel.html.erb index 4085520ef..45344ea55 100644 --- a/app/views/settings/providers/_enable_banking_panel.html.erb +++ b/app/views/settings/providers/_enable_banking_panel.html.erb @@ -71,10 +71,8 @@ <% if has_authenticated_connections && !is_new_record %> <%= render DS::Alert.new( variant: :warning, - message: safe_join([ - content_tag(:p, "Configuration locked", class: "font-medium"), - content_tag(:p, "Disconnect all linked banks before changing these credentials.", class: "mt-1") - ]) + title: "Configuration locked", + message: "Disconnect all linked banks before changing these credentials." ) %> <% end %> diff --git a/app/views/settings/providers/show.html.erb b/app/views/settings/providers/show.html.erb index b78a19a96..9108eabcd 100644 --- a/app/views/settings/providers/show.html.erb +++ b/app/views/settings/providers/show.html.erb @@ -4,10 +4,8 @@ <% if @encryption_error %> <%= render DS::Alert.new( variant: :error, - message: safe_join([ - content_tag(:h2, t("settings.providers.encryption_error.title"), class: "font-medium"), - content_tag(:p, t("settings.providers.encryption_error.message"), class: "text-sm mt-1") - ]) + title: t("settings.providers.encryption_error.title"), + message: t("settings.providers.encryption_error.message") ) %> <% else %>
diff --git a/config/locales/views/components/en.yml b/config/locales/views/components/en.yml index 9230a79ec..b17124e37 100644 --- a/config/locales/views/components/en.yml +++ b/config/locales/views/components/en.yml @@ -1,5 +1,13 @@ --- en: + ds: + alert: + variants: + info: Info + success: Success + warning: Warning + error: Error + destructive: Error provider_sync_summary: title: Sync summary last_sync: "Last sync: %{time_ago} ago" From 6b6c3bd34369a96ca7b38139a75b76247ea2e7bc Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 14:47:36 -0700 Subject: [PATCH 04/31] feat(exports): add attachment manifest (#1728) * feat(exports): add attachment manifest * fix(exports): include split parent receipts in manifest --- app/models/family/data_exporter.rb | 67 +++++++++++++++ test/models/family/data_exporter_test.rb | 105 ++++++++++++++++++++++- 2 files changed, 171 insertions(+), 1 deletion(-) diff --git a/app/models/family/data_exporter.rb b/app/models/family/data_exporter.rb index 1c3446f2d..34b6beba5 100644 --- a/app/models/family/data_exporter.rb +++ b/app/models/family/data_exporter.rb @@ -29,6 +29,10 @@ class Family::DataExporter zipfile.put_next_entry("rules.csv") zipfile.write generate_rules_csv + # Add attachment manifest metadata. Binary file payloads are not included. + zipfile.put_next_entry("attachments.json") + zipfile.write generate_attachments_manifest + # Add all.ndjson zipfile.put_next_entry("all.ndjson") zipfile.write generate_ndjson @@ -138,6 +142,69 @@ class Family::DataExporter end end + def generate_attachments_manifest + { + version: 1, + binary_included: false, + attachments: attachment_manifest_items + }.to_json + end + + def attachment_manifest_items + (transaction_attachment_manifest_items + family_document_attachment_manifest_items) + .sort_by { |item| [ item[:record_type], item[:record_id].to_s, item[:filename].to_s, item[:id].to_s ] } + end + + def transaction_attachment_manifest_items + @family.transactions + .with_attached_attachments + .includes(:attachments_attachments, entry: :account) + .flat_map do |transaction| + transaction.attachments.map do |attachment| + attachment_manifest_item( + attachment, + record_type: "Transaction", + record_id: transaction.id, + extra: { + entry_id: transaction.entry.id, + account_id: transaction.entry.account_id + } + ) + end + end + end + + def family_document_attachment_manifest_items + @family.family_documents.with_attached_file.filter_map do |document| + next unless document.file.attached? + + attachment_manifest_item( + document.file.attachment, + record_type: "FamilyDocument", + record_id: document.id, + extra: { + status: document.status + } + ) + end + end + + def attachment_manifest_item(attachment, record_type:, record_id:, extra: {}) + blob = attachment.blob + { + id: attachment.id, + record_type: record_type, + record_id: record_id, + name: attachment.name, + filename: blob.filename.to_s, + content_type: blob.content_type, + byte_size: blob.byte_size, + checksum: blob.checksum, + binary_included: false, + created_at: attachment.created_at + }.merge(extra) + end + def generate_ndjson lines = [] diff --git a/test/models/family/data_exporter_test.rb b/test/models/family/data_exporter_test.rb index b1f2e01eb..03376e08e 100644 --- a/test/models/family/data_exporter_test.rb +++ b/test/models/family/data_exporter_test.rb @@ -47,7 +47,7 @@ class Family::DataExporterTest < ActiveSupport::TestCase assert zip_data.is_a?(StringIO) # Check that the zip contains all expected files - expected_files = [ "accounts.csv", "transactions.csv", "trades.csv", "categories.csv", "rules.csv", "all.ndjson" ] + expected_files = [ "accounts.csv", "transactions.csv", "trades.csv", "categories.csv", "rules.csv", "attachments.json", "all.ndjson" ] Zip::File.open_buffer(zip_data) do |zip| actual_files = zip.entries.map(&:name) @@ -55,6 +55,109 @@ class Family::DataExporterTest < ActiveSupport::TestCase end end + test "exports attachment manifest metadata without binary payloads" do + entry = @account.entries.create!( + name: "Receipt Transaction", + amount: 12.34, + currency: "USD", + date: Date.current, + entryable: Transaction.new + ) + transaction = entry.transaction + transaction.attachments.attach( + io: StringIO.new("receipt bytes"), + filename: "receipt.pdf", + content_type: "application/pdf" + ) + + family_document = @family.family_documents.create!( + filename: "statement.pdf", + status: "ready" + ) + family_document.file.attach( + io: StringIO.new("statement bytes"), + filename: "statement.pdf", + content_type: "application/pdf" + ) + + other_account = @other_family.accounts.create!( + name: "Other Attachment Account", + accountable: Depository.new, + balance: 0, + currency: "USD" + ) + other_entry = other_account.entries.create!( + name: "Other Receipt", + amount: 1, + currency: "USD", + date: Date.current, + entryable: Transaction.new + ) + other_entry.transaction.attachments.attach( + io: StringIO.new("other bytes"), + filename: "other-receipt.pdf", + content_type: "application/pdf" + ) + + zip_data = @exporter.generate_export + + Zip::File.open_buffer(zip_data) do |zip| + manifest = JSON.parse(zip.read("attachments.json")) + attachments = manifest["attachments"] + filenames = attachments.map { |attachment| attachment["filename"] } + + assert_equal 1, manifest["version"] + assert_equal false, manifest["binary_included"] + assert_includes filenames, "receipt.pdf" + assert_includes filenames, "statement.pdf" + refute_includes filenames, "other-receipt.pdf" + + transaction_item = attachments.find { |attachment| attachment["record_type"] == "Transaction" } + assert_equal transaction.id, transaction_item["record_id"] + assert_equal entry.id, transaction_item["entry_id"] + assert_equal @account.id, transaction_item["account_id"] + assert_equal "attachments", transaction_item["name"] + assert_equal "application/pdf", transaction_item["content_type"] + assert_equal false, transaction_item["binary_included"] + + document_item = attachments.find { |attachment| attachment["record_type"] == "FamilyDocument" } + assert_equal family_document.id, document_item["record_id"] + assert_equal "ready", document_item["status"] + assert_equal "file", document_item["name"] + assert_equal false, document_item["binary_included"] + end + end + + test "exports split parent receipts in attachment manifest" do + split_parent = create_transaction_entry( + @account, + amount: 60, + date: Date.parse("2024-01-25"), + name: "Split parent receipt" + ) + split_parent.entryable.attachments.attach( + io: StringIO.new("split parent receipt bytes"), + filename: "split-parent-receipt.pdf", + content_type: "application/pdf" + ) + split_parent.split!([ + { name: "Split child", amount: 60, category_id: @category.id } + ]) + + zip_data = @exporter.generate_export + + Zip::File.open_buffer(zip_data) do |zip| + manifest = JSON.parse(zip.read("attachments.json")) + attachment = manifest["attachments"].find { |item| item["filename"] == "split-parent-receipt.pdf" } + + assert attachment + assert_equal "Transaction", attachment["record_type"] + assert_equal split_parent.entryable.id, attachment["record_id"] + assert_equal split_parent.id, attachment["entry_id"] + assert_equal @account.id, attachment["account_id"] + end + end + test "generates valid CSV files" do zip_data = @exporter.generate_export From 1fedc43f683dcc5713de4c490638d1bc4124868b Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 15:00:49 -0700 Subject: [PATCH 05/31] feat(api): add import preflight validation (#1755) * feat(api): add import preflight validation * fix(api): harden import preflight validation --- app/controllers/api/v1/base_controller.rb | 8 +- app/controllers/api/v1/imports_controller.rb | 49 +- app/models/import.rb | 5 + app/models/import/preflight.rb | 454 +++++++++++++++++ app/models/mint_import.rb | 31 +- app/models/sure_import.rb | 50 +- config/routes.rb | 1 + docs/api/openapi.yaml | 378 +++++++++++++- spec/requests/api/v1/imports_spec.rb | 126 ++++- spec/swagger_helper.rb | 88 ++++ .../api/v1/imports_controller_test.rb | 481 +++++++++++++++++- test/models/mint_import_test.rb | 8 + test/models/sure_import_test.rb | 28 + 13 files changed, 1649 insertions(+), 58 deletions(-) create mode 100644 app/models/import/preflight.rb diff --git a/app/controllers/api/v1/base_controller.rb b/app/controllers/api/v1/base_controller.rb index 4db722b52..ef175bf17 100644 --- a/app/controllers/api/v1/base_controller.rb +++ b/app/controllers/api/v1/base_controller.rb @@ -8,6 +8,12 @@ class Api::V1::BaseController < ApplicationController InvalidFilterError = Class.new(StandardError) + class << self + def valid_uuid?(value) + value.to_s.match?(UUID_PATTERN) + end + end + # Skip regular session-based authentication for API skip_authentication @@ -220,7 +226,7 @@ class Api::V1::BaseController < ApplicationController end def valid_uuid?(value) - value.to_s.match?(UUID_PATTERN) + self.class.valid_uuid?(value) end def safe_page_param diff --git a/app/controllers/api/v1/imports_controller.rb b/app/controllers/api/v1/imports_controller.rb index 23b93ae7d..5f1e9cf3e 100644 --- a/app/controllers/api/v1/imports_controller.rb +++ b/app/controllers/api/v1/imports_controller.rb @@ -4,7 +4,7 @@ class Api::V1::ImportsController < Api::V1::BaseController include Pagy::Backend # Ensure proper scope authorization - before_action :ensure_read_scope, only: [ :index, :show, :rows ] + before_action :ensure_read_scope, only: [ :index, :show, :rows, :preflight ] before_action :ensure_write_scope, only: [ :create ] before_action :set_import_with_rows, only: [ :show ] before_action :set_import, only: [ :rows ] @@ -77,10 +77,10 @@ class Api::V1::ImportsController < Api::V1::BaseController if params[:file].present? file = params[:file] - if file.size > Import::MAX_CSV_SIZE + if file.size > Import.max_csv_size return render json: { error: "file_too_large", - message: "File is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + message: "File is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." }, status: :unprocessable_entity end @@ -93,10 +93,10 @@ class Api::V1::ImportsController < Api::V1::BaseController @import.raw_file_str = file.read elsif params[:raw_file_content].present? - if params[:raw_file_content].bytesize > Import::MAX_CSV_SIZE + if params[:raw_file_content].bytesize > Import.max_csv_size return render json: { error: "content_too_large", - message: "Content is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + message: "Content is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." }, status: :unprocessable_entity end @@ -136,6 +136,30 @@ class Api::V1::ImportsController < Api::V1::BaseController render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error end + def preflight + preflight_result = Import::Preflight.new(family: current_resource_owner.family, params: preflight_params).call + render json: preflight_result.payload, status: preflight_result.status + rescue ActiveRecord::RecordNotFound + render json: { + error: "record_not_found", + message: "The requested resource was not found" + }, status: :not_found + rescue CSV::MalformedCSVError => e + render json: { + error: "invalid_csv", + message: "CSV content could not be parsed", + errors: [ e.message ] + }, status: :unprocessable_entity + rescue StandardError => e + Rails.logger.error "ImportsController#preflight error: #{e.message}" + e.backtrace&.each { |line| Rails.logger.error line } + + render json: { + error: "internal_server_error", + message: "Error: #{e.message}" + }, status: :internal_server_error + end + private def set_import @@ -186,10 +210,15 @@ class Api::V1::ImportsController < Api::V1::BaseController :signage_convention, :col_sep, :amount_type_strategy, - :amount_type_inflow_value + :amount_type_inflow_value, + :rows_to_skip ) end + def preflight_params + params.permit(*Import::Preflight::PARAM_KEYS) + end + def create_sure_import(family) content, filename, content_type = sure_import_upload_attributes return unless content @@ -282,10 +311,10 @@ class Api::V1::ImportsController < Api::V1::BaseController end def sure_import_file_upload_attributes(file) - if file.size > SureImport::MAX_NDJSON_SIZE + if file.size > SureImport.max_ndjson_size render json: { error: "file_too_large", - message: "File is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB." + message: "File is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." }, status: :unprocessable_entity return end @@ -308,10 +337,10 @@ class Api::V1::ImportsController < Api::V1::BaseController end def sure_import_raw_content_attributes(content) - if content.bytesize > SureImport::MAX_NDJSON_SIZE + if content.bytesize > SureImport.max_ndjson_size render json: { error: "content_too_large", - message: "Content is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB." + message: "Content is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." }, status: :unprocessable_entity return end diff --git a/app/models/import.rb b/app/models/import.rb index fba9c8c32..24b0aab71 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -2,6 +2,7 @@ class Import < ApplicationRecord MaxRowCountExceededError = Class.new(StandardError) MappingError = Class.new(StandardError) + # Shared CSV upload/content limit for web and API imports, including preflight. MAX_CSV_SIZE = 10.megabytes MAX_PDF_SIZE = 25.megabytes ALLOWED_CSV_MIME_TYPES = %w[text/csv text/plain application/vnd.ms-excel application/csv].freeze @@ -24,6 +25,10 @@ class Import < ApplicationRecord Date.new(1970, 1, 1)..Date.today.next_year(5) end + def self.max_csv_size + MAX_CSV_SIZE + end + AMOUNT_TYPE_STRATEGIES = %w[signed_amount custom_column].freeze belongs_to :family diff --git a/app/models/import/preflight.rb b/app/models/import/preflight.rb new file mode 100644 index 000000000..ef9429246 --- /dev/null +++ b/app/models/import/preflight.rb @@ -0,0 +1,454 @@ +# frozen_string_literal: true + +class Import::Preflight + Response = Struct.new(:status, :payload, keyword_init: true) + + class PreflightError < StandardError + attr_reader :status, :payload + + def initialize(response) + @status = response.status + @payload = response.payload + super(response.payload[:message]) + end + end + + CONFIG_PARAM_KEYS = %i[ + date_col_label + amount_col_label + name_col_label + category_col_label + tags_col_label + notes_col_label + account_col_label + qty_col_label + ticker_col_label + price_col_label + entity_type_col_label + currency_col_label + exchange_operating_mic_col_label + date_format + number_format + signage_convention + col_sep + amount_type_strategy + amount_type_inflow_value + rows_to_skip + ].freeze + + PARAM_KEYS = ([ + :type, + :account_id, + :file, + :raw_file_content + ] + CONFIG_PARAM_KEYS).freeze + + UNSUPPORTED_PREFLIGHT_IMPORT_TYPES = %w[PdfImport QifImport].freeze + IMPORT_TYPES = (Import::TYPES - UNSUPPORTED_PREFLIGHT_IMPORT_TYPES).freeze + + def initialize(family:, params:) + @family = family + @params = params.to_h.symbolize_keys + end + + def call + type = preflight_import_type + return invalid_import_type_response unless type + + type == "SureImport" ? sure_import_response : csv_import_response(type) + rescue PreflightError => e + Response.new(status: e.status, payload: e.payload) + end + + private + attr_reader :family, :params + + def preflight_import_type + type = params[:type].to_s + return "TransactionImport" if type.blank? + + type if IMPORT_TYPES.include?(type) + end + + def invalid_import_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "invalid_import_type", + message: "type must be one of: #{IMPORT_TYPES.join(', ')}" + } + ) + end + + def sure_import_response + upload_attributes = sure_import_upload_attributes + return missing_sure_content_response unless upload_attributes + + content, filename, content_type = upload_attributes + Response.new( + status: :ok, + payload: { + data: sure_import_preflight_payload(content, filename, content_type) + } + ) + end + + def csv_import_response(type) + upload_attributes = csv_upload_attributes + return missing_csv_content_response unless upload_attributes + + content, filename, content_type = upload_attributes + import = family.imports.build(import_config_params.merge(type: type, raw_file_str: content)) + import.account = preflight_account if params[:account_id].present? + apply_import_defaults(import) + + return unsupported_import_type_response unless import.requires_csv_workflow? + + unless import.valid? + return Response.new( + status: :ok, + payload: { + data: csv_preflight_payload( + import: import, + type: type, + filename: filename, + content_type: content_type, + content: content, + parsed_rows_count: 0, + csv_headers: [], + missing_required_headers: [], + errors: validation_errors(import), + warnings: [] + ) + } + ) + end + + csv_content = csv_content_for(import, content) + csv = Import.parse_csv_str(csv_content, col_sep: import.col_sep) + parsed_rows_count = csv.length + csv_headers = Array(csv.headers).compact + missing_required_headers = missing_required_headers(import, csv_headers) + errors = validation_errors(import) + + if missing_required_headers.any? + errors << { + code: "missing_required_headers", + message: "Missing required columns: #{missing_required_headers.join(', ')}" + } + end + + if parsed_rows_count.zero? + errors << { + code: "no_data_rows", + message: "No data rows were found." + } + end + + warnings = [] + warnings << "Row count exceeds this import type's publish limit." if parsed_rows_count > import.max_row_count + + Response.new( + status: :ok, + payload: { + data: csv_preflight_payload( + import: import, + type: type, + filename: filename, + content_type: content_type, + content: content, + parsed_rows_count: parsed_rows_count, + csv_headers: csv_headers, + missing_required_headers: missing_required_headers, + errors: errors, + warnings: warnings + ) + } + ) + end + + def import_config_params + params.slice(*CONFIG_PARAM_KEYS) + end + + def preflight_account + raise ActiveRecord::RecordNotFound unless Api::V1::BaseController.valid_uuid?(params[:account_id]) + + family.accounts.find(params[:account_id]) + end + + def csv_upload_attributes + if params[:file].present? + csv_file_upload_attributes(params[:file]) + elsif params[:raw_file_content].present? + csv_raw_content_attributes(params[:raw_file_content].to_s) + end + end + + def csv_file_upload_attributes(file) + raise_response csv_file_too_large_response if file.size > Import.max_csv_size + raise_response invalid_csv_file_type_response unless Import::ALLOWED_CSV_MIME_TYPES.include?(file.content_type) + + [ + file.read, + file.original_filename.presence || "import.csv", + file.content_type.presence || "text/csv" + ] + end + + def csv_raw_content_attributes(content) + raise_response csv_content_too_large_response if content.bytesize > Import.max_csv_size + + [ content, "import.csv", "text/csv" ] + end + + def sure_import_upload_attributes + if params[:file].present? + sure_import_file_upload_attributes(params[:file]) + elsif params[:raw_file_content].present? + sure_import_raw_content_attributes(params[:raw_file_content].to_s) + end + end + + def sure_import_file_upload_attributes(file) + raise_response sure_file_too_large_response if file.size > SureImport.max_ndjson_size + + extension = File.extname(file.original_filename.to_s).downcase + unless SureImport::ALLOWED_NDJSON_CONTENT_TYPES.include?(file.content_type) || extension.in?(%w[.ndjson .json]) + raise_response invalid_sure_file_type_response + end + + [ + file.read, + file.original_filename.presence || "sure-import.ndjson", + file.content_type.presence || "application/x-ndjson" + ] + end + + def sure_import_raw_content_attributes(content) + raise_response sure_content_too_large_response if content.bytesize > SureImport.max_ndjson_size + + [ content, "sure-import.ndjson", "application/x-ndjson" ] + end + + def sure_import_preflight_payload(content, filename, content_type) + line_counts = Hash.new(0) + errors = [] + valid_rows_count = 0 + nonblank_rows_count = 0 + + content.each_line.with_index(1) do |line, line_number| + next if line.strip.blank? + + nonblank_rows_count += 1 + record = JSON.parse(line) + + unless record.is_a?(Hash) + errors << { + code: "invalid_ndjson_record", + message: "Line #{line_number} must be a JSON object." + } + next + end + + if record["type"].blank? || !record.key?("data") + errors << { + code: "invalid_ndjson_record", + message: "Line #{line_number} must include type and data." + } + next + end + + valid_rows_count += 1 + line_counts[record["type"]] += 1 + rescue JSON::ParserError => e + errors << { + code: "invalid_json", + message: "Line #{line_number} is not valid JSON: #{e.message}" + } + end + + if nonblank_rows_count.zero? + errors << { + code: "no_data_rows", + message: "No data rows were found." + } + end + + entity_counts = SureImport.dry_run_totals_from_line_type_counts(line_counts) + unsupported_types = line_counts.keys - SureImport.importable_ndjson_types + warnings = [] + warnings << "No importable records were found." if nonblank_rows_count.positive? && entity_counts.values.sum.zero? + warnings << "Some records use unsupported types: #{unsupported_types.join(', ')}" if unsupported_types.any? + warnings << "Row count exceeds this import type's publish limit." if nonblank_rows_count > SureImport.max_row_count + + { + type: "SureImport", + valid: errors.empty?, + content: content_payload(filename, content_type, content), + stats: { + rows_count: nonblank_rows_count, + valid_rows_count: valid_rows_count, + invalid_rows_count: nonblank_rows_count - valid_rows_count, + entity_counts: entity_counts, + record_type_counts: line_counts + }, + errors: errors, + warnings: warnings + } + end + + def content_payload(filename, content_type, content) + { + filename: filename, + content_type: content_type, + byte_size: content.bytesize + } + end + + def csv_content_for(import, content) + return content unless import.rows_to_skip.to_i.positive? + + content.lines.drop(import.rows_to_skip.to_i).join + end + + def apply_import_defaults(import) + return unless import.is_a?(MintImport) + + MintImport.default_column_mappings.each do |attribute, value| + import.public_send("#{attribute}=", value) if import.public_send(attribute).blank? + end + end + + def validation_errors(import) + import.errors.full_messages.map { |message| { code: "validation_failed", message: message } } + end + + def csv_preflight_payload(import:, type:, filename:, content_type:, content:, parsed_rows_count:, csv_headers:, missing_required_headers:, errors:, warnings:) + { + type: type, + valid: errors.empty?, + content: content_payload(filename, content_type, content), + stats: { + rows_count: parsed_rows_count + }, + headers: csv_headers, + required_headers: required_header_labels(import), + missing_required_headers: missing_required_headers, + errors: errors, + warnings: warnings + } + end + + def required_header_labels(import) + import.required_column_keys.filter_map do |key| + import.respond_to?("#{key}_col_label") ? import.public_send("#{key}_col_label").presence || key.to_s : key.to_s + end + end + + def missing_required_headers(import, headers) + normalized_headers = Array(headers).compact.to_h { |header| [ normalized_header(header), header ] } + + required_header_labels(import).reject do |header| + normalized_headers.key?(normalized_header(header)) + end + end + + def normalized_header(header) + header.to_s.strip.downcase.gsub(/\*/, "").gsub(/[\s-]+/, "_") + end + + def missing_csv_content_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "missing_content", + message: "Provide a CSV file or raw_file_content." + } + ) + end + + def missing_sure_content_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "missing_content", + message: "Provide a Sure NDJSON file or raw_file_content." + } + ) + end + + def csv_file_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "file_too_large", + message: "File is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." + } + ) + end + + def csv_content_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "content_too_large", + message: "Content is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." + } + ) + end + + def invalid_csv_file_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "invalid_file_type", + message: "Invalid file type. Please upload a CSV file." + } + ) + end + + def sure_file_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "file_too_large", + message: "File is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." + } + ) + end + + def sure_content_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "content_too_large", + message: "Content is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." + } + ) + end + + def invalid_sure_file_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "invalid_file_type", + message: "Invalid file type. Please upload a Sure NDJSON file." + } + ) + end + + def raise_response(response) + raise PreflightError, response + end + + def unsupported_import_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "unsupported_import_type", + message: "Preflight supports CSV import types and SureImport." + } + ) + end +end diff --git a/app/models/mint_import.rb b/app/models/mint_import.rb index 87997d400..1dc5e5fb5 100644 --- a/app/models/mint_import.rb +++ b/app/models/mint_import.rb @@ -1,6 +1,24 @@ class MintImport < Import after_create :set_mappings + DEFAULT_COLUMN_MAPPINGS = { + signage_convention: "inflows_positive", + date_col_label: "Date", + date_format: "%m/%d/%Y", + name_col_label: "Description", + amount_col_label: "Amount", + currency_col_label: "Currency", + account_col_label: "Account Name", + category_col_label: "Category", + tags_col_label: "Labels", + notes_col_label: "Notes", + entity_type_col_label: "Transaction Type" + }.freeze + + def self.default_column_mappings + DEFAULT_COLUMN_MAPPINGS + end + def generate_rows_from_csv rows.destroy_all @@ -83,18 +101,7 @@ class MintImport < Import private def set_mappings - self.signage_convention = "inflows_positive" - self.date_col_label = "Date" - self.date_format = "%m/%d/%Y" - self.name_col_label = "Description" - self.amount_col_label = "Amount" - self.currency_col_label = "Currency" - self.account_col_label = "Account Name" - self.category_col_label = "Category" - self.tags_col_label = "Labels" - self.notes_col_label = "Notes" - self.entity_type_col_label = "Transaction Type" - + assign_attributes(self.class.default_column_mappings) save! end end diff --git a/app/models/sure_import.rb b/app/models/sure_import.rb index 6666fc575..23815437a 100644 --- a/app/models/sure_import.rb +++ b/app/models/sure_import.rb @@ -1,5 +1,17 @@ class SureImport < Import MAX_NDJSON_SIZE = 10.megabytes + IMPORTABLE_NDJSON_TYPES = { + "Account" => :accounts, + "Category" => :categories, + "Tag" => :tags, + "Merchant" => :merchants, + "Transaction" => :transactions, + "Trade" => :trades, + "Valuation" => :valuations, + "Budget" => :budgets, + "BudgetCategory" => :budget_categories, + "Rule" => :rules + }.freeze ALLOWED_NDJSON_CONTENT_TYPES = %w[ application/x-ndjson application/ndjson @@ -11,6 +23,14 @@ class SureImport < Import has_one_attached :ndjson_file, dependent: :purge_later class << self + def max_row_count + 100_000 + end + + def max_ndjson_size + MAX_NDJSON_SIZE + end + # Counts JSON lines by top-level "type" (used for dry-run summaries and row limits). def ndjson_line_type_counts(content) return {} unless content.present? @@ -21,7 +41,7 @@ class SureImport < Import begin record = JSON.parse(line) - counts[record["type"]] += 1 if record["type"] + counts[record["type"]] += 1 if record.is_a?(Hash) && record["type"] && record.key?("data") rescue JSON::ParserError # Skip invalid lines end @@ -30,19 +50,17 @@ class SureImport < Import end def dry_run_totals_from_ndjson(content) - counts = ndjson_line_type_counts(content) - { - accounts: counts["Account"] || 0, - categories: counts["Category"] || 0, - tags: counts["Tag"] || 0, - merchants: counts["Merchant"] || 0, - transactions: counts["Transaction"] || 0, - trades: counts["Trade"] || 0, - valuations: counts["Valuation"] || 0, - budgets: counts["Budget"] || 0, - budget_categories: counts["BudgetCategory"] || 0, - rules: counts["Rule"] || 0 - } + dry_run_totals_from_line_type_counts(ndjson_line_type_counts(content)) + end + + def dry_run_totals_from_line_type_counts(counts) + IMPORTABLE_NDJSON_TYPES.to_h do |record_type, entity_key| + [ entity_key, counts[record_type] || 0 ] + end + end + + def importable_ndjson_types + IMPORTABLE_NDJSON_TYPES.keys end def valid_ndjson_first_line?(str) @@ -53,7 +71,7 @@ class SureImport < Import begin record = JSON.parse(first_line) - record.key?("type") && record.key?("data") + record.is_a?(Hash) && record.key?("type") && record.key?("data") rescue JSON::ParserError false end @@ -121,7 +139,7 @@ class SureImport < Import end def max_row_count - 100_000 + self.class.max_row_count end # Row total for max-row enforcement (counts every parsed line with a "type", including unsupported types). diff --git a/config/routes.rb b/config/routes.rb index 3a59f340b..93f8655e7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -453,6 +453,7 @@ Rails.application.routes.draw do get :download, on: :member end resources :imports, only: [ :index, :show, :create ] do + post :preflight, on: :collection get :rows, on: :member end resource :usage, only: [ :show ], controller: :usage diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9db36b8f7..ab6a787bb 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1733,6 +1733,114 @@ components: unassigned_mappings_count: type: integer minimum: 0 + ImportPreflightContent: + type: object + required: + - filename + - content_type + - byte_size + properties: + filename: + type: string + content_type: + type: string + byte_size: + type: integer + minimum: 0 + ImportPreflightError: + type: object + required: + - code + - message + properties: + code: + type: string + message: + type: string + ImportPreflightStats: + type: object + required: + - rows_count + properties: + rows_count: + type: integer + minimum: 0 + description: CSV parsed non-header rows, or nonblank Sure NDJSON lines. + valid_rows_count: + type: integer + minimum: 0 + description: SureImport only. Valid NDJSON records. + invalid_rows_count: + type: integer + minimum: 0 + description: SureImport only. Invalid NDJSON records. CSV malformed content + returns a 422 instead. + entity_counts: + type: object + additionalProperties: + type: integer + nullable: true + record_type_counts: + type: object + additionalProperties: + type: integer + nullable: true + ImportPreflight: + type: object + required: + - type + - valid + - content + - stats + - errors + - warnings + properties: + type: + type: string + enum: + - TransactionImport + - TradeImport + - AccountImport + - MintImport + - CategoryImport + - RuleImport + - SureImport + valid: + type: boolean + content: + "$ref": "#/components/schemas/ImportPreflightContent" + stats: + "$ref": "#/components/schemas/ImportPreflightStats" + headers: + type: array + items: + type: string + nullable: true + required_headers: + type: array + items: + type: string + nullable: true + missing_required_headers: + type: array + items: + type: string + nullable: true + errors: + type: array + items: + "$ref": "#/components/schemas/ImportPreflightError" + warnings: + type: array + items: + type: string + ImportPreflightResponse: + type: object + required: + - data + properties: + data: + "$ref": "#/components/schemas/ImportPreflight" ImportStatusSummary: type: object required: @@ -4387,7 +4495,7 @@ paths: post: summary: Create import description: Create a new import from raw CSV content, inline Sure NDJSON content, - or an uploaded Sure NDJSON file. + or an uploaded Sure NDJSON file. CSV content is limited to 10MB. tags: - Imports security: @@ -4422,8 +4530,9 @@ paths: properties: raw_file_content: type: string - description: Raw CSV or Sure NDJSON content as a string. Required - for SureImport unless a multipart file is uploaded. + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. Required for SureImport unless a multipart + file is uploaded. type: type: string enum: @@ -4527,8 +4636,9 @@ paths: properties: raw_file_content: type: string - description: Raw CSV or Sure NDJSON content as a string. Required - for SureImport unless a multipart file is uploaded. + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. Required for SureImport unless a multipart + file is uploaded. type: type: string enum: @@ -4715,6 +4825,264 @@ paths: application/json: schema: "$ref": "#/components/schemas/ErrorResponse" + "/api/v1/imports/preflight": + post: + summary: Validate import content without creating an import + description: Validate CSV or Sure NDJSON import content and return counts, headers, + warnings, and validation errors without persisting an import or enqueueing + jobs. CSV content is limited to 10MB. + tags: + - Imports + security: + - apiKeyAuth: [] + parameters: [] + responses: + '200': + description: import content preflighted + content: + application/json: + schema: + "$ref": "#/components/schemas/ImportPreflightResponse" + '401': + description: unauthorized + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorResponse" + '422': + description: missing or invalid content + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorResponse" + '404': + description: account not found + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorResponse" + requestBody: + content: + application/json: + schema: + type: object + properties: + raw_file_content: + type: string + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. + file: + type: string + format: binary + description: CSV or Sure NDJSON upload when using multipart/form-data. + CSV files are limited to 10MB. + type: + type: string + enum: + - TransactionImport + - TradeImport + - AccountImport + - MintImport + - CategoryImport + - RuleImport + - SureImport + description: Import type to validate (defaults to TransactionImport) + account_id: + type: string + format: uuid + description: Account ID used for account-scoped CSV import validation + date_col_label: + type: string + description: CSV imports only. Header name for the date column + amount_col_label: + type: string + description: CSV imports only. Header name for the amount column + name_col_label: + type: string + description: CSV imports only. Header name for the transaction name + column + category_col_label: + type: string + description: CSV imports only. Header name for the category column + tags_col_label: + type: string + description: CSV imports only. Header name for the tags column + notes_col_label: + type: string + description: CSV imports only. Header name for the notes column + account_col_label: + type: string + description: CSV imports only. Header name for the account column + qty_col_label: + type: string + description: CSV trade imports only. Header name for the quantity + column + ticker_col_label: + type: string + description: CSV trade imports only. Header name for the ticker + column + price_col_label: + type: string + description: CSV trade imports only. Header name for the price column + entity_type_col_label: + type: string + description: CSV imports only. Header name for the entity type column + currency_col_label: + type: string + description: CSV imports only. Header name for the currency column + exchange_operating_mic_col_label: + type: string + description: CSV trade imports only. Header name for the exchange + operating MIC column + date_format: + type: string + description: CSV imports only. Date format pattern + number_format: + type: string + enum: + - '1,234.56' + - 1.234,56 + - 1 234,56 + - '1,234' + description: CSV imports only. Number format for parsing amounts + signage_convention: + type: string + enum: + - inflows_positive + - inflows_negative + description: CSV imports only. How to interpret positive/negative + amounts + col_sep: + type: string + enum: + - "," + - ";" + description: CSV imports only. Column separator + rows_to_skip: + type: integer + minimum: 0 + description: CSV imports only. Number of leading rows to skip before + reading headers + amount_type_strategy: + type: string + enum: + - signed_amount + - custom_column + description: CSV imports only. Amount parsing strategy + amount_type_inflow_value: + type: string + description: CSV imports only. Column value that marks an amount + as an inflow when using custom_column strategy + multipart/form-data: + schema: + type: object + properties: + raw_file_content: + type: string + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. + file: + type: string + format: binary + description: CSV or Sure NDJSON upload when using multipart/form-data. + CSV files are limited to 10MB. + type: + type: string + enum: + - TransactionImport + - TradeImport + - AccountImport + - MintImport + - CategoryImport + - RuleImport + - SureImport + description: Import type to validate (defaults to TransactionImport) + account_id: + type: string + format: uuid + description: Account ID used for account-scoped CSV import validation + date_col_label: + type: string + description: CSV imports only. Header name for the date column + amount_col_label: + type: string + description: CSV imports only. Header name for the amount column + name_col_label: + type: string + description: CSV imports only. Header name for the transaction name + column + category_col_label: + type: string + description: CSV imports only. Header name for the category column + tags_col_label: + type: string + description: CSV imports only. Header name for the tags column + notes_col_label: + type: string + description: CSV imports only. Header name for the notes column + account_col_label: + type: string + description: CSV imports only. Header name for the account column + qty_col_label: + type: string + description: CSV trade imports only. Header name for the quantity + column + ticker_col_label: + type: string + description: CSV trade imports only. Header name for the ticker + column + price_col_label: + type: string + description: CSV trade imports only. Header name for the price column + entity_type_col_label: + type: string + description: CSV imports only. Header name for the entity type column + currency_col_label: + type: string + description: CSV imports only. Header name for the currency column + exchange_operating_mic_col_label: + type: string + description: CSV trade imports only. Header name for the exchange + operating MIC column + date_format: + type: string + description: CSV imports only. Date format pattern + number_format: + type: string + enum: + - '1,234.56' + - 1.234,56 + - 1 234,56 + - '1,234' + description: CSV imports only. Number format for parsing amounts + signage_convention: + type: string + enum: + - inflows_positive + - inflows_negative + description: CSV imports only. How to interpret positive/negative + amounts + col_sep: + type: string + enum: + - "," + - ";" + description: CSV imports only. Column separator + rows_to_skip: + type: integer + minimum: 0 + description: CSV imports only. Number of leading rows to skip before + reading headers + amount_type_strategy: + type: string + enum: + - signed_amount + - custom_column + description: CSV imports only. Amount parsing strategy + amount_type_inflow_value: + type: string + description: CSV imports only. Column value that marks an amount + as an inflow when using custom_column strategy "/api/v1/merchants": get: summary: List merchants diff --git a/spec/requests/api/v1/imports_spec.rb b/spec/requests/api/v1/imports_spec.rb index 540422c33..a12cb7a8c 100644 --- a/spec/requests/api/v1/imports_spec.rb +++ b/spec/requests/api/v1/imports_spec.rb @@ -123,7 +123,7 @@ RSpec.describe 'API V1 Imports', type: :request do end post 'Create import' do - description 'Create a new import from raw CSV content, inline Sure NDJSON content, or an uploaded Sure NDJSON file.' + description 'Create a new import from raw CSV content, inline Sure NDJSON content, or an uploaded Sure NDJSON file. CSV content is limited to 10MB.' tags 'Imports' security [ { apiKeyAuth: [] } ] consumes 'application/json', 'multipart/form-data' @@ -134,7 +134,7 @@ RSpec.describe 'API V1 Imports', type: :request do properties: { raw_file_content: { type: :string, - description: 'Raw CSV or Sure NDJSON content as a string. Required for SureImport unless a multipart file is uploaded.' + description: 'Raw CSV or Sure NDJSON content as a string. CSV content is limited to 10MB. Required for SureImport unless a multipart file is uploaded.' }, type: { type: :string, @@ -365,4 +365,126 @@ RSpec.describe 'API V1 Imports', type: :request do end end end + + path '/api/v1/imports/preflight' do + post 'Validate import content without creating an import' do + description 'Validate CSV or Sure NDJSON import content and return counts, headers, warnings, and validation errors without persisting an import or enqueueing jobs. CSV content is limited to 10MB.' + tags 'Imports' + security [ { apiKeyAuth: [] } ] + consumes 'application/json', 'multipart/form-data' + produces 'application/json' + + parameter name: :body, in: :body, required: false, schema: { + type: :object, + properties: { + raw_file_content: { + type: :string, + description: 'Raw CSV or Sure NDJSON content as a string. CSV content is limited to 10MB.' + }, + file: { + type: :string, + format: :binary, + description: 'CSV or Sure NDJSON upload when using multipart/form-data. CSV files are limited to 10MB.' + }, + type: { + type: :string, + enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport], + description: 'Import type to validate (defaults to TransactionImport)' + }, + account_id: { + type: :string, + format: :uuid, + description: 'Account ID used for account-scoped CSV import validation' + }, + date_col_label: { type: :string, description: 'CSV imports only. Header name for the date column' }, + amount_col_label: { type: :string, description: 'CSV imports only. Header name for the amount column' }, + name_col_label: { type: :string, description: 'CSV imports only. Header name for the transaction name column' }, + category_col_label: { type: :string, description: 'CSV imports only. Header name for the category column' }, + tags_col_label: { type: :string, description: 'CSV imports only. Header name for the tags column' }, + notes_col_label: { type: :string, description: 'CSV imports only. Header name for the notes column' }, + account_col_label: { type: :string, description: 'CSV imports only. Header name for the account column' }, + qty_col_label: { type: :string, description: 'CSV trade imports only. Header name for the quantity column' }, + ticker_col_label: { type: :string, description: 'CSV trade imports only. Header name for the ticker column' }, + price_col_label: { type: :string, description: 'CSV trade imports only. Header name for the price column' }, + entity_type_col_label: { type: :string, description: 'CSV imports only. Header name for the entity type column' }, + currency_col_label: { type: :string, description: 'CSV imports only. Header name for the currency column' }, + exchange_operating_mic_col_label: { type: :string, description: 'CSV trade imports only. Header name for the exchange operating MIC column' }, + date_format: { type: :string, description: 'CSV imports only. Date format pattern' }, + number_format: { + type: :string, + enum: [ '1,234.56', '1.234,56', '1 234,56', '1,234' ], + description: 'CSV imports only. Number format for parsing amounts' + }, + signage_convention: { + type: :string, + enum: %w[inflows_positive inflows_negative], + description: 'CSV imports only. How to interpret positive/negative amounts' + }, + col_sep: { + type: :string, + enum: [ ',', ';' ], + description: 'CSV imports only. Column separator' + }, + rows_to_skip: { + type: :integer, + minimum: 0, + description: 'CSV imports only. Number of leading rows to skip before reading headers' + }, + amount_type_strategy: { + type: :string, + enum: %w[signed_amount custom_column], + description: 'CSV imports only. Amount parsing strategy' + }, + amount_type_inflow_value: { + type: :string, + description: 'CSV imports only. Column value that marks an amount as an inflow when using custom_column strategy' + } + } + } + + response '200', 'import content preflighted' do + schema '$ref' => '#/components/schemas/ImportPreflightResponse' + + let(:body) do + { + raw_file_content: "date,amount,name\n01/15/2024,50.00,New Transaction", + type: 'TransactionImport', + account_id: account.id, + date_col_label: 'date', + amount_col_label: 'amount', + name_col_label: 'name' + } + end + + run_test! + end + + response '401', 'unauthorized' do + schema '$ref' => '#/components/schemas/ErrorResponse' + let(:'X-Api-Key') { nil } + let(:body) { { raw_file_content: "date,amount\n01/15/2024,50.00" } } + + run_test! + end + + response '422', 'missing or invalid content' do + schema '$ref' => '#/components/schemas/ErrorResponse' + let(:body) { { type: 'SureImport' } } + + run_test! + end + + response '404', 'account not found' do + schema '$ref' => '#/components/schemas/ErrorResponse' + let(:body) do + { + raw_file_content: "date,amount,name\n01/15/2024,50.00,New Transaction", + account_id: SecureRandom.uuid + } + end + + run_test! + end + end + end end diff --git a/spec/swagger_helper.rb b/spec/swagger_helper.rb index 0692d2aad..d27ad818f 100644 --- a/spec/swagger_helper.rb +++ b/spec/swagger_helper.rb @@ -947,6 +947,94 @@ RSpec.configure do |config| unassigned_mappings_count: { type: :integer, minimum: 0 } } }, + ImportPreflightContent: { + type: :object, + required: %w[filename content_type byte_size], + properties: { + filename: { type: :string }, + content_type: { type: :string }, + byte_size: { type: :integer, minimum: 0 } + } + }, + ImportPreflightError: { + type: :object, + required: %w[code message], + properties: { + code: { type: :string }, + message: { type: :string } + } + }, + ImportPreflightStats: { + type: :object, + required: %w[rows_count], + properties: { + rows_count: { + type: :integer, + minimum: 0, + description: 'CSV parsed non-header rows, or nonblank Sure NDJSON lines.' + }, + valid_rows_count: { + type: :integer, + minimum: 0, + description: 'SureImport only. Valid NDJSON records.' + }, + invalid_rows_count: { + type: :integer, + minimum: 0, + description: 'SureImport only. Invalid NDJSON records. CSV malformed content returns a 422 instead.' + }, + entity_counts: { + type: :object, + additionalProperties: { type: :integer }, + nullable: true + }, + record_type_counts: { + type: :object, + additionalProperties: { type: :integer }, + nullable: true + } + } + }, + ImportPreflight: { + type: :object, + required: %w[type valid content stats errors warnings], + properties: { + type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] }, + valid: { type: :boolean }, + content: { '$ref' => '#/components/schemas/ImportPreflightContent' }, + stats: { '$ref' => '#/components/schemas/ImportPreflightStats' }, + headers: { + type: :array, + items: { type: :string }, + nullable: true + }, + required_headers: { + type: :array, + items: { type: :string }, + nullable: true + }, + missing_required_headers: { + type: :array, + items: { type: :string }, + nullable: true + }, + errors: { + type: :array, + items: { '$ref' => '#/components/schemas/ImportPreflightError' } + }, + warnings: { + type: :array, + items: { type: :string } + } + } + }, + ImportPreflightResponse: { + type: :object, + required: %w[data], + properties: { + data: { '$ref' => '#/components/schemas/ImportPreflight' } + } + }, ImportStatusSummary: { type: :object, required: %w[uploaded configured terminal], diff --git a/test/controllers/api/v1/imports_controller_test.rb b/test/controllers/api/v1/imports_controller_test.rb index bab5e3787..d414e7e1b 100644 --- a/test/controllers/api/v1/imports_controller_test.rb +++ b/test/controllers/api/v1/imports_controller_test.rb @@ -405,9 +405,7 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest original_filename: "large.ndjson" ) - original_value = SureImport::MAX_NDJSON_SIZE - SureImport.send(:remove_const, :MAX_NDJSON_SIZE) - SureImport.const_set(:MAX_NDJSON_SIZE, test_limit) + SureImport.stubs(:max_ndjson_size).returns(test_limit) assert_no_difference("Import.count") do post api_v1_imports_url, @@ -421,9 +419,6 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest assert_response :unprocessable_entity json_response = JSON.parse(response.body) assert_equal "file_too_large", json_response["error"] - ensure - SureImport.send(:remove_const, :MAX_NDJSON_SIZE) - SureImport.const_set(:MAX_NDJSON_SIZE, original_value) end test "should reject Sure import uploaded file with invalid type" do @@ -551,6 +546,473 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest assert_equal "invalid_ndjson", json_response["error"] end + test "should preflight CSV import without persisting records" do + csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction" + + assert_no_difference([ "Import.count", "Import::Row.count" ]) do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@api_key) + end + + assert_response :success + json_response = JSON.parse(response.body) + data = json_response["data"] + + assert_equal "TransactionImport", data["type"] + assert_equal true, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_not data["stats"].key?("valid_rows_count") + assert_not data["stats"].key?("invalid_rows_count") + assert_equal %w[date amount name], data["headers"] + assert_empty data["missing_required_headers"] + assert_empty data["errors"] + end + + test "should report missing required CSV headers during preflight" do + csv_content = "name\nMissing Amount" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_not data["stats"].key?("valid_rows_count") + assert_not data["stats"].key?("invalid_rows_count") + assert_equal [ "date", "amount" ], data["missing_required_headers"] + assert_equal "missing_required_headers", data["errors"].first["code"] + end + + test "should apply rows_to_skip before CSV preflight header validation" do + csv_content = [ + "Generated by bank export", + "posted,amount,description", + "2024-01-01,-10.00,Coffee" + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + rows_to_skip: 1, + date_col_label: "posted", + amount_col_label: "amount", + name_col_label: "description", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal true, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_equal %w[posted amount description], data["headers"] + assert_empty data["missing_required_headers"] + end + + test "should preflight semicolon separated CSV content" do + csv_content = "date;amount;name\n2024-01-01;-10.00;Coffee" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + col_sep: ";", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal true, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_equal %w[date amount name], data["headers"] + end + + test "should report invalid preflight CSV parser config without parsing" do + csv_content = "date,amount,name\n2024-01-01,-10.00,Coffee" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + col_sep: "", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 0, data["stats"]["rows_count"] + assert_empty data["headers"] + assert_equal "validation_failed", data["errors"].first["code"] + end + + test "should reject malformed CSV during preflight" do + csv_content = "date,amount,name\n2024-01-01,-10.00,\"Coffee Shop" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "invalid_csv", json_response["error"] + end + + test "should include preflight exception message in internal server error response" do + Import::Preflight.any_instance.stubs(:call).raises(StandardError, "boom") + + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n2024-01-01,-10.00,Coffee", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name" + }, + headers: api_headers(@read_only_api_key) + + assert_response :internal_server_error + json_response = JSON.parse(response.body) + assert_equal "internal_server_error", json_response["error"] + assert_equal "Error: boom", json_response["message"] + end + + test "should reject unknown preflight import type" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "FakeImport", + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + response_data = JSON.parse(response.body) + assert_equal "invalid_import_type", response_data["error"] + assert_not response_data.key?("errors") + end + + test "should reject import types excluded from preflight" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "QifImport", + raw_file_content: "!Type:Bank\nD01/01/2024\nT-10.00\nPTest\n^" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + response_data = JSON.parse(response.body) + assert_equal "invalid_import_type", response_data["error"] + assert_not response_data.key?("errors") + assert_not_includes response_data["message"], "QifImport" + assert_not_includes response_data["message"], "PdfImport" + end + + test "should report empty CSV preflight content as invalid" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 0, data["stats"]["rows_count"] + assert_equal "no_data_rows", data["errors"].first["code"] + assert_empty data["warnings"] + end + + test "should preflight Sure import without persisting records" do + ndjson_content = [ + { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json, + { type: "Transaction", data: { id: "entry_1", account_id: "account_1" } }.to_json + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content + }, + headers: api_headers(@api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal "SureImport", data["type"] + assert_equal true, data["valid"] + assert_equal 2, data["stats"]["rows_count"] + assert_equal 1, data["stats"]["entity_counts"]["accounts"] + assert_equal 1, data["stats"]["entity_counts"]["transactions"] + assert_empty data["errors"] + end + + test "should report invalid Sure import NDJSON during preflight" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: "not ndjson" + }, + headers: api_headers(@api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 1, data["stats"]["invalid_rows_count"] + assert_equal "invalid_json", data["errors"].first["code"] + end + + test "should report non-object Sure import NDJSON records during preflight" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: "[]" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 1, data["stats"]["invalid_rows_count"] + assert_equal "invalid_ndjson_record", data["errors"].first["code"] + end + + test "should report empty Sure import file as invalid during preflight" do + empty_file = Rack::Test::UploadedFile.new( + StringIO.new(""), + "application/x-ndjson", + original_filename: "empty.ndjson" + ) + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + file: empty_file + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 0, data["stats"]["rows_count"] + assert_equal "no_data_rows", data["errors"].first["code"] + assert_empty data["warnings"] + end + + test "should reject preflight with no file or raw content" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { type: "SureImport" }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + assert_equal "missing_content", JSON.parse(response.body)["error"] + end + + test "should reject oversized file uploads during preflight" do + test_limit = 1.kilobyte + large_file = Rack::Test::UploadedFile.new( + StringIO.new("x" * (test_limit + 1)), + "text/csv", + original_filename: "large.csv" + ) + + Import.stubs(:max_csv_size).returns(test_limit) + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { file: large_file }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + assert_equal "file_too_large", JSON.parse(response.body)["error"] + end + + test "should preflight with read-only API key" do + csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + assert_equal true, JSON.parse(response.body)["data"]["valid"] + end + + test "should require authentication for preflight" do + post preflight_api_v1_imports_url, params: { + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction" + } + + assert_response :unauthorized + end + + test "should return not found for preflight account outside family" do + other_family = Family.create!(name: "Other Family", currency: "USD", locale: "en") + other_depository = Depository.create!(subtype: "checking") + other_account = Account.create!( + family: other_family, + name: "Other Account", + currency: "USD", + classification: "asset", + accountable: other_depository, + balance: 0 + ) + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: other_account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :not_found + assert_equal "record_not_found", JSON.parse(response.body)["error"] + end + + test "should return not found for malformed preflight account id" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: "not-a-uuid" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :not_found + assert_equal "record_not_found", JSON.parse(response.body)["error"] + end + + test "should apply Mint defaults before preflight header validation" do + mint_content = [ + "Date,Amount,Account Name,Description,Category,Labels,Currency,Notes,Transaction Type", + "01/01/2024,-8.55,Checking,Starbucks,Food & Drink,Coffee,USD,Morning coffee,debit" + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "MintImport", + raw_file_content: mint_content + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal "MintImport", data["type"] + assert_equal true, data["valid"] + assert_empty data["missing_required_headers"] + assert_includes data["required_headers"], "Date" + assert_includes data["required_headers"], "Amount" + end + + test "should not overwrite explicit Mint preflight column mappings with defaults" do + mint_content = [ + "Posted On,Value,Description", + "01/01/2024,-8.55,Starbucks" + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "MintImport", + raw_file_content: mint_content, + date_col_label: "Posted On", + amount_col_label: "Value" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal true, data["valid"] + assert_equal [ "Posted On", "Value" ], data["required_headers"] + assert_empty data["missing_required_headers"] + end + test "should create import and auto-publish when configured and requested" do csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction" @@ -633,9 +1095,7 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest test_limit = 1.kilobyte large_content = "x" * (test_limit + 1) - original_value = Import::MAX_CSV_SIZE - Import.send(:remove_const, :MAX_CSV_SIZE) - Import.const_set(:MAX_CSV_SIZE, test_limit) + Import.stubs(:max_csv_size).returns(test_limit) assert_no_difference("Import.count") do post api_v1_imports_url, @@ -646,9 +1106,6 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest assert_response :unprocessable_entity json_response = JSON.parse(response.body) assert_equal "content_too_large", json_response["error"] - ensure - Import.send(:remove_const, :MAX_CSV_SIZE) - Import.const_set(:MAX_CSV_SIZE, original_value) end test "should accept file upload with valid csv mime type" do diff --git a/test/models/mint_import_test.rb b/test/models/mint_import_test.rb index 15095cefa..599e18b10 100644 --- a/test/models/mint_import_test.rb +++ b/test/models/mint_import_test.rb @@ -5,6 +5,14 @@ class MintImportTest < ActiveSupport::TestCase @family = families(:dylan_family) end + test "default column mappings are applied after create" do + import = @family.imports.create!(type: "MintImport") + + MintImport.default_column_mappings.each do |attribute, value| + assert_equal value, import.public_send(attribute) + end + end + test "generated rows preserve stable source row numbers" do import = @family.imports.create!( type: "MintImport", diff --git a/test/models/sure_import_test.rb b/test/models/sure_import_test.rb index 65fbe483c..9345dea60 100644 --- a/test/models/sure_import_test.rb +++ b/test/models/sure_import_test.rb @@ -37,9 +37,37 @@ class SureImportTest < ActiveSupport::TestCase end test "max_row_count is higher than standard imports" do + assert_equal 100_000, SureImport.max_row_count assert_equal 100_000, @import.max_row_count end + test "dry_run totals can be derived from existing line type counts" do + counts = { + "Account" => 2, + "Transaction" => 3, + "UnknownType" => 4 + } + + dry_run = SureImport.dry_run_totals_from_line_type_counts(counts) + + assert_equal 2, dry_run[:accounts] + assert_equal 3, dry_run[:transactions] + assert_equal 0, dry_run[:categories] + assert_not dry_run.key?(:unknown_type) + end + + test "ndjson line type counts ignore records without data" do + ndjson = [ + { type: "Account", data: { id: "uuid-1" } }, + { type: "Transaction" }, + { data: { id: "uuid-2" } } + ].map(&:to_json).join("\n") + + counts = SureImport.ndjson_line_type_counts(ndjson) + + assert_equal({ "Account" => 1 }, counts) + end + test "csv_template returns nil" do assert_nil @import.csv_template end From 9e6fbc1374142833ec220e57cacda9be7afc2111 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 15:02:15 -0700 Subject: [PATCH 06/31] test(recurring): prove export roundtrip semantics (#1727) * test(recurring): prove export roundtrip semantics * test(recurring): assert export roundtrip payload --- test/models/family/data_importer_test.rb | 79 ++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb index b216acc9d..22691648c 100644 --- a/test/models/family/data_importer_test.rb +++ b/test/models/family/data_importer_test.rb @@ -313,6 +313,85 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal(-89.99, recurring_transaction.expected_amount_avg.to_f) end + test "round trips recurring transaction export semantics" do + source_family = Family.create!(name: "Recurring Source", currency: "USD") + source_account = source_family.accounts.create!( + name: "Source Checking", + accountable: Depository.new, + balance: 1000, + currency: "USD" + ) + source_merchant = source_family.merchants.create!(name: "Internet Provider") + + source_family.recurring_transactions.create!( + account: source_account, + merchant: source_merchant, + amount: -89.99, + currency: "USD", + expected_day_of_month: 14, + last_occurrence_date: Date.parse("2024-01-14"), + next_expected_date: Date.parse("2024-02-14"), + status: "active", + occurrence_count: 6, + manual: true, + expected_amount_min: -95, + expected_amount_max: -85, + expected_amount_avg: -89.99 + ) + + source_family.recurring_transactions.create!( + name: "Quarterly Insurance", + amount: 240, + currency: "USD", + expected_day_of_month: 28, + last_occurrence_date: Date.parse("2024-01-28"), + next_expected_date: Date.parse("2024-04-28"), + status: "inactive", + occurrence_count: 2, + manual: false + ) + + ndjson = nil + Zip::File.open_buffer(Family::DataExporter.new(source_family).generate_export) do |zip| + ndjson = zip.read("all.ndjson") + end + + assert_not_nil ndjson + assert ndjson.include?('"type":"RecurringTransaction"') + + Family::DataImporter.new(@family, ndjson).import! + + assert_equal 2, @family.recurring_transactions.count + + restored_account = @family.accounts.find_by!(name: "Source Checking") + restored_merchant = @family.merchants.find_by!(name: "Internet Provider") + restored_provider = @family.recurring_transactions.find_by!(merchant: restored_merchant) + + assert_equal restored_account, restored_provider.account + assert_equal(-89.99, restored_provider.amount.to_f) + assert_equal "USD", restored_provider.currency + assert_equal 14, restored_provider.expected_day_of_month + assert_equal Date.parse("2024-01-14"), restored_provider.last_occurrence_date + assert_equal Date.parse("2024-02-14"), restored_provider.next_expected_date + assert_equal "active", restored_provider.status + assert_equal 6, restored_provider.occurrence_count + assert_equal true, restored_provider.manual + assert_equal(-95.0, restored_provider.expected_amount_min.to_f) + assert_equal(-85.0, restored_provider.expected_amount_max.to_f) + assert_equal(-89.99, restored_provider.expected_amount_avg.to_f) + + restored_named = @family.recurring_transactions.find_by!(name: "Quarterly Insurance") + assert_nil restored_named.account + assert_nil restored_named.merchant + assert_equal 240.0, restored_named.amount.to_f + assert_equal 28, restored_named.expected_day_of_month + assert_equal Date.parse("2024-01-28"), restored_named.last_occurrence_date + assert_equal Date.parse("2024-04-28"), restored_named.next_expected_date + assert_equal "inactive", restored_named.status + assert_equal 2, restored_named.occurrence_count + assert_equal false, restored_named.manual + end + test "imports recurring transactions with unknown status fallback" do ndjson = build_ndjson([ { From 325084e342552f51caf46001e56afe0decbfc6ba Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 15:14:13 -0700 Subject: [PATCH 07/31] fix(api): include disabled-account transaction history (#1723) * fix(api): include disabled-account transaction history * fix(api): hide pending deletion transaction history --- .../api/v1/transactions_controller.rb | 9 +- docs/api/openapi.yaml | 7 +- spec/requests/api/v1/transactions_spec.rb | 3 +- .../api/v1/transactions_controller_test.rb | 117 +++++++++++++++++- 4 files changed, 130 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/v1/transactions_controller.rb b/app/controllers/api/v1/transactions_controller.rb index 5dd65aba3..f208d83b2 100644 --- a/app/controllers/api/v1/transactions_controller.rb +++ b/app/controllers/api/v1/transactions_controller.rb @@ -10,8 +10,11 @@ class Api::V1::TransactionsController < Api::V1::BaseController def index family = current_resource_owner.family - accessible_account_ids = family.accounts.accessible_by(current_resource_owner).select(:id) - transactions_query = family.transactions.visible + accessible_account_ids = family.accounts + .accessible_by(current_resource_owner) + .where.not(status: "pending_deletion") + .select(:id) + transactions_query = family.transactions .joins(:entry).where(entries: { account_id: accessible_account_ids }) # Apply filters @@ -198,6 +201,8 @@ end private def set_transaction + raise ActiveRecord::RecordNotFound unless valid_uuid?(params[:id]) + family = current_resource_owner.family @transaction = family.transactions .joins(entry: :account) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index ab6a787bb..ebc4c4425 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -6505,6 +6505,8 @@ paths: - Transactions security: - apiKeyAuth: [] + description: Returns global ledger history for accessible accounts, including + disabled accounts but excluding accounts pending deletion. parameters: - name: page in: query @@ -6716,10 +6718,11 @@ paths: parameters: - name: id in: path - required: true - description: Transaction ID schema: type: string + format: uuid + required: true + description: Transaction ID get: summary: Retrieve a transaction tags: diff --git a/spec/requests/api/v1/transactions_spec.rb b/spec/requests/api/v1/transactions_spec.rb index d57a40b6a..2a346906c 100644 --- a/spec/requests/api/v1/transactions_spec.rb +++ b/spec/requests/api/v1/transactions_spec.rb @@ -89,6 +89,7 @@ RSpec.describe 'API V1 Transactions', type: :request do get 'List transactions' do tags 'Transactions' security [ { apiKeyAuth: [] } ] + description 'Returns global ledger history for accessible accounts, including disabled accounts but excluding accounts pending deletion.' produces 'application/json' parameter name: :page, in: :query, type: :integer, required: false, description: 'Page number (default: 1)' @@ -268,7 +269,7 @@ RSpec.describe 'API V1 Transactions', type: :request do end path '/api/v1/transactions/{id}' do - parameter name: :id, in: :path, type: :string, required: true, description: 'Transaction ID' + parameter name: :id, in: :path, schema: { type: :string, format: :uuid }, required: true, description: 'Transaction ID' get 'Retrieve a transaction' do tags 'Transactions' diff --git a/test/controllers/api/v1/transactions_controller_test.rb b/test/controllers/api/v1/transactions_controller_test.rb index 002c80c30..1f5123d9e 100644 --- a/test/controllers/api/v1/transactions_controller_test.rb +++ b/test/controllers/api/v1/transactions_controller_test.rb @@ -66,6 +66,44 @@ class Api::V1::TransactionsControllerTest < ActionDispatch::IntegrationTest end end + test "should include disabled account transactions in index history" do + disabled_transaction = create_disabled_account_transaction(name: "Closed Account Grocery") + + get api_v1_transactions_url, headers: api_headers(@api_key) + assert_response :success + + response_data = JSON.parse(response.body) + transaction_ids = response_data["transactions"].map { |transaction| transaction["id"] } + assert_includes transaction_ids, disabled_transaction.id + end + + test "should exclude pending deletion account transactions from index history" do + pending_deletion_transaction = create_account_transaction( + status: "pending_deletion", + name: "Pending Delete Account Grocery" + ) + + get api_v1_transactions_url, headers: api_headers(@api_key) + assert_response :success + + response_data = JSON.parse(response.body) + transaction_ids = response_data["transactions"].map { |transaction| transaction["id"] } + assert_not_includes transaction_ids, pending_deletion_transaction.id + end + + test "should filter disabled account transactions by account_id" do + disabled_transaction = create_disabled_account_transaction(name: "Closed Account Filter") + disabled_account = disabled_transaction.entry.account + + get api_v1_transactions_url, + params: { account_id: disabled_account.id }, + headers: api_headers(@api_key) + assert_response :success + + response_data = JSON.parse(response.body) + assert_equal [ disabled_transaction.id ], response_data["transactions"].map { |transaction| transaction["id"] } + end + test "should filter transactions by date range" do start_date = 1.month.ago.to_date end_date = Date.current @@ -83,6 +121,22 @@ class Api::V1::TransactionsControllerTest < ActionDispatch::IntegrationTest end end + test "should filter disabled account transactions by date range" do + disabled_transaction = create_disabled_account_transaction( + name: "Closed Account Date Range", + date: Date.current - 3.days + ) + + get api_v1_transactions_url, + params: { start_date: Date.current - 4.days, end_date: Date.current - 2.days }, + headers: api_headers(@api_key) + assert_response :success + + response_data = JSON.parse(response.body) + transaction_ids = response_data["transactions"].map { |transaction| transaction["id"] } + assert_includes transaction_ids, disabled_transaction.id + end + test "should search transactions" do # Create a transaction with a specific name for testing entry = @account.entries.create!( @@ -103,6 +157,19 @@ class Api::V1::TransactionsControllerTest < ActionDispatch::IntegrationTest assert_not_nil found_transaction, "Should find the coffee transaction" end + test "should search disabled account transactions" do + disabled_transaction = create_disabled_account_transaction(name: "Closed Account Coffee") + + get api_v1_transactions_url, + params: { search: "Closed Account Coffee" }, + headers: api_headers(@api_key) + assert_response :success + + response_data = JSON.parse(response.body) + found_transaction = response_data["transactions"].find { |transaction| transaction["id"] == disabled_transaction.id } + assert_not_nil found_transaction, "Should find disabled account transactions in global history search" + end + test "should paginate transactions" do get api_v1_transactions_url, params: { page: 1, per_page: 5 }, @@ -144,9 +211,33 @@ class Api::V1::TransactionsControllerTest < ActionDispatch::IntegrationTest assert_response :success end - test "should return 404 for non-existent transaction" do + test "should show disabled account transaction" do + disabled_transaction = create_disabled_account_transaction(name: "Closed Account Show") + + get api_v1_transaction_url(disabled_transaction), headers: api_headers(@api_key) + assert_response :success + + response_data = JSON.parse(response.body) + assert_equal disabled_transaction.id, response_data["id"] + assert_equal disabled_transaction.entry.account_id, response_data["account"]["id"] + end + + test "should return 404 for valid missing transaction id" do + get api_v1_transaction_url(SecureRandom.uuid), headers: api_headers(@api_key) + assert_response :not_found + + response_data = JSON.parse(response.body) + assert_equal "not_found", response_data["error"] + assert_equal "Transaction not found", response_data["message"] + end + + test "should return 404 for malformed id" do get api_v1_transaction_url(999999), headers: api_headers(@api_key) assert_response :not_found + + response_data = JSON.parse(response.body) + assert_equal "not_found", response_data["error"] + assert_equal "Transaction not found", response_data["message"] end test "should reject show request without API key" do @@ -689,4 +780,28 @@ end "non-income transactions should have non-positive signed_amount_cents" end end + + def create_disabled_account_transaction(name:, date: Date.current) + create_account_transaction(status: "disabled", name: name, date: date) + end + + def create_account_transaction(status:, name:, date: Date.current) + account = @family.accounts.create!( + name: "#{status.titleize} Checking #{SecureRandom.hex(4)}", + balance: 0, + currency: "USD", + status: status, + accountable: Depository.new + ) + + entry = account.entries.create!( + name: name, + amount: 12.34, + currency: "USD", + date: date, + entryable: Transaction.new + ) + + entry.transaction + end end From 33bc6b59c8d4d5f0fd53a3f7420c76756879b1d8 Mon Sep 17 00:00:00 2001 From: CrossDrain <32982516+CrossDrain@users.noreply.github.com> Date: Mon, 11 May 2026 22:17:49 +0000 Subject: [PATCH 08/31] fix(enable-banking): import transactions missing transaction_id and entry_reference (#1767) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(enable-banking): handle transactions missing transaction_id and entry_reference Some ASPSPs omit both transaction_id and entry_reference from their transaction payloads, which is valid per the PSD2/Berlin Group spec. Previously, every such transaction raised an ArgumentError and was silently dropped during sync. compute_external_id now falls back to a deterministic MD5 fingerprint (prefixed enable_banking_content_) derived from date, amount, currency, direction, counterparty, and remittance info. This fingerprint is stable across re-syncs, so duplicate imports are still correctly prevented. An ArgumentError is only raised for truly empty/unidentifiable payloads. The importer is updated in three places to use compute_external_id consistently: the pending pre-filter (before combining with booked), the C4 stored-pending cleanup, and the new_transactions dedup. This means ID-less pending entries are now also removed when their settled booked counterpart arrives. Tests cover compute_external_id directly (all 5 cases), end-to-end fingerprint import, idempotency, and importer storage/dedup behaviour for ID-less transactions including the pending→booked settlement path. * fix(enable-banking): implement dual-strategy matching for transaction settlement When a stored pending row had only entry_reference (no transaction_id) and the settled BOOK row arrived with a new transaction_id, compute_external_id produced different fingerprints for each side (enable_banking_ vs enable_banking_). The fingerprint-only comparison introduced in the previous commit never matched, leaving the stale pending entry in raw_transactions_payload. Both rows were then imported as separate visible transactions. Restore a book_entry_refs set alongside book_fingerprints in both the pending pre-filter and the C4 stored-pending cleanup. A pending entry is now removed when either its fingerprint or its entry_reference matches a booked counterpart — covering same-ID settlement, content-fingerprint settlement, and the entry_reference cross-match settlement path. Also updates the ArgumentError message in external_id to accurately reflect that transaction_id, entry_reference, and content fingerprint are all accepted identifiers, and aligns build_transaction_content_key to use transaction_date as a fallback (matching compute_external_id). Adds a regression test that stores a pending-only row and asserts it is removed when the booked counterpart arrives with a new transaction_id. --- app/models/enable_banking_entry/processor.rb | 22 ++- app/models/enable_banking_item/importer.rb | 45 +++--- .../transactions/processor_test.rb | 32 ++++ .../enable_banking_entry/processor_test.rb | 76 ++++++++- .../importer_id_less_test.rb | 145 ++++++++++++++++++ 5 files changed, 297 insertions(+), 23 deletions(-) create mode 100644 test/models/enable_banking_item/importer_id_less_test.rb diff --git a/app/models/enable_banking_entry/processor.rb b/app/models/enable_banking_entry/processor.rb index e08d37acc..4f41b21f8 100644 --- a/app/models/enable_banking_entry/processor.rb +++ b/app/models/enable_banking_entry/processor.rb @@ -13,7 +13,25 @@ class EnableBankingEntry::Processor def self.compute_external_id(raw_transaction_data) data = raw_transaction_data.with_indifferent_access id = data[:transaction_id].presence || data[:entry_reference].presence - id ? "enable_banking_#{id}" : nil + return "enable_banking_#{id}" if id + + # Some ASPSPs omit both transaction_id and entry_reference (both are optional + # in PSD2). Generate a deterministic content-based ID so these transactions + # can still be imported idempotently. Uses the same fields as the importer's + # dedup key so the two strategies stay in sync. + date = data[:booking_date].presence || data[:value_date].presence || data[:transaction_date] + amount = data.dig(:transaction_amount, :amount).presence || data[:amount] + currency = data.dig(:transaction_amount, :currency).presence || data[:currency] + direction = data[:credit_debit_indicator] + creditor = data.dig(:creditor, :name).presence || data[:creditor_name] + debtor = data.dig(:debtor, :name).presence || data[:debtor_name] + remittance = data[:remittance_information] + remittance_key = remittance.is_a?(Array) ? remittance.compact.map(&:to_s).sort.join("|") : remittance.to_s + + content = [ date, amount, currency, direction, creditor, debtor, remittance_key ].map(&:to_s).join("\x1F") + return nil if content.gsub("\x1F", "").blank? + + "enable_banking_content_#{Digest::MD5.hexdigest(content)}" end def initialize(enable_banking_transaction, enable_banking_account:, import_adapter: nil) @@ -75,7 +93,7 @@ class EnableBankingEntry::Processor def external_id id = self.class.compute_external_id(data) - raise ArgumentError, "Enable Banking transaction missing required field 'transaction_id'" unless id + raise ArgumentError, "Enable Banking transaction missing required identifier (transaction_id, entry_reference, or identifiable content)" unless id id end diff --git a/app/models/enable_banking_item/importer.rb b/app/models/enable_banking_item/importer.rb index 504b22c9a..708664212 100644 --- a/app/models/enable_banking_item/importer.rb +++ b/app/models/enable_banking_item/importer.rb @@ -251,18 +251,23 @@ class EnableBankingItem::Importer ) end - book_ids = all_transactions - .map { |tx| tx.with_indifferent_access[:transaction_id].presence } + book_fingerprints = all_transactions + .map { |tx| EnableBankingEntry::Processor.compute_external_id(tx) } .compact.to_set + # Also index all booked entry_references so a pending row that lacks + # transaction_id can still be matched when the settled BOOK row adds one + # (fingerprints differ; entry_reference stays the same across settlement). book_entry_refs = all_transactions - .select { |tx| tx.with_indifferent_access[:transaction_id].blank? } .map { |tx| tx.with_indifferent_access[:entry_reference].presence } .compact.to_set pending_transactions.reject! do |tx| - tx = tx.with_indifferent_access - tx[:transaction_id].present? ? book_ids.include?(tx[:transaction_id]) : book_entry_refs.include?(tx[:entry_reference].presence) + tx_ia = tx.with_indifferent_access + fp = EnableBankingEntry::Processor.compute_external_id(tx_ia) + entry_ref = tx_ia[:entry_reference].presence + (fp.present? && book_fingerprints.include?(fp)) || + (entry_ref.present? && book_entry_refs.include?(entry_ref)) end all_transactions = all_transactions + tag_as_pending(pending_transactions) @@ -291,14 +296,17 @@ class EnableBankingItem::Importer if all_transactions.any? # C4: Remove stored PDNG entries that have now settled as BOOK. - # When a BOOK transaction arrives with the same transaction_id as a stored - # PDNG entry, the pending entry is stale — drop it to avoid duplicates. - book_ids = all_transactions + # Two match strategies run in parallel: + # 1. Fingerprint: covers same-ID rows and ID-less rows matched by content. + # 2. Entry-reference cross-match: covers the case where a pending row had + # no transaction_id but the settled BOOK row gained one — fingerprints + # diverge (enable_banking_ vs enable_banking_) but the + # shared entry_reference is a reliable settlement signal. + book_fingerprints = all_transactions .reject { |tx| tx.with_indifferent_access[:_pending] } - .map { |tx| tx.with_indifferent_access[:transaction_id].presence } + .map { |tx| EnableBankingEntry::Processor.compute_external_id(tx) } .compact.to_set - # Fallback: collect entry_references for BOOK rows that have no transaction_id book_entry_refs = all_transactions .reject { |tx| tx.with_indifferent_access[:_pending] } .map { |tx| tx.with_indifferent_access[:entry_reference].presence } @@ -310,21 +318,20 @@ class EnableBankingItem::Importer pending_flag = tx.dig(:extra, :enable_banking, :pending) || tx[:_pending] next false unless pending_flag - tx[:transaction_id].present? ? - book_ids.include?(tx[:transaction_id]) : - book_entry_refs.include?(tx[:entry_reference].presence) + fp = EnableBankingEntry::Processor.compute_external_id(tx) + entry_ref = tx[:entry_reference].presence + (fp.present? && book_fingerprints.include?(fp)) || + (entry_ref.present? && book_entry_refs.include?(entry_ref)) end end existing_ids = existing_transactions.map { |tx| - tx = tx.with_indifferent_access - tx[:transaction_id].presence || tx[:entry_reference].presence + EnableBankingEntry::Processor.compute_external_id(tx) }.compact.to_set new_transactions = all_transactions.select do |tx| - # Use transaction_id if present, otherwise fall back to entry_reference - tx_id = tx[:transaction_id].presence || tx[:entry_reference].presence - tx_id.present? && !existing_ids.include?(tx_id) + ext_id = EnableBankingEntry::Processor.compute_external_id(tx) + ext_id.present? && !existing_ids.include?(ext_id) end if new_transactions.any? || removed_pending @@ -398,7 +405,7 @@ class EnableBankingItem::Importer # omit transaction_id rarely produce such exact duplicates in the same # API response; timestamps or remittance info usually differ. (Issue #954) def build_transaction_content_key(tx) - date = tx[:booking_date].presence || tx[:value_date] + date = tx[:booking_date].presence || tx[:value_date].presence || tx[:transaction_date] amount = tx.dig(:transaction_amount, :amount).presence || tx[:amount] currency = tx.dig(:transaction_amount, :currency).presence || tx[:currency] creditor = tx.dig(:creditor, :name).presence || tx[:creditor_name] diff --git a/test/models/enable_banking_account/transactions/processor_test.rb b/test/models/enable_banking_account/transactions/processor_test.rb index f136ab806..00982bdef 100644 --- a/test/models/enable_banking_account/transactions/processor_test.rb +++ b/test/models/enable_banking_account/transactions/processor_test.rb @@ -157,6 +157,38 @@ class EnableBankingAccount::Transactions::ProcessorTest < ActiveSupport::TestCas assert_equal 1, result[:imported] end + test "imports id-less transaction using content fingerprint" do + tx = { + "booking_date" => Date.current.to_s, + "transaction_amount" => { "amount" => "19.99", "currency" => "EUR" }, + "credit_debit_indicator" => "DBIT", + "creditor" => { "name" => "Spotify" } + } + @enable_banking_account.update!(raw_transactions_payload: [ tx ]) + + assert_difference "@account.entries.count", 1 do + EnableBankingAccount::Transactions::Processor.new(@enable_banking_account).process + end + + expected_id = EnableBankingEntry::Processor.compute_external_id(tx) + assert @account.entries.exists?(external_id: expected_id, source: "enable_banking") + end + + test "id-less transaction does not appear in failed count" do + tx = { + "booking_date" => Date.current.to_s, + "transaction_amount" => { "amount" => "5.00", "currency" => "EUR" }, + "credit_debit_indicator" => "CRDT", + "debtor" => { "name" => "Employer" } + } + @enable_banking_account.update!(raw_transactions_payload: [ tx ]) + + result = EnableBankingAccount::Transactions::Processor.new(@enable_banking_account).process + + assert_equal 0, result[:failed] + assert_equal 1, result[:imported] + end + test "handles empty raw_transactions_payload gracefully" do @enable_banking_account.update!(raw_transactions_payload: nil) diff --git a/test/models/enable_banking_entry/processor_test.rb b/test/models/enable_banking_entry/processor_test.rb index 9a4623ee6..5b54b29be 100644 --- a/test/models/enable_banking_entry/processor_test.rb +++ b/test/models/enable_banking_entry/processor_test.rb @@ -83,17 +83,89 @@ class EnableBankingEntry::ProcessorTest < ActiveSupport::TestCase end end - test "raises ArgumentError when both transaction_id and entry_reference are nil" do + # --- compute_external_id unit tests --- + + test "compute_external_id returns transaction_id-based id when present" do + assert_equal "enable_banking_txn_abc", + EnableBankingEntry::Processor.compute_external_id(transaction_id: "txn_abc", entry_reference: "ref_xyz") + end + + test "compute_external_id falls back to entry_reference when transaction_id is blank" do + assert_equal "enable_banking_ref_xyz", + EnableBankingEntry::Processor.compute_external_id(transaction_id: nil, entry_reference: "ref_xyz") + end + + test "compute_external_id returns content fingerprint when both id fields are absent" do + tx = { + booking_date: "2026-03-15", + transaction_amount: { amount: "42.00", currency: "EUR" }, + credit_debit_indicator: "DBIT", + creditor: { name: "Spar" } + } + result = EnableBankingEntry::Processor.compute_external_id(tx) + assert result.start_with?("enable_banking_content_"), "Expected content fingerprint, got: #{result}" + end + + test "compute_external_id fingerprint is stable across calls" do + tx = { + booking_date: "2026-03-15", + transaction_amount: { amount: "42.00", currency: "EUR" }, + credit_debit_indicator: "DBIT", + creditor: { name: "Spar" } + } + assert_equal EnableBankingEntry::Processor.compute_external_id(tx), + EnableBankingEntry::Processor.compute_external_id(tx) + end + + test "compute_external_id returns nil for transaction with no identifiable content" do + assert_nil EnableBankingEntry::Processor.compute_external_id({}) + assert_nil EnableBankingEntry::Processor.compute_external_id(transaction_id: nil, entry_reference: nil) + end + + # --- ID-less transaction processing --- + + test "imports transaction using content fingerprint when transaction_id and entry_reference are absent" do tx = { transaction_id: nil, entry_reference: nil, booking_date: Date.current.to_s, transaction_amount: { amount: "10.00", currency: "EUR" }, - creditor: { name: "Test" }, + creditor: { name: "Lidl" }, credit_debit_indicator: "DBIT", status: "BOOK" } + assert_difference "@account.entries.count", 1 do + EnableBankingEntry::Processor.new(tx, enable_banking_account: @enable_banking_account).process + end + + expected_id = EnableBankingEntry::Processor.compute_external_id(tx) + assert @account.entries.exists?(external_id: expected_id, source: "enable_banking") + end + + test "does not create duplicate when same id-less transaction is processed twice" do + tx = { + transaction_id: nil, + entry_reference: nil, + booking_date: Date.current.to_s, + transaction_amount: { amount: "10.00", currency: "EUR" }, + creditor: { name: "Lidl" }, + credit_debit_indicator: "DBIT", + status: "BOOK" + } + + assert_difference "@account.entries.count", 1 do + EnableBankingEntry::Processor.new(tx, enable_banking_account: @enable_banking_account).process + end + + assert_no_difference "@account.entries.count" do + EnableBankingEntry::Processor.new(tx, enable_banking_account: @enable_banking_account).process + end + end + + test "raises ArgumentError for transaction with no identifiable content at all" do + tx = { transaction_id: nil, entry_reference: nil } + assert_raises(ArgumentError) do EnableBankingEntry::Processor.new(tx, enable_banking_account: @enable_banking_account).process end diff --git a/test/models/enable_banking_item/importer_id_less_test.rb b/test/models/enable_banking_item/importer_id_less_test.rb new file mode 100644 index 000000000..2ca068fcc --- /dev/null +++ b/test/models/enable_banking_item/importer_id_less_test.rb @@ -0,0 +1,145 @@ +require "test_helper" + +class EnableBankingItem::ImporterIdLessTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @account = accounts(:depository) + + @enable_banking_item = EnableBankingItem.create!( + family: @family, + name: "Test EB", + country_code: "RO", + application_id: "test_app_id", + client_certificate: "test_cert", + session_id: "test_session", + session_expires_at: 1.day.from_now, + sync_start_date: 1.month.ago.to_date + ) + @enable_banking_account = EnableBankingAccount.create!( + enable_banking_item: @enable_banking_item, + name: "Current Account", + uid: "hash_idless_test", + account_id: "uuid-idless-1234-abcd", + currency: "RON" + ) + AccountProvider.create!(account: @account, provider: @enable_banking_account) + + @mock_provider = mock() + @importer = EnableBankingItem::Importer.new(@enable_banking_item, enable_banking_provider: @mock_provider) + end + + def id_less_tx(amount: "50.00", creditor: "Kaufland", date: Date.current.to_s) + { + booking_date: date, + transaction_amount: { amount: amount, currency: "RON" }, + credit_debit_indicator: "DBIT", + creditor: { name: creditor } + } + end + + test "stores id-less transactions in raw_transactions_payload on first sync" do + tx = id_less_tx + + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "BOOK")).returns([ tx ]) + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "PDNG")).returns([]) + @importer.stubs(:include_pending?).returns(false) + @importer.stubs(:determine_sync_start_date).returns(1.month.ago.to_date) + + @importer.send(:fetch_and_store_transactions, @enable_banking_account) + + @enable_banking_account.reload + assert_equal 1, @enable_banking_account.raw_transactions_payload.count + end + + test "does not re-store id-less transaction on second sync" do + tx = id_less_tx + + # First sync + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "BOOK")).returns([ tx ]) + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "PDNG")).returns([]) + @importer.stubs(:include_pending?).returns(false) + @importer.stubs(:determine_sync_start_date).returns(1.month.ago.to_date) + + @importer.send(:fetch_and_store_transactions, @enable_banking_account) + @enable_banking_account.reload + assert_equal 1, @enable_banking_account.raw_transactions_payload.count + + # Second sync with the same transaction + @importer.send(:fetch_and_store_transactions, @enable_banking_account) + @enable_banking_account.reload + assert_equal 1, @enable_banking_account.raw_transactions_payload.count + end + + test "stores multiple distinct id-less transactions separately" do + tx1 = id_less_tx(amount: "50.00", creditor: "Kaufland") + tx2 = id_less_tx(amount: "12.50", creditor: "Starbucks") + + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "BOOK")).returns([ tx1, tx2 ]) + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "PDNG")).returns([]) + @importer.stubs(:include_pending?).returns(false) + @importer.stubs(:determine_sync_start_date).returns(1.month.ago.to_date) + + @importer.send(:fetch_and_store_transactions, @enable_banking_account) + + @enable_banking_account.reload + assert_equal 2, @enable_banking_account.raw_transactions_payload.count + end + + test "removes stored id-less pending entry when its booked counterpart arrives" do + tx = id_less_tx(amount: "30.00", creditor: "Netflix") + pending_tx = tx.merge(_pending: true) + + @enable_banking_account.update!(raw_transactions_payload: [ pending_tx ]) + + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "BOOK")).returns([ tx ]) + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "PDNG")).returns([]) + @importer.stubs(:include_pending?).returns(true) + @importer.stubs(:determine_sync_start_date).returns(1.month.ago.to_date) + + @importer.send(:fetch_and_store_transactions, @enable_banking_account) + + @enable_banking_account.reload + stored = @enable_banking_account.raw_transactions_payload + assert_equal 1, stored.count + assert_nil stored.first["_pending"] + end + + # Regression: pending row has entry_reference only; booked counterpart gains + # transaction_id on settlement. Fingerprints diverge but entry_reference is + # stable — the pending entry must still be removed from stored payload. + test "removes stored pending entry when settled book row gains a transaction_id" do + entry_ref = "REF-SETTLE-123" + + pending_tx = { + "entry_reference" => entry_ref, + "booking_date" => Date.current.to_s, + "transaction_amount" => { "amount" => "15.00", "currency" => "RON" }, + "credit_debit_indicator" => "DBIT", + "creditor" => { "name" => "Bolt" }, + "_pending" => true + } + + booked_tx = { + transaction_id: "TXN-NEW-456", + entry_reference: entry_ref, + booking_date: Date.current.to_s, + transaction_amount: { amount: "15.00", currency: "RON" }, + credit_debit_indicator: "DBIT", + creditor: { name: "Bolt" } + } + + @enable_banking_account.update!(raw_transactions_payload: [ pending_tx ]) + + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "BOOK")).returns([ booked_tx ]) + @importer.stubs(:fetch_paginated_transactions).with(@enable_banking_account, has_entry(transaction_status: "PDNG")).returns([]) + @importer.stubs(:include_pending?).returns(true) + @importer.stubs(:determine_sync_start_date).returns(1.month.ago.to_date) + + @importer.send(:fetch_and_store_transactions, @enable_banking_account) + + @enable_banking_account.reload + stored = @enable_banking_account.raw_transactions_payload + assert_equal 1, stored.count, "Stale pending entry should have been removed" + assert_nil stored.first["_pending"], "Remaining entry should be the booked row" + end +end From be598aecf01d77409673fc0ac0368606dd978731 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 15:22:37 -0700 Subject: [PATCH 09/31] feat(providers): add Kraken exchange sync (#1759) * feat(providers): add Kraken exchange sync Adds family-scoped Kraken API-key connections, read-only balance and trade import, account setup/linking flows, provider status wiring, and focused test coverage. Closes #1758 * test(providers): avoid Kraken sample secret false positive * fix(providers): address Kraken review findings * fix(providers): address Kraken review cleanup * test(imports): stabilize transaction import ordering --- app/controllers/kraken_items_controller.rb | 241 +++++++++++++++ .../settings/providers_controller.rb | 6 + app/helpers/settings_helper.rb | 3 + app/models/account.rb | 27 ++ app/models/family.rb | 2 +- app/models/family/kraken_connectable.rb | 29 ++ app/models/kraken_account.rb | 39 +++ app/models/kraken_account/asset_normalizer.rb | 67 +++++ .../kraken_account/holdings_processor.rb | 84 ++++++ app/models/kraken_account/processor.rb | 122 ++++++++ .../kraken_account/security_resolver.rb | 16 + app/models/kraken_account/usd_converter.rb | 32 ++ app/models/kraken_item.rb | 153 ++++++++++ app/models/kraken_item/importer.rb | 187 ++++++++++++ app/models/kraken_item/provided.rb | 15 + app/models/kraken_item/sync_complete_event.rb | 20 ++ app/models/kraken_item/syncer.rb | 81 +++++ app/models/kraken_item/unlinking.rb | 37 +++ app/models/provider/kraken.rb | 153 ++++++++++ app/models/provider/kraken_adapter.rb | 110 +++++++ app/models/provider/metadata.rb | 1 + app/models/provider_connection_status.rb | 1 + app/views/kraken_items/_kraken_item.html.erb | 115 ++++++++ .../select_existing_account.html.erb | 42 +++ .../kraken_items/setup_accounts.html.erb | 89 ++++++ .../settings/providers/_kraken_panel.html.erb | 141 +++++++++ config/locales/views/kraken_items/en.yml | 85 ++++++ config/locales/views/settings/en.yml | 23 ++ config/routes.rb | 15 + ...090000_create_kraken_items_and_accounts.rb | 53 ++++ db/schema.rb | 49 ++- .../provider_connections_controller_test.rb | 16 + .../kraken_items_controller_test.rb | 278 ++++++++++++++++++ test/fixtures/kraken_accounts.yml | 8 + test/fixtures/kraken_items.yml | 20 ++ .../kraken_account/asset_normalizer_test.rb | 28 ++ .../kraken_account/holdings_processor_test.rb | 100 +++++++ test/models/kraken_account/processor_test.rb | 100 +++++++ test/models/kraken_item/importer_test.rb | 116 ++++++++ test/models/kraken_item_test.rb | 100 +++++++ test/models/provider/kraken_adapter_test.rb | 134 +++++++++ test/models/provider/kraken_test.rb | 163 ++++++++++ .../models/provider_connection_status_test.rb | 11 + test/models/transaction_import_test.rb | 2 +- 44 files changed, 3108 insertions(+), 6 deletions(-) create mode 100644 app/controllers/kraken_items_controller.rb create mode 100644 app/models/family/kraken_connectable.rb create mode 100644 app/models/kraken_account.rb create mode 100644 app/models/kraken_account/asset_normalizer.rb create mode 100644 app/models/kraken_account/holdings_processor.rb create mode 100644 app/models/kraken_account/processor.rb create mode 100644 app/models/kraken_account/security_resolver.rb create mode 100644 app/models/kraken_account/usd_converter.rb create mode 100644 app/models/kraken_item.rb create mode 100644 app/models/kraken_item/importer.rb create mode 100644 app/models/kraken_item/provided.rb create mode 100644 app/models/kraken_item/sync_complete_event.rb create mode 100644 app/models/kraken_item/syncer.rb create mode 100644 app/models/kraken_item/unlinking.rb create mode 100644 app/models/provider/kraken.rb create mode 100644 app/models/provider/kraken_adapter.rb create mode 100644 app/views/kraken_items/_kraken_item.html.erb create mode 100644 app/views/kraken_items/select_existing_account.html.erb create mode 100644 app/views/kraken_items/setup_accounts.html.erb create mode 100644 app/views/settings/providers/_kraken_panel.html.erb create mode 100644 config/locales/views/kraken_items/en.yml create mode 100644 db/migrate/20260511090000_create_kraken_items_and_accounts.rb create mode 100644 test/controllers/kraken_items_controller_test.rb create mode 100644 test/fixtures/kraken_accounts.yml create mode 100644 test/fixtures/kraken_items.yml create mode 100644 test/models/kraken_account/asset_normalizer_test.rb create mode 100644 test/models/kraken_account/holdings_processor_test.rb create mode 100644 test/models/kraken_account/processor_test.rb create mode 100644 test/models/kraken_item/importer_test.rb create mode 100644 test/models/kraken_item_test.rb create mode 100644 test/models/provider/kraken_adapter_test.rb create mode 100644 test/models/provider/kraken_test.rb diff --git a/app/controllers/kraken_items_controller.rb b/app/controllers/kraken_items_controller.rb new file mode 100644 index 000000000..6daba850f --- /dev/null +++ b/app/controllers/kraken_items_controller.rb @@ -0,0 +1,241 @@ +# frozen_string_literal: true + +class KrakenItemsController < ApplicationController + before_action :set_kraken_item, only: %i[update destroy sync setup_accounts complete_account_setup] + before_action :require_admin!, only: %i[create select_accounts link_accounts select_existing_account link_existing_account update destroy sync setup_accounts complete_account_setup] + + def create + @kraken_item = Current.family.kraken_items.build(kraken_item_params) + @kraken_item.name ||= t(".default_name") + + if @kraken_item.save + @kraken_item.set_kraken_institution_defaults! + @kraken_item.sync_later + render_panel_success(t(".success")) + else + render_panel_error(@kraken_item.errors.full_messages.join(", ")) + end + end + + def update + if @kraken_item.update(kraken_item_params) + render_panel_success(t(".success")) + else + render_panel_error(@kraken_item.errors.full_messages.join(", ")) + end + end + + def destroy + @kraken_item.unlink_all!(dry_run: false) + @kraken_item.destroy_later + redirect_to settings_providers_path, notice: t(".success") + end + + def sync + @kraken_item.sync_later unless @kraken_item.syncing? + + respond_to do |format| + format.html { redirect_back_or_to settings_providers_path } + format.json { head :ok } + end + end + + def select_accounts + account_flow = kraken_item_account_flow_context + kraken_item = account_flow[:kraken_item] + + unless kraken_item + redirect_to settings_providers_path, alert: kraken_item_selection_message(account_flow[:credentialed_items]) + return + end + + redirect_to setup_accounts_kraken_item_path(kraken_item, return_to: safe_return_to_path), status: :see_other + end + + def link_accounts + kraken_item = kraken_item_account_flow_context[:kraken_item] + unless kraken_item + redirect_to settings_providers_path, alert: t(".select_connection") + return + end + + redirect_to setup_accounts_kraken_item_path(kraken_item), status: :see_other + end + + def select_existing_account + @account = Current.family.accounts.find(params[:account_id]) + account_flow = kraken_item_account_flow_context + @kraken_item = account_flow[:kraken_item] + + unless manual_crypto_exchange_account?(@account) + redirect_to accounts_path, alert: t("kraken_items.link_existing_account.errors.only_manual") + return + end + + unless @kraken_item + redirect_to settings_providers_path, alert: kraken_item_selection_message(account_flow[:credentialed_items]) + return + end + + @available_kraken_accounts = @kraken_item.kraken_accounts + .left_joins(:account_provider) + .where(account_providers: { id: nil }) + .order(:name) + + render :select_existing_account, layout: false + end + + def link_existing_account + @account = Current.family.accounts.find(params[:account_id]) + kraken_item = kraken_item_account_flow_context[:kraken_item] + + unless manual_crypto_exchange_account?(@account) + return redirect_or_flash_error(t(".errors.only_manual"), account_path(@account)) + end + + unless kraken_item + redirect_to settings_providers_path, alert: t(".select_connection") + return + end + + kraken_account = kraken_item.kraken_accounts.find_by(id: params[:kraken_account_id]) + unless kraken_account + return redirect_or_flash_error(t(".errors.invalid_kraken_account"), account_path(@account)) + end + if kraken_account.account_provider.present? + return redirect_or_flash_error(t(".errors.kraken_account_already_linked"), account_path(@account)) + end + + AccountProvider.create!(account: @account, provider: kraken_account) + kraken_item.sync_later + + redirect_to accounts_path, notice: t(".success") + end + + def setup_accounts + @kraken_accounts = unlinked_accounts_for(@kraken_item) + end + + def complete_account_setup + selected_accounts = Array(params[:selected_accounts]).reject(&:blank?) + created_accounts = [] + + selected_accounts.each do |kraken_account_id| + kraken_account = @kraken_item.kraken_accounts.find_by(id: kraken_account_id) + next unless kraken_account + + kraken_account.with_lock do + next if kraken_account.account_provider.present? + + account = Account.create_from_kraken_account(kraken_account) + provider_link = kraken_account.ensure_account_provider!(account) + provider_link ? created_accounts << account : account.destroy! + end + + KrakenAccount::Processor.new(kraken_account.reload).process + rescue StandardError => e + Rails.logger.error("Failed to setup account for KrakenAccount #{kraken_account_id}: #{e.message}") + end + + @kraken_item.update!(pending_account_setup: unlinked_accounts_for(@kraken_item).exists?) + @kraken_item.sync_later if created_accounts.any? + + notice = if created_accounts.any? + t(".success", count: created_accounts.count) + elsif selected_accounts.empty? + t(".none_selected") + else + t(".no_accounts") + end + + redirect_to accounts_path, notice: notice, status: :see_other + end + + private + + def set_kraken_item + @kraken_item = Current.family.kraken_items.find(params[:id]) + end + + def kraken_item_params + permitted = params.require(:kraken_item).permit(:name, :sync_start_date, :api_key, :api_secret) + if @kraken_item&.persisted? + permitted.delete(:api_key) if permitted[:api_key].blank? + permitted.delete(:api_secret) if permitted[:api_secret].blank? + end + permitted + end + + def render_panel_success(message) + if turbo_frame_request? + flash.now[:notice] = message + @kraken_items = Current.family.kraken_items.active.ordered + stream = turbo_stream.update("kraken-providers-panel", partial: "settings/providers/kraken_panel", locals: { kraken_items: @kraken_items }) + render turbo_stream: [ stream, *flash_notification_stream_items ] + else + redirect_to settings_providers_path, notice: message, status: :see_other + end + end + + def render_panel_error(message) + if turbo_frame_request? + render turbo_stream: turbo_stream.replace( + "kraken-providers-panel", + partial: "settings/providers/kraken_panel", + locals: { error_message: message } + ), status: :unprocessable_entity + else + redirect_to settings_providers_path, alert: message, status: :see_other + end + end + + def kraken_item_account_flow_context + credentialed_items = Current.family.kraken_items.active.credentials_configured.ordered.select(&:credentials_configured?) + item = if params[:kraken_item_id].present? + credentialed_items.find { |candidate| candidate.id.to_s == params[:kraken_item_id].to_s } + elsif credentialed_items.one? + credentialed_items.first + end + + { kraken_item: item, credentialed_items: credentialed_items } + end + + def unlinked_accounts_for(kraken_item) + kraken_item.kraken_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).order(:name) + end + + def kraken_item_selection_message(credentialed_items) + if credentialed_items.count > 1 && params[:kraken_item_id].blank? + t("kraken_items.select_accounts.select_connection") + else + t("kraken_items.select_accounts.no_credentials_configured") + end + end + + def manual_crypto_exchange_account?(account) + account.manual_crypto_exchange? + end + + def redirect_or_flash_error(message, fallback_path) + if turbo_frame_request? + flash.now[:alert] = message + render turbo_stream: Array(flash_notification_stream_items) + else + redirect_to fallback_path, alert: message + end + end + + def safe_return_to_path + return nil if params[:return_to].blank? + + value = params[:return_to].to_s + uri = URI.parse(value) + return nil if uri.scheme.present? + return nil if uri.host.present? + return nil unless value.start_with?("/") + + value + rescue URI::InvalidURIError + nil + end +end diff --git a/app/controllers/settings/providers_controller.rb b/app/controllers/settings/providers_controller.rb index 361097ae1..a3feda136 100644 --- a/app/controllers/settings/providers_controller.rb +++ b/app/controllers/settings/providers_controller.rb @@ -189,6 +189,7 @@ class Settings::ProvidersController < ApplicationController { key: "mercury", title: "Mercury", turbo_id: "mercury", partial: "mercury_panel" }, { key: "coinbase", title: "Coinbase", turbo_id: "coinbase", partial: "coinbase_panel" }, { key: "binance", title: "Binance", turbo_id: "binance", partial: "binance_panel" }, + { key: "kraken", title: "Kraken", turbo_id: "kraken", partial: "kraken_panel" }, { key: "snaptrade", title: "SnapTrade", turbo_id: "snaptrade", partial: "snaptrade_panel", auto_open: "manage" }, { key: "indexa_capital", title: "Indexa Capital", turbo_id: "indexa_capital", partial: "indexa_capital_panel" }, { key: "sophtron", title: "Sophtron", turbo_id: "sophtron", partial: "sophtron_panel" } @@ -205,6 +206,7 @@ class Settings::ProvidersController < ApplicationController "mercury" => "MercuryItem", "coinbase" => "CoinbaseItem", "binance" => "BinanceItem", + "kraken" => "KrakenItem", "snaptrade" => "SnaptradeItem", "indexa_capital" => "IndexaCapitalItem", "sophtron" => "SophtronItem" @@ -226,6 +228,8 @@ class Settings::ProvidersController < ApplicationController @coinbase_items = Current.family.coinbase_items.ordered when "binance" @binance_items = Current.family.binance_items.active.ordered + when "kraken" + @kraken_items = Current.family.kraken_items.active.ordered when "snaptrade" @snaptrade_items = Current.family.snaptrade_items.includes(:snaptrade_accounts).ordered when "indexa_capital" @@ -255,6 +259,7 @@ class Settings::ProvidersController < ApplicationController @snaptrade_items = Current.family.snaptrade_items.ordered @indexa_capital_items = Current.family.indexa_capital_items.ordered.select(:id) @binance_items = Current.family.binance_items.active.ordered + @kraken_items = Current.family.kraken_items.active.ordered @provider_sync_health = compute_provider_sync_health(family_panel_items) @@ -279,6 +284,7 @@ class Settings::ProvidersController < ApplicationController "mercury" => @mercury_items, "coinbase" => @coinbase_items, "binance" => @binance_items, + "kraken" => @kraken_items, "snaptrade" => @snaptrade_items, "indexa_capital" => @indexa_capital_items, "sophtron" => @sophtron_items diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index 3fc4f1af2..84aba4b58 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -92,6 +92,9 @@ module SettingsHelper when "binance" return { status: :off } unless @binance_items&.any? sync_based_summary(key) + when "kraken" + return { status: :off } unless @kraken_items&.any? + sync_based_summary(key) when "snaptrade" configured_item = @snaptrade_items&.find(&:credentials_configured?) return { status: :off } unless configured_item diff --git a/app/models/account.rb b/app/models/account.rb index 8a31a8d5e..0c50e1f70 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -266,6 +266,25 @@ class Account < ApplicationRecord create_and_sync(attributes, skip_initial_sync: true) end + def create_from_kraken_account(kraken_account) + family = kraken_account.kraken_item.family + + attributes = { + family: family, + name: kraken_account.name, + balance: (kraken_account.current_balance || 0).to_d, + cash_balance: 0, + currency: kraken_account.currency.presence || family.currency, + accountable_type: "Crypto", + accountable_attributes: { + subtype: "exchange", + tax_treatment: "taxable" + } + } + + create_and_sync(attributes, skip_initial_sync: true) + end + private @@ -298,6 +317,14 @@ class Account < ApplicationRecord read_attribute(:institution_domain).presence || provider&.institution_domain end + def manual_crypto_exchange? + accountable_type == "Crypto" && + accountable&.subtype == "exchange" && + account_providers.none? && + plaid_account_id.blank? && + simplefin_account_id.blank? + end + def logo_url if institution_domain.present? && Setting.brand_fetch_client_id.present? logo_size = Setting.brand_fetch_logo_size diff --git a/app/models/family.rb b/app/models/family.rb index 3a3547b58..fa7d1222d 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -1,7 +1,7 @@ class Family < ApplicationRecord include Syncable, AutoTransferMatchable, Subscribeable, VectorSearchable include PlaidConnectable, SimplefinConnectable, LunchflowConnectable, EnableBankingConnectable - include CoinbaseConnectable, BinanceConnectable, CoinstatsConnectable, SnaptradeConnectable, MercuryConnectable, SophtronConnectable + include CoinbaseConnectable, BinanceConnectable, KrakenConnectable, CoinstatsConnectable, SnaptradeConnectable, MercuryConnectable, SophtronConnectable include IndexaCapitalConnectable DATE_FORMATS = [ diff --git a/app/models/family/kraken_connectable.rb b/app/models/family/kraken_connectable.rb new file mode 100644 index 000000000..6bc02d235 --- /dev/null +++ b/app/models/family/kraken_connectable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Family::KrakenConnectable + extend ActiveSupport::Concern + + included do + has_many :kraken_items, dependent: :destroy + end + + def can_connect_kraken? + true + end + + def create_kraken_item!(api_key:, api_secret:, item_name: nil) + item = kraken_items.create!( + name: item_name || "Kraken", + api_key: api_key, + api_secret: api_secret + ) + + item.set_kraken_institution_defaults! + item.sync_later + item + end + + def has_kraken_credentials? + kraken_items.active.any?(&:credentials_configured?) + end +end diff --git a/app/models/kraken_account.rb b/app/models/kraken_account.rb new file mode 100644 index 000000000..e968f1e48 --- /dev/null +++ b/app/models/kraken_account.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class KrakenAccount < ApplicationRecord + include Encryptable + + STABLECOINS = %w[USDT USDC DAI PYUSD USDP TUSD USDG].freeze + FIAT_CURRENCIES = %w[USD EUR GBP CAD AUD CHF JPY AED].freeze + + if encryption_ready? + encrypts :raw_payload + encrypts :raw_transactions_payload + end + + belongs_to :kraken_item + + has_one :account_provider, as: :provider, dependent: :destroy + has_one :account, through: :account_provider, source: :account + + validates :name, :account_id, :account_type, :currency, presence: true + + def current_account + account + end + + def ensure_account_provider!(target_account = nil) + acct = target_account || current_account + return nil unless acct + + AccountProvider + .find_or_initialize_by(provider_type: "KrakenAccount", provider_id: id) + .tap do |ap| + ap.account = acct + ap.save! + end + rescue StandardError => e + Rails.logger.warn("KrakenAccount #{id}: failed to link account provider - #{e.class}: #{e.message}") + nil + end +end diff --git a/app/models/kraken_account/asset_normalizer.rb b/app/models/kraken_account/asset_normalizer.rb new file mode 100644 index 000000000..7ad0a9e1c --- /dev/null +++ b/app/models/kraken_account/asset_normalizer.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +class KrakenAccount::AssetNormalizer + SUFFIX_PATTERN = /(\.[A-Z])\z/ + FIAT_PREFIXES = { + "ZUSD" => "USD", + "ZEUR" => "EUR", + "ZGBP" => "GBP", + "ZCAD" => "CAD", + "ZAUD" => "AUD", + "ZCHF" => "CHF", + "ZJPY" => "JPY" + }.freeze + SYMBOL_FALLBACKS = { + "XBT" => "BTC", + "XXBT" => "BTC", + "XETH" => "ETH", + "ZUSD" => "USD" + }.freeze + + def initialize(asset_metadata = {}) + @asset_metadata = asset_metadata || {} + end + + def normalize(raw_asset) + raw = raw_asset.to_s.upcase + suffix = raw[SUFFIX_PATTERN, 1] + raw_base = suffix ? raw.delete_suffix(suffix) : raw + + metadata = metadata_for(raw, raw_base) + base_symbol = metadata_symbol(metadata, raw_base) + normalized_base = normalize_base_symbol(base_symbol) + symbol = suffix.present? ? "#{normalized_base}#{suffix}" : normalized_base + + { + raw_asset: raw, + raw_base: raw_base, + symbol: symbol, + price_symbol: normalized_base, + suffix: suffix, + metadata: metadata + } + end + + private + + attr_reader :asset_metadata + + def metadata_for(raw, raw_base) + asset_metadata[raw] || asset_metadata[raw_base] || asset_metadata.values.find do |metadata| + candidate = metadata_symbol(metadata, raw_base) + [ raw, raw_base ].include?(candidate.to_s.upcase) + end + end + + def metadata_symbol(metadata, fallback) + return fallback unless metadata.is_a?(Hash) + + metadata["altname"].presence || metadata["display_name"].presence || fallback + end + + def normalize_base_symbol(symbol) + value = symbol.to_s.upcase + value = FIAT_PREFIXES[value] if FIAT_PREFIXES.key?(value) + SYMBOL_FALLBACKS[value] || value + end +end diff --git a/app/models/kraken_account/holdings_processor.rb b/app/models/kraken_account/holdings_processor.rb new file mode 100644 index 000000000..d0a589ca4 --- /dev/null +++ b/app/models/kraken_account/holdings_processor.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +class KrakenAccount::HoldingsProcessor + include KrakenAccount::UsdConverter + + def initialize(kraken_account) + @kraken_account = kraken_account + end + + def process + return unless account&.accountable_type == "Crypto" + + raw_assets.each { |asset| process_asset(asset) } + rescue StandardError => e + Rails.logger.error "KrakenAccount::HoldingsProcessor - error: #{e.message}" + nil + end + + private + + attr_reader :kraken_account + + def target_currency + kraken_account.kraken_item&.family&.currency + end + + def account + kraken_account.current_account + end + + def raw_assets + kraken_account.raw_payload&.dig("assets") || [] + end + + def process_asset(asset) + symbol = asset["symbol"] || asset[:symbol] + price_symbol = asset["price_symbol"] || asset[:price_symbol] || symbol + total = (asset["balance"] || asset[:balance] || 0).to_d + price_usd = asset["price_usd"] || asset[:price_usd] + source = asset["source"] || asset[:source] || "spot" + + return if symbol.blank? || total.zero? || price_usd.blank? + + security = resolve_security(symbol) + return unless security + + amount_usd = total * price_usd.to_d + amount, amount_stale, amount_rate_date = convert_from_usd(amount_usd, date: Date.current) + price, price_stale, price_rate_date = convert_from_usd(price_usd.to_d, date: Date.current) + log_stale_rate(symbol, "amount", amount_rate_date) if amount_stale + log_stale_rate(symbol, "price", price_rate_date) if price_stale + + import_adapter.import_holding( + security: security, + quantity: total, + amount: amount, + currency: target_currency, + date: Date.current, + price: price, + cost_basis: nil, + external_id: "kraken_#{symbol}_#{source}_#{Date.current}", + account_provider_id: kraken_account.account_provider&.id, + source: "kraken", + delete_future_holdings: false + ) + rescue StandardError => e + Rails.logger.error "KrakenAccount::HoldingsProcessor - failed asset symbol=#{symbol.presence || "unknown"}: #{e.message}" + end + + def import_adapter + @import_adapter ||= Account::ProviderImportAdapter.new(account) + end + + def resolve_security(symbol) + ticker = symbol.to_s.include?(":") ? symbol.to_s : "CRYPTO:#{symbol}" + KrakenAccount::SecurityResolver.resolve(ticker, symbol) + end + + def log_stale_rate(symbol, field, rate_date) + Rails.logger.warn( + "KrakenAccount::HoldingsProcessor - stale FX rate for #{field} symbol=#{symbol} rate_date=#{rate_date || "unknown"}" + ) + end +end diff --git a/app/models/kraken_account/processor.rb b/app/models/kraken_account/processor.rb new file mode 100644 index 000000000..483ac1f1e --- /dev/null +++ b/app/models/kraken_account/processor.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +class KrakenAccount::Processor + include KrakenAccount::UsdConverter + + attr_reader :kraken_account + + def initialize(kraken_account) + @kraken_account = kraken_account + end + + def process + return unless kraken_account.current_account.present? + + KrakenAccount::HoldingsProcessor.new(kraken_account).process + process_account! + process_trades + end + + private + + def target_currency + kraken_account.kraken_item&.family&.currency + end + + def process_account! + account = kraken_account.current_account + amount, stale, rate_date = convert_from_usd((kraken_account.current_balance || 0).to_d, date: Date.current) + + account.update!( + balance: amount, + cash_balance: 0, + currency: target_currency + ) + + kraken_account.update!(extra: kraken_account.extra.to_h.deep_merge(build_stale_extra(stale, rate_date, Date.current))) + end + + def process_trades + raw_trades.each do |txid, trade| + process_trade(txid, trade) + end + rescue StandardError => e + Rails.logger.error "KrakenAccount::Processor - trade processing failed: #{e.message}" + end + + def raw_trades + kraken_account.raw_transactions_payload&.dig("trades") || {} + end + + def process_trade(txid, trade) + account = kraken_account.current_account + return unless account + + external_id = "kraken_trade_#{txid}" + return if account.entries.exists?(external_id: external_id, source: "kraken") + + type = trade["type"].to_s.downcase + return unless %w[buy sell].include?(type) + + pair = trade["pair"].to_s + base_symbol, quote_symbol = infer_pair_symbols(pair, trade) + return if base_symbol.blank? + + qty = trade["vol"].to_d + return if qty.zero? + + price = trade["price"].to_d + cost = trade["cost"].presence&.to_d + cost ||= (qty * price).round(8) + fee = trade["fee"].presence&.to_d || 0 + currency = quote_symbol.presence || "USD" + date = Time.zone.at(trade["time"].to_d).to_date + security = KrakenAccount::SecurityResolver.resolve("CRYPTO:#{base_symbol}", base_symbol) + return unless security + + entry_amount = type == "buy" ? -cost : cost + trade_qty = type == "buy" ? qty : -qty + label = type == "buy" ? "Buy" : "Sell" + + account.entries.create!( + date: date, + name: "#{label} #{qty.round(8)} #{base_symbol}", + amount: entry_amount, + currency: currency, + external_id: external_id, + source: "kraken", + notes: trade["ordertxid"].presence, + entryable: Trade.new( + security: security, + qty: trade_qty, + price: price, + currency: currency, + fee: fee, + investment_activity_label: label + ) + ) + rescue StandardError => e + Rails.logger.error "KrakenAccount::Processor - failed to process trade #{txid}: #{e.message}" + end + + def infer_pair_symbols(pair, trade) + pair_metadata = kraken_account.raw_payload&.dig("pair_metadata") || {} + metadata = pair_metadata[pair] || pair_metadata.values.find { |candidate| candidate["altname"].to_s == pair } + normalizer = KrakenAccount::AssetNormalizer.new(kraken_account.raw_payload&.dig("asset_metadata") || {}) + + if metadata + base = normalizer.normalize(metadata["base"])[:symbol] + quote = normalizer.normalize(metadata["quote"])[:symbol] + return [ base, quote ] + end + + altname = trade["pair"].to_s + %w[USDT USDC USD EUR GBP BTC ETH].each do |quote| + next unless altname.end_with?(quote) + + return [ normalizer.normalize(altname.delete_suffix(quote))[:symbol], quote ] + end + + [ altname, "USD" ] + end +end diff --git a/app/models/kraken_account/security_resolver.rb b/app/models/kraken_account/security_resolver.rb new file mode 100644 index 000000000..036f9f986 --- /dev/null +++ b/app/models/kraken_account/security_resolver.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class KrakenAccount::SecurityResolver + EXCHANGE_MIC = "XKRA" + + def self.resolve(ticker, symbol) + Security::Resolver.new(ticker).resolve + rescue StandardError => e + Rails.logger.warn "KrakenAccount::SecurityResolver - resolver failed for #{ticker}: #{e.message}" + Security.find_or_initialize_by(ticker: ticker, exchange_operating_mic: EXCHANGE_MIC).tap do |security| + security.name = symbol if security.name.blank? + security.offline = true unless security.offline + security.save! if security.changed? + end + end +end diff --git a/app/models/kraken_account/usd_converter.rb b/app/models/kraken_account/usd_converter.rb new file mode 100644 index 000000000..054c587f0 --- /dev/null +++ b/app/models/kraken_account/usd_converter.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module KrakenAccount::UsdConverter + private + + def convert_from_usd(amount, date: Date.current) + return [ amount.to_d, false, nil ] if target_currency == "USD" + + rate = ExchangeRate.find_or_fetch_rate(from: "USD", to: target_currency, date: date) + return [ amount.to_d, true, nil ] if rate.nil? + + converted = Money.new(amount, "USD").exchange_to(target_currency, custom_rate: rate.rate).amount + stale = rate.date != date + rate_date = stale ? rate.date : nil + + [ converted, stale, rate_date ] + end + + def build_stale_extra(stale, rate_date, target_date) + kraken_meta = if stale + { + "stale_rate" => true, + "rate_date_used" => rate_date&.to_s, + "rate_target_date" => target_date&.to_s + } + else + { "stale_rate" => false } + end + + { "kraken" => kraken_meta } + end +end diff --git a/app/models/kraken_item.rb b/app/models/kraken_item.rb new file mode 100644 index 000000000..2c47d5f8b --- /dev/null +++ b/app/models/kraken_item.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +class KrakenItem < ApplicationRecord + include Syncable, Provided, Unlinking, Encryptable + + enum :status, { good: "good", requires_update: "requires_update" }, default: :good + + if encryption_ready? + encrypts :api_key, deterministic: true + encrypts :api_secret + encrypts :raw_payload + end + + validates :name, presence: true + validates :api_key, presence: true + validates :api_secret, presence: true + + belongs_to :family + has_one_attached :logo, dependent: :purge_later + + has_many :kraken_accounts, dependent: :destroy + has_many :accounts, through: :kraken_accounts + + scope :active, -> { where(scheduled_for_deletion: false) } + scope :syncable, -> { active } + scope :ordered, -> { order(created_at: :desc) } + scope :needs_update, -> { where(status: :requires_update) } + scope :credentials_configured, -> { where.not(api_key: [ nil, "" ]).where.not(api_secret: nil) } + + before_validation :strip_credentials + + def destroy_later + update!(scheduled_for_deletion: true) + DestroyJob.perform_later(self) + end + + def import_latest_kraken_data + provider = kraken_provider + raise StandardError, "Kraken credentials not configured" unless provider + + KrakenItem::Importer.new(self, kraken_provider: provider).import + rescue StandardError => e + Rails.logger.error "KrakenItem #{id} - Failed to import: #{e.full_message}" + raise + end + + def process_accounts + return [] if kraken_accounts.empty? + + results = [] + kraken_accounts.joins(:account).merge(Account.visible).each do |kraken_account| + begin + result = KrakenAccount::Processor.new(kraken_account).process + results << { kraken_account_id: kraken_account.id, success: true, result: result } + rescue StandardError => e + Rails.logger.error "KrakenItem #{id} - Failed to process account #{kraken_account.id}: #{e.full_message}" + results << { kraken_account_id: kraken_account.id, success: false, error: e.message } + end + end + + results + end + + def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) + return [] if accounts.empty? + + accounts.visible.map do |account| + account.sync_later( + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: window_end_date + ) + { account_id: account.id, success: true } + rescue StandardError => e + Rails.logger.error "KrakenItem #{id} - Failed to schedule sync for account #{account.id}: #{e.full_message}" + { account_id: account.id, success: false, error: e.message } + end + end + + def upsert_kraken_snapshot!(payload) + update!(raw_payload: payload) + end + + def has_completed_initial_setup? + accounts.any? + end + + def sync_status_summary + total = total_accounts_count + linked = linked_accounts_count + unlinked = unlinked_accounts_count + + if total.zero? + I18n.t("kraken_items.kraken_item.sync_status.no_accounts") + elsif unlinked.zero? + I18n.t("kraken_items.kraken_item.sync_status.all_synced", count: linked) + else + I18n.t("kraken_items.kraken_item.sync_status.partial_sync", linked_count: linked, unlinked_count: unlinked) + end + end + + def linked_accounts_count + kraken_accounts.joins(:account_provider).count + end + + def unlinked_accounts_count + kraken_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).count + end + + def total_accounts_count + kraken_accounts.count + end + + def stale_rate_accounts + kraken_accounts + .joins(:account) + .where(accounts: { status: "active" }) + .where("kraken_accounts.extra -> 'kraken' ->> 'stale_rate' = 'true'") + end + + def institution_display_name + institution_name.presence || institution_domain.presence || name + end + + def credentials_configured? + api_key.to_s.strip.present? && api_secret.to_s.strip.present? + end + + def next_nonce! + with_lock do + candidate = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond) + candidate = last_nonce.to_i + 1 if candidate <= last_nonce.to_i + update!(last_nonce: candidate) + candidate.to_s + end + end + + def set_kraken_institution_defaults! + update!( + institution_name: "Kraken", + institution_domain: "kraken.com", + institution_url: "https://www.kraken.com", + institution_color: "#5841D8" + ) + end + + private + + def strip_credentials + self.api_key = api_key.to_s.strip if api_key_changed? && !api_key.nil? + self.api_secret = api_secret.to_s.strip if api_secret_changed? && !api_secret.nil? + end +end diff --git a/app/models/kraken_item/importer.rb b/app/models/kraken_item/importer.rb new file mode 100644 index 000000000..38944fe0f --- /dev/null +++ b/app/models/kraken_item/importer.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +class KrakenItem::Importer + MAX_TRADE_PAGES = 200 + TRADE_PAGE_SIZE = 50 + + attr_reader :kraken_item, :kraken_provider + + def initialize(kraken_item, kraken_provider:) + @kraken_item = kraken_item + @kraken_provider = kraken_provider + end + + def import + api_key_info = kraken_provider.get_api_key_info + + asset_metadata = kraken_provider.get_asset_info || {} + pair_metadata = kraken_provider.get_asset_pairs || {} + balances = kraken_provider.get_extended_balance || {} + assets = parse_assets(balances, asset_metadata) + trades = fetch_trades + + total_usd = assets.sum { |asset| asset[:amount_usd].to_d }.round(2) + kraken_account = upsert_kraken_account( + assets: assets, + balances: balances, + trades: trades, + asset_metadata: asset_metadata, + pair_metadata: pair_metadata, + api_key_info: api_key_info, + total_usd: total_usd + ) + + kraken_item.upsert_kraken_snapshot!({ + "api_key_info" => api_key_info, + "balances" => balances, + "asset_metadata" => asset_metadata, + "pair_metadata" => pair_metadata, + "imported_at" => Time.current.iso8601 + }) + + { success: true, account_id: kraken_account.id, assets_imported: assets.size, trades_imported: trades.size, total_usd: total_usd } + rescue Provider::Kraken::PermissionError => e + kraken_item.update!(status: :requires_update) + raise e + end + + private + def parse_assets(balances, asset_metadata) + normalizer = KrakenAccount::AssetNormalizer.new(asset_metadata) + + balances.filter_map do |raw_asset, balance_data| + parsed = normalizer.normalize(raw_asset) + balance = balance_data.fetch("balance", "0").to_d + credit = balance_data.fetch("credit", "0").to_d + credit_used = balance_data.fetch("credit_used", "0").to_d + hold_trade = balance_data.fetch("hold_trade", "0").to_d + available = balance + credit - credit_used - hold_trade + + next if balance.zero? && hold_trade.zero? + + price_usd, price_status = price_for(parsed[:price_symbol]) + amount_usd = price_usd ? (balance * price_usd).round(2) : 0.to_d + + parsed.merge( + balance: balance.to_s("F"), + available: available.to_s("F"), + hold_trade: hold_trade.to_s("F"), + price_usd: price_usd&.to_s("F"), + amount_usd: amount_usd.to_s("F"), + price_status: price_status, + source: "spot" + ) + end + end + + def price_for(symbol) + return [ 1.to_d, "exact" ] if symbol == "USD" || KrakenAccount::STABLECOINS.include?(symbol) + + if KrakenAccount::FIAT_CURRENCIES.include?(symbol) + rate = ExchangeRate.find_or_fetch_rate(from: symbol, to: "USD", date: Date.current) + return [ rate.rate.to_d, rate.date == Date.current ? "exact" : "stale" ] if rate + + return [ nil, "missing" ] + end + + ticker_price = ticker_price_for(symbol) + return [ ticker_price, "exact" ] if ticker_price + + [ nil, "missing" ] + rescue StandardError => e + Rails.logger.warn "KrakenItem::Importer - could not price #{symbol}: #{e.message}" + [ nil, "missing" ] + end + + def ticker_price_for(symbol) + pair_candidates_for(symbol).each do |pair| + response = kraken_provider.get_ticker(pair) + ticker_payload = response&.values&.first + price = ticker_payload&.dig("c", 0) + return price.to_d if price.present? + rescue Provider::Kraken::ApiError + next + end + + nil + end + + def pair_candidates_for(symbol) + kraken_symbol = symbol == "BTC" ? "XBT" : symbol + [ + "#{kraken_symbol}USD", + "#{symbol}USD", + "X#{kraken_symbol}ZUSD", + "#{kraken_symbol}USDT", + "#{symbol}USDT" + ].uniq + end + + def fetch_trades + start_time = kraken_item.sync_start_date&.to_i + offset = 0 + all_trades = {} + + MAX_TRADE_PAGES.times do + result = kraken_provider.get_trades_history(start: start_time, offset: offset) + trades = result.to_h.fetch("trades", {}) + duplicate_trade_ids = all_trades.keys & trades.keys + if duplicate_trade_ids.any? + Rails.logger.warn("KrakenItem::Importer - #{duplicate_trade_ids.size} duplicate trade ids from Kraken page ignored") + end + all_trades.merge!(trades.except(*duplicate_trade_ids)) + + count = result.to_h["count"].to_i + break if trades.size < TRADE_PAGE_SIZE + + offset += trades.size + break if count.positive? && offset >= count + end + + all_trades + end + + def upsert_kraken_account(assets:, balances:, trades:, asset_metadata:, pair_metadata:, api_key_info:, total_usd:) + kraken_item.kraken_accounts.find_or_initialize_by(account_id: "combined").tap do |account| + account.assign_attributes( + name: kraken_item.institution_name.presence || "Kraken", + account_type: "combined", + currency: "USD", + current_balance: total_usd, + institution_metadata: institution_metadata(assets), + raw_payload: { + "balances" => balances, + "assets" => assets.map(&:stringify_keys), + "asset_metadata" => asset_metadata, + "pair_metadata" => pair_metadata, + "api_key_info" => api_key_info, + "fetched_at" => Time.current.iso8601 + }, + raw_transactions_payload: { + "trades" => trades, + "fetched_at" => Time.current.iso8601 + }, + extra: account.extra.to_h.deep_merge(price_metadata(assets)) + ) + account.save! + end + end + + def institution_metadata(assets) + { + "name" => "Kraken", + "domain" => "kraken.com", + "url" => "https://www.kraken.com", + "color" => "#5841D8", + "asset_count" => assets.size, + "assets" => assets.map { |asset| asset[:symbol] } + } + end + + def price_metadata(assets) + missing = assets.select { |asset| asset[:price_status] == "missing" }.map { |asset| asset[:symbol] } + stale = assets.select { |asset| asset[:price_status] == "stale" }.map { |asset| asset[:symbol] } + + { "kraken" => { "missing_prices" => missing, "stale_prices" => stale } } + end +end diff --git a/app/models/kraken_item/provided.rb b/app/models/kraken_item/provided.rb new file mode 100644 index 000000000..830a6fd11 --- /dev/null +++ b/app/models/kraken_item/provided.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module KrakenItem::Provided + extend ActiveSupport::Concern + + def kraken_provider + return nil unless credentials_configured? + + Provider::Kraken.new( + api_key: api_key.to_s.strip, + api_secret: api_secret.to_s.strip, + nonce_generator: -> { next_nonce! } + ) + end +end diff --git a/app/models/kraken_item/sync_complete_event.rb b/app/models/kraken_item/sync_complete_event.rb new file mode 100644 index 000000000..a2b07d69e --- /dev/null +++ b/app/models/kraken_item/sync_complete_event.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class KrakenItem::SyncCompleteEvent + def initialize(kraken_item) + raise ArgumentError, "kraken_item is required" unless kraken_item.respond_to?(:family) && kraken_item.respond_to?(:id) + + @kraken_item = kraken_item + end + + def broadcast + Turbo::StreamsChannel.broadcast_replace_to( + @kraken_item.family, + target: ActionView::RecordIdentifier.dom_id(@kraken_item), + partial: "kraken_items/kraken_item", + locals: { kraken_item: @kraken_item } + ) + rescue StandardError => e + Rails.logger.warn("KrakenItem::SyncCompleteEvent failed for #{@kraken_item.id}: #{e.class}") + end +end diff --git a/app/models/kraken_item/syncer.rb b/app/models/kraken_item/syncer.rb new file mode 100644 index 000000000..80b066d36 --- /dev/null +++ b/app/models/kraken_item/syncer.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +class KrakenItem::Syncer + include SyncStats::Collector + + attr_reader :kraken_item + + def initialize(kraken_item) + @kraken_item = kraken_item + end + + def perform_sync(sync) + sync.update!(status_text: I18n.t("kraken_item.syncer.checking_credentials")) if sync.respond_to?(:status_text) + unless kraken_item.credentials_configured? + kraken_item.update!(status: :requires_update) + mark_failed(sync, I18n.t("kraken_item.syncer.credentials_invalid")) + return + end + + sync.update!(status_text: I18n.t("kraken_item.syncer.importing_accounts")) if sync.respond_to?(:status_text) + kraken_item.import_latest_kraken_data + kraken_item.update!(status: :good) if kraken_item.requires_update? + + sync.update!(status_text: I18n.t("kraken_item.syncer.checking_configuration")) if sync.respond_to?(:status_text) + collect_setup_stats(sync, provider_accounts: kraken_item.kraken_accounts.to_a) + + unlinked = kraken_item.kraken_accounts.left_joins(:account_provider).where(account_providers: { id: nil }) + linked = kraken_item.kraken_accounts.joins(:account_provider).joins(:account).merge(Account.visible) + + if unlinked.any? + kraken_item.update!(pending_account_setup: true) + sync.update!(status_text: I18n.t("kraken_item.syncer.accounts_need_setup", count: unlinked.count)) if sync.respond_to?(:status_text) + else + kraken_item.update!(pending_account_setup: false) + end + + return unless linked.any? + + sync.update!(status_text: I18n.t("kraken_item.syncer.processing_accounts")) if sync.respond_to?(:status_text) + kraken_item.process_accounts + + sync.update!(status_text: I18n.t("kraken_item.syncer.calculating_balances")) if sync.respond_to?(:status_text) + kraken_item.schedule_account_syncs( + parent_sync: sync, + window_start_date: sync.window_start_date, + window_end_date: sync.window_end_date + ) + + account_ids = linked.map { |kraken_account| kraken_account.current_account&.id }.compact + if account_ids.any? + collect_transaction_stats(sync, account_ids: account_ids, source: "kraken") + collect_trades_stats(sync, account_ids: account_ids, source: "kraken") + end + rescue Provider::Kraken::AuthenticationError, Provider::Kraken::PermissionError, Provider::Kraken::OTPRequiredError => e + kraken_item.update!(status: :requires_update) + mark_failed(sync, e.message) + raise + rescue StandardError => e + Rails.logger.error "KrakenItem::Syncer - unexpected error during sync: #{e.message}\n#{e.backtrace&.first(5)&.join("\n")}" + mark_failed(sync, e.message) + raise + end + + def perform_post_sync + end + + private + + def mark_failed(sync, error_message) + sync.start! if sync.respond_to?(:may_start?) && sync.may_start? + + if sync.respond_to?(:may_fail?) && sync.may_fail? + sync.fail! + elsif sync.respond_to?(:status) + sync.update!(status: :failed) + end + + sync.update!(error: error_message) if sync.respond_to?(:error) + sync.update!(status_text: error_message) if sync.respond_to?(:status_text) + end +end diff --git a/app/models/kraken_item/unlinking.rb b/app/models/kraken_item/unlinking.rb new file mode 100644 index 000000000..d3ef80bb7 --- /dev/null +++ b/app/models/kraken_item/unlinking.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module KrakenItem::Unlinking + extend ActiveSupport::Concern + + def unlink_all!(dry_run: false) + results = [] + links_by_provider_id = AccountProvider + .where(provider_type: KrakenAccount.name, provider_id: kraken_accounts.select(:id)) + .group_by { |link| link.provider_id.to_s } + + kraken_accounts.find_each do |provider_account| + links = links_by_provider_id[provider_account.id.to_s] || [] + link_ids = links.map(&:id) + result = { + provider_account_id: provider_account.id, + name: provider_account.name, + provider_link_ids: link_ids + } + results << result + + next if dry_run + + begin + ActiveRecord::Base.transaction do + Holding.where(account_provider_id: link_ids).update_all(account_provider_id: nil) if link_ids.any? + links.each(&:destroy!) + end + rescue StandardError => e + Rails.logger.warn("KrakenItem Unlinker: failed to unlink ##{provider_account.id}: #{e.class} - #{e.message}") + result[:error] = e.message + end + end + + results + end +end diff --git a/app/models/provider/kraken.rb b/app/models/provider/kraken.rb new file mode 100644 index 000000000..38a2db6ca --- /dev/null +++ b/app/models/provider/kraken.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +class Provider::Kraken + include HTTParty + extend SslConfigurable + + class Error < StandardError; end + class AuthenticationError < Error; end + class PermissionError < Error; end + class RateLimitError < Error; end + class NonceError < Error; end + class OTPRequiredError < Error; end + class ApiError < Error; end + + BASE_URL = "https://api.kraken.com" + PRIVATE_PREFIX = "/0/private" + PUBLIC_PREFIX = "/0/public" + + base_uri BASE_URL + default_options.merge!({ timeout: 30 }.merge(httparty_ssl_options)) + + attr_reader :api_key, :api_secret + + def initialize(api_key:, api_secret:, nonce_generator: nil) + @api_key = api_key # pipelock:ignore user-supplied Kraken credential kept in memory for signed requests + @api_secret = api_secret # pipelock:ignore user-supplied Kraken credential kept in memory for signed requests + @nonce_generator = nonce_generator || -> { Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond).to_s } + end + + def get_api_key_info + private_post("GetApiKeyInfo") + end + + def get_extended_balance + private_post("BalanceEx") + end + + def get_trades_history(start: nil, offset: nil) + params = {} + params["start"] = start.to_i.to_s if start.present? + params["ofs"] = offset.to_i.to_s if offset.present? + + private_post("TradesHistory", params) + end + + def get_asset_info(asset: nil) + params = {} + params["asset"] = asset if asset.present? + public_get("Assets", params) + end + + def get_asset_pairs(pair: nil) + params = {} + params["pair"] = pair if pair.present? + public_get("AssetPairs", params) + end + + def get_ticker(pair) + public_get("Ticker", "pair" => pair) + end + + def get_ohlc(pair, interval: 1440, since: nil) + params = { "pair" => pair, "interval" => interval.to_s } + params["since"] = since.to_i.to_s if since.present? + public_get("OHLC", params) + end + + private + + attr_reader :nonce_generator + + def public_get(method, params = {}) + response = self.class.get("#{PUBLIC_PREFIX}/#{method}", query: params) + handle_response(response) + end + + def private_post(method, params = {}) + path = "#{PRIVATE_PREFIX}/#{method}" + request_params = { "nonce" => nonce_generator.call.to_s }.merge(stringify_params(params)) + body = URI.encode_www_form(request_params) + + response = self.class.post( + path, + body: body, + headers: auth_headers(path, request_params).merge("Content-Type" => "application/x-www-form-urlencoded") + ) + + handle_response(response) + end + + def stringify_params(params) + params.each_with_object({}) { |(key, value), hash| hash[key.to_s] = value.to_s } + end + + def auth_headers(path, params) + { + "API-Key" => api_key, + "API-Sign" => sign(path, params) + } + end + + def sign(path, params) + encoded_payload = URI.encode_www_form(params) + nonce = params.fetch("nonce").to_s + digest = OpenSSL::Digest::SHA256.digest(nonce + encoded_payload) + hmac = OpenSSL::HMAC.digest("sha512", Base64.decode64(api_secret), path + digest) + Base64.strict_encode64(hmac) + end + + def handle_response(response) + parsed = response.parsed_response + + unless response.code.between?(200, 299) + raise ApiError, "Kraken API request failed: #{response.code}" + end + + unless parsed.is_a?(Hash) + raise ApiError, "Malformed Kraken API response" + end + + unless parsed.key?("error") + raise ApiError, "Malformed Kraken API response: missing error" + end + + errors = Array(parsed["error"]).reject(&:blank?) + raise classified_error(errors) if errors.any? + + unless parsed.key?("result") + raise ApiError, "Malformed Kraken API response: missing result" + end + + parsed["result"] + end + + def classified_error(errors) + message = errors.join(", ") + + case message + when /Invalid key|Invalid signature|Temporary lockout/i + AuthenticationError.new(message) + when /Invalid nonce/i + NonceError.new(message) + when /Permission denied|Invalid permissions/i + PermissionError.new(message) + when /Rate limit exceeded|Too many requests|limit exceeded|Throttled/i + RateLimitError.new(message) + when /otp|2fa|two.factor/i + OTPRequiredError.new(message) + else + ApiError.new(message) + end + end +end diff --git a/app/models/provider/kraken_adapter.rb b/app/models/provider/kraken_adapter.rb new file mode 100644 index 000000000..c932a9efb --- /dev/null +++ b/app/models/provider/kraken_adapter.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +class Provider::KrakenAdapter < Provider::Base + include Provider::Syncable + include Provider::InstitutionMetadata + + Provider::Factory.register("KrakenAccount", self) + + def self.supported_account_types + %w[Crypto] + end + + def self.connection_configs(family:) + return [] unless family.can_connect_kraken? + + kraken_items = family.kraken_items.active.credentials_configured.ordered.select(&:credentials_configured?) + return [ connection_config_for(nil) ] if kraken_items.empty? + + kraken_items.map { |kraken_item| connection_config_for(kraken_item) } + end + + def self.build_provider(family: nil, kraken_item_id: nil) + return nil unless family.present? + + kraken_item = resolve_kraken_item(family, kraken_item_id) + return nil unless kraken_item&.credentials_configured? + + kraken_item.kraken_provider + end + + def provider_name + "kraken" + end + + def sync_path + return unless item + + Rails.application.routes.url_helpers.sync_kraken_item_path(item) + end + + def item + provider_account.kraken_item + end + + def can_delete_holdings? + false + end + + def institution_domain + institution_metadata_value("domain") + end + + def institution_name + institution_metadata_value("name") + end + + def institution_url + institution_metadata_value("url") + end + + def institution_color + institution_metadata_value("color") + end + + def self.connection_config_for(kraken_item) + path_params = ->(extra = {}) do + kraken_item.present? ? extra.merge(kraken_item_id: kraken_item.id) : extra + end + + { + key: kraken_item.present? ? "kraken_#{kraken_item.id}" : "kraken", + name: kraken_item.present? ? I18n.t("kraken_items.provider_connection.name", name: kraken_item.name) : I18n.t("kraken_items.provider_connection.default_name"), + description: kraken_item.present? ? I18n.t("kraken_items.provider_connection.description", name: kraken_item.name) : I18n.t("kraken_items.provider_connection.default_description"), + can_connect: true, + new_account_path: ->(accountable_type, return_to) { + Rails.application.routes.url_helpers.select_accounts_kraken_items_path( + path_params.call(accountable_type: accountable_type, return_to: return_to) + ) + }, + existing_account_path: ->(account_id) { + Rails.application.routes.url_helpers.select_existing_account_kraken_items_path( + path_params.call(account_id: account_id) + ) + } + } + end + private_class_method :connection_config_for + + def self.resolve_kraken_item(family, kraken_item_id) + if kraken_item_id.present? + item = family.kraken_items.active.credentials_configured.find_by(id: kraken_item_id) + return item if item&.credentials_configured? + + return nil + end + + credentialed_items = family.kraken_items.active.credentials_configured.ordered.select(&:credentials_configured?) + return credentialed_items.first if credentialed_items.one? + + nil + end + private_class_method :resolve_kraken_item + + private + + def institution_metadata_value(key) + metadata = provider_account.institution_metadata || {} + metadata[key] || item&.public_send("institution_#{key}") + end +end diff --git a/app/models/provider/metadata.rb b/app/models/provider/metadata.rb index 5cdd8e72c..0850c524c 100644 --- a/app/models/provider/metadata.rb +++ b/app/models/provider/metadata.rb @@ -8,6 +8,7 @@ class Provider mercury: { region: "US", kind: "Bank", maturity: :beta, logo_text: "ME", logo_bg: "bg-cyan-600" }, coinbase: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "CB", logo_bg: "bg-blue-500" }, binance: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "BI", logo_bg: "bg-yellow-600" }, + kraken: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "KR", logo_bg: "bg-violet-600" }, snaptrade: { region: "US / CA", kind: "Investment", maturity: :beta, logo_text: "ST", logo_bg: "bg-green-600" }, indexa_capital: { region: "ES", kind: "Investment", maturity: :alpha, logo_text: "IC", logo_bg: "bg-red-600" }, sophtron: { region: "US", kind: "Bank", maturity: :alpha, logo_text: "SO", logo_bg: "bg-teal-600" }, diff --git a/app/models/provider_connection_status.rb b/app/models/provider_connection_status.rb index e1a4b4100..6ed6f4a34 100644 --- a/app/models/provider_connection_status.rb +++ b/app/models/provider_connection_status.rb @@ -8,6 +8,7 @@ class ProviderConnectionStatus { key: "enable_banking", type: "EnableBankingItem", association: :enable_banking_items, accounts: :enable_banking_accounts }, { key: "coinbase", type: "CoinbaseItem", association: :coinbase_items, accounts: :coinbase_accounts }, { key: "binance", type: "BinanceItem", association: :binance_items, accounts: :binance_accounts }, + { key: "kraken", type: "KrakenItem", association: :kraken_items, accounts: :kraken_accounts }, { key: "coinstats", type: "CoinstatsItem", association: :coinstats_items, accounts: :coinstats_accounts }, { key: "snaptrade", type: "SnaptradeItem", association: :snaptrade_items, accounts: :snaptrade_accounts, linked_accounts: :linked_accounts }, { key: "mercury", type: "MercuryItem", association: :mercury_items, accounts: :mercury_accounts }, diff --git a/app/views/kraken_items/_kraken_item.html.erb b/app/views/kraken_items/_kraken_item.html.erb new file mode 100644 index 000000000..590d900fa --- /dev/null +++ b/app/views/kraken_items/_kraken_item.html.erb @@ -0,0 +1,115 @@ +<%# locals: (kraken_item:, unlinked_count: kraken_item.unlinked_accounts_count) %> + +<%= tag.div id: dom_id(kraken_item) do %> +
+ + <%= icon "chevron-right", class: "group-open:transform group-open:rotate-90" %> + +
+ <%= icon "waves", size: "sm", class: "text-primary" %> +
+ +
+
+ <%= tag.p kraken_item.institution_display_name, class: "font-medium text-primary" %> + <% if kraken_item.scheduled_for_deletion? %> +

<%= t(".deletion_in_progress") %>

+ <% end %> +
+

<%= t(".provider_name") %>

+ + <% if kraken_item.syncing? %> +
+ <%= icon "loader", size: "sm", class: "animate-spin" %> + <%= tag.span t(".syncing") %> +
+ <% elsif kraken_item.requires_update? %> +
+ <%= icon "alert-triangle", size: "sm", color: "warning" %> + <%= tag.span t(".reconnect") %> +
+ <% else %> +

+ <% if kraken_item.last_synced_at %> + <% if kraken_item.sync_status_summary %> + <%= t(".status_with_summary", timestamp: time_ago_in_words(kraken_item.last_synced_at), summary: kraken_item.sync_status_summary) %> + <% else %> + <%= t(".status", timestamp: time_ago_in_words(kraken_item.last_synced_at)) %> + <% end %> + <% else %> + <%= t(".status_never") %> + <% end %> +

+ <% end %> +
+ + + <% if Current.user&.admin? %> +
+ <%= icon( + "refresh-cw", + as_button: true, + href: sync_kraken_item_path(kraken_item), + disabled: kraken_item.syncing? + ) %> + + <%= render DS::Menu.new do |menu| %> + <% if unlinked_count.to_i > 0 %> + <% menu.with_item( + variant: "link", + text: t(".import_accounts_menu"), + icon: "plus", + href: setup_accounts_kraken_item_path(kraken_item), + frame: :modal + ) %> + <% end %> + <% menu.with_item( + variant: "button", + text: t(".delete"), + icon: "trash-2", + href: kraken_item_path(kraken_item), + method: :delete, + confirm: CustomConfirm.for_resource_deletion(kraken_item.institution_display_name, high_severity: true) + ) %> + <% end %> +
+ <% end %> + + <% unless kraken_item.scheduled_for_deletion? %> +
+ <% if kraken_item.accounts.any? %> + <%= render "accounts/index/account_groups", accounts: kraken_item.accounts %> + <% kraken_item.stale_rate_accounts.each do |kraken_account| %> +
+ ~ + <%= icon "triangle-alert", size: "sm" %> + <%= t(".stale_rate_warning", date: kraken_account.extra.dig("kraken", "rate_target_date")) %> +
+ <% end %> + <% end %> + + <% stats = kraken_item.syncs.ordered.first&.sync_stats || {} %> + <%= render ProviderSyncSummary.new(stats: stats, provider_item: kraken_item) %> + + <% if unlinked_count.to_i > 0 && kraken_item.accounts.empty? %> +
+

<%= t(".setup_needed") %>

+

<%= t(".setup_description") %>

+ <%= render DS::Link.new( + text: t(".setup_action"), + icon: "plus", + variant: "primary", + href: setup_accounts_kraken_item_path(kraken_item), + frame: :modal + ) %> +
+ <% elsif kraken_item.accounts.empty? && kraken_item.kraken_accounts.none? %> +
+

<%= t(".no_accounts_title") %>

+

<%= t(".no_accounts_message") %>

+
+ <% end %> +
+ <% end %> +
+<% end %> diff --git a/app/views/kraken_items/select_existing_account.html.erb b/app/views/kraken_items/select_existing_account.html.erb new file mode 100644 index 000000000..d7e9e5807 --- /dev/null +++ b/app/views/kraken_items/select_existing_account.html.erb @@ -0,0 +1,42 @@ +<%# Modal: Link an existing manual crypto exchange account to a Kraken account %> +<%= turbo_frame_tag "modal" do %> + <%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title")) %> + + <% dialog.with_body do %> + <% if @available_kraken_accounts.blank? %> +
+

<%= t(".no_accounts_found") %>

+
    +
  • <%= t(".wait_for_sync") %>
    • +
  • <%= t(".check_provider_health") %>
    • +
+
+ <% else %> + <%= form_with url: link_existing_account_kraken_items_path, method: :post, class: "space-y-4" do %> + <%= hidden_field_tag :account_id, @account.id %> + <%= hidden_field_tag :kraken_item_id, @kraken_item.id %> + +
+ <% @available_kraken_accounts.each do |kraken_account| %> + + <% end %> +
+ +
+ <%= render DS::Button.new(text: t(".link"), variant: :primary, icon: "link-2", type: :submit) %> + <%= render DS::Link.new(text: t(".cancel"), variant: :secondary, href: accounts_path, data: { turbo_frame: "_top" }) %> +
+ <% end %> + <% end %> + <% end %> + <% end %> +<% end %> diff --git a/app/views/kraken_items/setup_accounts.html.erb b/app/views/kraken_items/setup_accounts.html.erb new file mode 100644 index 000000000..5b757f492 --- /dev/null +++ b/app/views/kraken_items/setup_accounts.html.erb @@ -0,0 +1,89 @@ +<% content_for :title, t(".title") %> + +<%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title")) do %> +
+ <%= icon "waves", class: "text-primary" %> + <%= t(".subtitle") %> +
+ <% end %> + + <% dialog.with_body do %> + <%= form_with url: complete_account_setup_kraken_item_path(@kraken_item), + method: :post, + local: true, + id: "kraken-setup-form", + data: { + controller: "loading-button", + action: "submit->loading-button#showLoading", + loading_button_loading_text_value: t(".creating"), + turbo_frame: "_top" + }, + class: "space-y-6" do |form| %> +
+
+
+ <%= icon "info", size: "sm", class: "text-primary mt-0.5 flex-shrink-0" %> +

<%= t(".instructions") %>

+
+
+ + <% if @kraken_accounts.empty? %> +
+

<%= t(".no_accounts") %>

+
+ <% else %> +
+
+ + <%= t(".accounts_count", count: @kraken_accounts.count) %> + + +
+ +
+ <% @kraken_accounts.each do |kraken_account| %> + + <% end %> +
+
+ <% end %> +
+ +
+ <%= render DS::Button.new( + text: t(".import_selected"), + variant: "primary", + icon: "plus", + type: "submit", + class: "flex-1", + data: { loading_button_target: "button" } + ) %> + <%= render DS::Link.new(text: t(".cancel"), variant: "secondary", href: accounts_path) %> +
+ <% end %> + <% end %> +<% end %> diff --git a/app/views/settings/providers/_kraken_panel.html.erb b/app/views/settings/providers/_kraken_panel.html.erb new file mode 100644 index 000000000..7de8e7831 --- /dev/null +++ b/app/views/settings/providers/_kraken_panel.html.erb @@ -0,0 +1,141 @@ +
+ <% items = local_assigns[:kraken_items] || @kraken_items || Current.family.kraken_items.active.ordered %> + + <%= render DS::Alert.new( + variant: :warning, + message: safe_join([ + content_tag(:p, t("settings.providers.kraken_panel.read_only_title"), class: "font-medium"), + content_tag(:p, t("settings.providers.kraken_panel.read_only_body"), class: "mt-1") + ]) + ) %> + + <%= render "settings/providers/setup_steps", + steps: [ + t("settings.providers.kraken_panel.step1_html").html_safe, + t("settings.providers.kraken_panel.step2"), + t("settings.providers.kraken_panel.step3") + ] %> + + <% error_msg = local_assigns[:error_message] || @error_message %> + <% if error_msg.present? %> + <%= render DS::Alert.new(message: error_msg, variant: :error) %> + <% end %> + + <% if items.any? %> +
+ <% items.each do |item| %> +
+ +
+
+ <%= icon "waves", size: "sm", class: "text-primary" %> +
+
+

<%= item.name %>

+

+ <% if item.syncing? %> + <%= t("settings.providers.kraken_panel.syncing") %> + <% else %> + <%= item.sync_status_summary %> + <% end %> +

+
+
+ + +
+
+ <%= button_to sync_kraken_item_path(item), + method: :post, + class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-secondary hover:text-primary border border-secondary rounded-lg hover:border-primary", + disabled: item.syncing? do %> + <%= icon "refresh-cw", size: "sm" %> + <%= t("settings.providers.kraken_panel.sync") %> + <% end %> + + <%= render DS::Link.new( + text: t("settings.providers.kraken_panel.setup_accounts"), + icon: "settings", + variant: "secondary", + href: setup_accounts_kraken_item_path(item), + frame: :modal + ) %> + + <%= button_to kraken_item_path(item), + method: :delete, + class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-destructive hover:bg-destructive/10 rounded-lg", + data: { turbo_confirm: t("settings.providers.kraken_panel.disconnect_confirm", name: item.name) } do %> + <%= icon "trash-2", size: "sm" %> + <%= t("settings.providers.kraken_panel.disconnect") %> + <% end %> +
+ + <%= styled_form_with model: item, + url: kraken_item_path(item), + scope: :kraken_item, + method: :patch, + data: { turbo: true }, + class: "space-y-3" do |form| %> + <%= form.text_field :name, + label: t("settings.providers.kraken_panel.connection_name_label"), + placeholder: t("settings.providers.kraken_panel.connection_name_placeholder") %> + + <%= form.text_field :api_key, + label: t("settings.providers.kraken_panel.api_key_label"), + placeholder: t("settings.providers.kraken_panel.keep_api_key_placeholder"), + type: :password, + value: nil %> + + <%= form.text_field :api_secret, + label: t("settings.providers.kraken_panel.api_secret_label"), + placeholder: t("settings.providers.kraken_panel.keep_api_secret_placeholder"), + type: :password, + value: nil %> + +
+ <%= form.submit t("settings.providers.kraken_panel.update_connection"), + class: "inline-flex items-center justify-center rounded-lg px-4 py-2 text-sm font-medium text-inverse bg-inverse hover:bg-inverse-hover focus:outline-none focus:ring-2 focus:ring-primary transition-colors" %> +
+ <% end %> +
+
+ <% end %> +
+ <% end %> + + <% kraken_item = Current.family.kraken_items.build(name: t("settings.providers.kraken_panel.default_connection_name")) %> + <% if items.any? %> +

+ <%= icon "plus", size: "sm" %> + <%= t("settings.providers.kraken_panel.add_connection") %> +

+ <% end %> + + <%= styled_form_with model: kraken_item, + url: kraken_items_path, + scope: :kraken_item, + method: :post, + data: { turbo: true }, + class: "space-y-3" do |form| %> + <%= form.text_field :name, + label: t("settings.providers.kraken_panel.connection_name_label"), + placeholder: t("settings.providers.kraken_panel.connection_name_placeholder") %> + + <%= form.text_field :api_key, + label: t("settings.providers.kraken_panel.api_key_label"), + placeholder: t("settings.providers.kraken_panel.api_key_placeholder"), + type: :password, + value: nil %> + + <%= form.text_field :api_secret, + label: t("settings.providers.kraken_panel.api_secret_label"), + placeholder: t("settings.providers.kraken_panel.api_secret_placeholder"), + type: :password, + value: nil %> + +
+ <%= form.submit t("settings.providers.kraken_panel.add_connection"), + class: "inline-flex items-center justify-center rounded-lg px-4 py-2 text-sm font-medium text-inverse bg-inverse hover:bg-inverse-hover focus:outline-none focus:ring-2 focus:ring-primary transition-colors" %> +
+ <% end %> +
diff --git a/config/locales/views/kraken_items/en.yml b/config/locales/views/kraken_items/en.yml new file mode 100644 index 000000000..eec60a9e4 --- /dev/null +++ b/config/locales/views/kraken_items/en.yml @@ -0,0 +1,85 @@ +--- +en: + kraken_items: + provider_connection: + default_name: Kraken + default_description: Link to a Kraken exchange account + name: "Kraken - %{name}" + description: "Link to %{name}" + create: + default_name: Kraken + success: Successfully connected to Kraken. Your exchange account is being synced. + update: + success: Successfully updated Kraken connection. + destroy: + success: Scheduled Kraken connection for deletion. + select_accounts: + select_connection: Choose a Kraken connection in Provider Settings. + no_credentials_configured: Add Kraken API credentials before setting up accounts. + link_accounts: + select_connection: Choose a Kraken connection before linking accounts. + select_existing_account: + title: Link Kraken Account + no_accounts_found: No Kraken accounts found. + wait_for_sync: Wait for Kraken to finish syncing. + check_provider_health: Check that your Kraken API credentials are valid. + link: Link + cancel: Cancel + link_existing_account: + success: Successfully linked to Kraken account + select_connection: Choose a Kraken connection before linking accounts. + errors: + only_manual: Only manual Crypto exchange accounts without an existing provider link can be linked to Kraken + invalid_kraken_account: Invalid Kraken account + kraken_account_already_linked: This Kraken account is already linked + setup_accounts: + title: Import Kraken Account + subtitle: Select the exchange account to track + instructions: Kraken imports one combined Crypto exchange account for this connection, with holdings and spot trade fills only. + no_accounts: All Kraken accounts have been imported. + accounts_count: + one: "%{count} account available" + other: "%{count} accounts available" + select_all: Select all + import_selected: Import Selected + cancel: Cancel + creating: Importing... + complete_account_setup: + success: + one: "Imported %{count} account" + other: "Imported %{count} accounts" + none_selected: No accounts selected + no_accounts: No accounts to import + kraken_item: + provider_name: Kraken + syncing: Syncing... + reconnect: Credentials need updating + deletion_in_progress: Deleting... + sync_status: + no_accounts: No accounts found + all_synced: + one: "%{count} account synced" + other: "%{count} accounts synced" + partial_sync: "%{linked_count} synced, %{unlinked_count} need setup" + status: "Last synced %{timestamp} ago" + status_with_summary: "Last synced %{timestamp} ago - %{summary}" + status_never: Never synced + delete: Delete + no_accounts_title: No accounts found + no_accounts_message: Your Kraken exchange account will appear here after syncing. + setup_needed: Account ready to import + setup_description: Import this Kraken connection as a Crypto exchange account. + setup_action: Import Account + import_accounts_menu: Import Account + stale_rate_warning: "Balance is approximate because the exact exchange rate for %{date} was unavailable. Will update on next sync." + kraken_item: + syncer: + checking_credentials: Checking credentials... + credentials_invalid: Invalid Kraken API credentials. Please check your API key and secret. + importing_accounts: Importing accounts from Kraken... + checking_configuration: Checking account configuration... + accounts_need_setup: + one: "%{count} account needs setup" + other: "%{count} accounts need setup" + processing_accounts: Processing account data... + calculating_balances: Calculating balances... diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index f3753374a..87114313b 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -235,6 +235,7 @@ en: mercury: Sync your Mercury business banking accounts automatically. coinbase: Import your Coinbase crypto holdings and track performance. binance: Sync your Binance spot balances using a read-only API key. + kraken: Sync Kraken balances and spot trade fills using a read-only API key. snaptrade: Connect brokerage accounts via the SnapTrade aggregation network. indexa_capital: Track your Indexa Capital automated investment portfolio. sophtron: Connect US & Canadian banks and utilities. @@ -284,6 +285,28 @@ en: syncing: Syncing... sync: Sync disconnect_confirm: "Are you sure you want to disconnect Binance?" + kraken_panel: + step1_html: 'Go to Kraken API settings' + step2: "Create an API key with Query Funds and Query Closed Orders & Trades only." + step3: "Paste the API key and private key below." + read_only_title: "Read-only exchange sync only" + read_only_body: "Do not grant trading, cancellation, withdrawal, export, ledger, Earn, staking, or transfer permissions. Sure only imports balances, holdings, and spot trade fills." + default_connection_name: Kraken + add_connection: Add Kraken connection + update_connection: Update connection + connection_name_label: Connection name + connection_name_placeholder: Main Kraken + api_key_label: API Key + api_key_placeholder: Paste your Kraken API key + keep_api_key_placeholder: Leave blank to keep the existing API key + api_secret_label: Private Key + api_secret_placeholder: Paste your Kraken private key + keep_api_secret_placeholder: Leave blank to keep the existing private key + setup_accounts: Setup account + syncing: Syncing... + sync: Sync + disconnect: Disconnect + disconnect_confirm: "Are you sure you want to disconnect %{name}?" enable_banking_panel: callback_url_instruction: "For the callback URL, use %{callback_url}." connection_error: Connection Error diff --git a/config/routes.rb b/config/routes.rb index 93f8655e7..1ab28a7c1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -64,6 +64,21 @@ Rails.application.routes.draw do end end + resources :kraken_items, only: [ :create, :update, :destroy ] do + collection do + get :select_accounts + post :link_accounts + get :select_existing_account + post :link_existing_account + end + + member do + post :sync + get :setup_accounts + post :complete_account_setup + end + end + resources :snaptrade_items, only: [ :index, :new, :create, :show, :edit, :update, :destroy ] do collection do get :preload_accounts diff --git a/db/migrate/20260511090000_create_kraken_items_and_accounts.rb b/db/migrate/20260511090000_create_kraken_items_and_accounts.rb new file mode 100644 index 000000000..48890a85c --- /dev/null +++ b/db/migrate/20260511090000_create_kraken_items_and_accounts.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class CreateKrakenItemsAndAccounts < ActiveRecord::Migration[7.2] + def change + create_table :kraken_items, id: :uuid do |t| + t.references :family, null: false, foreign_key: true, type: :uuid + t.string :name + + t.string :institution_name + t.string :institution_domain + t.string :institution_url + t.string :institution_color + + t.string :status, default: "good", null: false + t.boolean :scheduled_for_deletion, default: false, null: false + t.boolean :pending_account_setup, default: false, null: false + + t.datetime :sync_start_date + t.jsonb :raw_payload + + t.text :api_key + t.text :api_secret + t.bigint :last_nonce, default: 0, null: false + + t.timestamps + end + + add_index :kraken_items, :status + + create_table :kraken_accounts, id: :uuid do |t| + t.references :kraken_item, null: false, foreign_key: true, type: :uuid + + t.string :name + t.string :account_id, null: false + t.string :account_type + t.string :currency + t.decimal :current_balance, precision: 19, scale: 4 + + t.jsonb :institution_metadata + t.jsonb :raw_payload + t.jsonb :raw_transactions_payload + t.jsonb :extra, default: {}, null: false + + t.timestamps + end + + add_index :kraken_accounts, :account_type + add_index :kraken_accounts, + [ :kraken_item_id, :account_id ], + unique: true, + name: "index_kraken_accounts_on_item_and_account_id" + end +end diff --git a/db/schema.rb b/db/schema.rb index b40605ab9..1dbf42e98 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_05_10_120000) do +ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -201,9 +201,9 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_10_120000) do t.string "institution_domain" t.string "institution_url" t.string "institution_color" - t.string "status", default: "good" - t.boolean "scheduled_for_deletion", default: false - t.boolean "pending_account_setup", default: false + t.string "status", default: "good", null: false + t.boolean "scheduled_for_deletion", default: false, null: false + t.boolean "pending_account_setup", default: false, null: false t.datetime "sync_start_date" t.jsonb "raw_payload" t.text "api_key" @@ -853,6 +853,45 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_10_120000) do t.index ["token_digest"], name: "index_invite_codes_on_token_digest", unique: true, where: "(token_digest IS NOT NULL)" end + create_table "kraken_accounts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "kraken_item_id", null: false + t.string "name" + t.string "account_id", null: false + t.string "account_type" + t.string "currency" + t.decimal "current_balance", precision: 19, scale: 4 + t.jsonb "institution_metadata" + t.jsonb "raw_payload" + t.jsonb "raw_transactions_payload" + t.jsonb "extra", default: {}, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_type"], name: "index_kraken_accounts_on_account_type" + t.index ["kraken_item_id", "account_id"], name: "index_kraken_accounts_on_item_and_account_id", unique: true + t.index ["kraken_item_id"], name: "index_kraken_accounts_on_kraken_item_id" + end + + create_table "kraken_items", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "family_id", null: false + t.string "name" + t.string "institution_name" + t.string "institution_domain" + t.string "institution_url" + t.string "institution_color" + t.string "status", default: "good", null: false + t.boolean "scheduled_for_deletion", default: false, null: false + t.boolean "pending_account_setup", default: false, null: false + t.datetime "sync_start_date" + t.jsonb "raw_payload" + t.text "api_key" + t.text "api_secret" + t.bigint "last_nonce", default: 0, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["family_id"], name: "index_kraken_items_on_family_id" + t.index ["status"], name: "index_kraken_items_on_status" + end + create_table "llm_usages", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "family_id", null: false t.string "provider", null: false @@ -1718,6 +1757,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_10_120000) do add_foreign_key "indexa_capital_items", "families" add_foreign_key "invitations", "families" add_foreign_key "invitations", "users", column: "inviter_id" + add_foreign_key "kraken_accounts", "kraken_items" + add_foreign_key "kraken_items", "families" add_foreign_key "llm_usages", "families" add_foreign_key "lunchflow_accounts", "lunchflow_items" add_foreign_key "lunchflow_items", "families" diff --git a/test/controllers/api/v1/provider_connections_controller_test.rb b/test/controllers/api/v1/provider_connections_controller_test.rb index 815aeb9e7..ebd637380 100644 --- a/test/controllers/api/v1/provider_connections_controller_test.rb +++ b/test/controllers/api/v1/provider_connections_controller_test.rb @@ -104,12 +104,28 @@ class Api::V1::ProviderConnectionsControllerTest < ActionDispatch::IntegrationTe failed_at: Time.current, error: "raw provider token secret" ) + kraken_item = kraken_items(:one) + kraken_item.syncs.create!( + status: "failed", + failed_at: Time.current, + error: "raw kraken key secret" + ) get api_v1_provider_connections_url, headers: api_headers(@api_key) assert_response :success + json_response = JSON.parse(response.body) + kraken_connection = json_response["data"].detect do |connection| + connection["id"] == kraken_item.id && connection["provider"] == "kraken" + end + + assert_not_nil kraken_connection + assert_equal "KrakenItem", kraken_connection["provider_type"] refute_includes response.body, @mercury_item.token + refute_includes response.body, kraken_item.api_key + refute_includes response.body, kraken_item.api_secret refute_includes response.body, "raw provider token secret" + refute_includes response.body, "raw kraken key secret" end test "fails closed when credential readiness is unknown" do diff --git a/test/controllers/kraken_items_controller_test.rb b/test/controllers/kraken_items_controller_test.rb new file mode 100644 index 000000000..e287fcecb --- /dev/null +++ b/test/controllers/kraken_items_controller_test.rb @@ -0,0 +1,278 @@ +# frozen_string_literal: true + +require "test_helper" + +class KrakenItemsControllerTest < ActionDispatch::IntegrationTest + setup do + sign_in users(:family_admin) + SyncJob.stubs(:perform_later) + + @family = families(:dylan_family) + @existing_item = kraken_items(:one) + kraken_items(:requires_update).update!(scheduled_for_deletion: true) + @second_item = KrakenItem.create!( + family: @family, + name: "Business Kraken", + api_key: "second_kraken_key", + api_secret: "second_kraken_secret" + ) + end + + test "create adds a new kraken connection without overwriting existing credentials" do + existing_key = @existing_item.api_key + existing_secret = @existing_item.api_secret + + assert_difference "KrakenItem.count", 1 do + post kraken_items_url, params: { + kraken_item: { + name: "Joint Kraken", + api_key: "joint_kraken_key", + api_secret: "joint_kraken_secret" + } + } + end + + assert_redirected_to settings_providers_path + assert_equal existing_key, @existing_item.reload.api_key + assert_equal existing_secret, @existing_item.api_secret + assert_equal "joint_kraken_key", @family.kraken_items.find_by!(name: "Joint Kraken").api_key + end + + test "update changes only the selected kraken connection" do + existing_key = @existing_item.api_key + + patch kraken_item_url(@second_item), params: { + kraken_item: { + name: "Renamed Business Kraken", + api_key: "updated_second_key", + api_secret: "updated_second_secret" + } + } + + assert_redirected_to settings_providers_path + assert_equal existing_key, @existing_item.reload.api_key + assert_equal "Renamed Business Kraken", @second_item.reload.name + assert_equal "updated_second_key", @second_item.api_key + assert_equal "updated_second_secret", @second_item.api_secret + end + + test "blank secret update preserves the selected kraken credentials" do + original_key = @second_item.api_key + original_secret = @second_item.api_secret + + patch kraken_item_url(@second_item), params: { + kraken_item: { + name: "Renamed Business Kraken", + api_key: "", + api_secret: "" + } + } + + assert_redirected_to settings_providers_path + assert_equal "Renamed Business Kraken", @second_item.reload.name + assert_equal original_key, @second_item.api_key + assert_equal original_secret, @second_item.api_secret + end + + test "create rejects whitespace-only credentials" do + assert_no_difference "KrakenItem.count" do + post kraken_items_url, params: { + kraken_item: { + name: "Blank Kraken", + api_key: " ", + api_secret: "\n" + } + } + end + + assert_redirected_to settings_providers_path + assert_match(/API key can't be blank/i, flash[:alert]) + end + + test "select accounts requires an explicit connection when multiple kraken items exist" do + get select_accounts_kraken_items_url, params: { accountable_type: "Crypto" } + + assert_redirected_to settings_providers_path + assert_equal "Choose a Kraken connection in Provider Settings.", flash[:alert] + end + + test "select accounts targets selected kraken item" do + get select_accounts_kraken_items_url, params: { + kraken_item_id: @second_item.id, + accountable_type: "Crypto" + } + + assert_redirected_to setup_accounts_kraken_item_path(@second_item, return_to: nil) + end + + test "select accounts rejects protocol-relative return paths" do + get select_accounts_kraken_items_url, params: { + kraken_item_id: @second_item.id, + accountable_type: "Crypto", + return_to: "//evil.example/accounts" + } + + assert_redirected_to setup_accounts_kraken_item_path(@second_item, return_to: nil) + end + + test "sync only queues a sync for the selected kraken item" do + assert_difference -> { Sync.where(syncable: @second_item).count }, 1 do + assert_no_difference -> { Sync.where(syncable: @existing_item).count } do + post sync_kraken_item_url(@second_item) + end + end + + assert_response :redirect + end + + test "setup accounts creates crypto exchange account for selected item only" do + first_account = kraken_accounts(:one) + second_account = @second_item.kraken_accounts.create!( + name: "Second Kraken", + account_id: "combined", + account_type: "combined", + currency: "USD", + current_balance: 1000 + ) + KrakenAccount::Processor.any_instance.stubs(:process).returns(nil) + + assert_difference "Account.count", 1 do + post complete_account_setup_kraken_item_url(@second_item), params: { + selected_accounts: [ second_account.id ] + } + end + + assert_redirected_to accounts_path + assert_nil first_account.reload.current_account + assert_equal "Crypto", second_account.reload.current_account.accountable_type + assert_equal "exchange", second_account.current_account.accountable.subtype + end + + test "link existing account links manual crypto exchange account to selected kraken account" do + manual_account = manual_crypto_exchange_account + kraken_account = @second_item.kraken_accounts.create!( + name: "Kraken", + account_id: "combined", + account_type: "combined", + currency: "USD", + current_balance: 1000 + ) + + assert_difference "AccountProvider.count", 1 do + post link_existing_account_kraken_items_url, params: { + kraken_item_id: @second_item.id, + account_id: manual_account.id, + kraken_account_id: kraken_account.id + } + end + + assert_redirected_to accounts_path + assert_equal manual_account, kraken_account.reload.current_account + end + + test "link existing account requires explicit connection when multiple items exist" do + account = manual_crypto_exchange_account + + assert_no_difference "AccountProvider.count" do + post link_existing_account_kraken_items_url, params: { + account_id: account.id, + kraken_account_id: "combined" + } + end + + assert_redirected_to settings_providers_path + assert_equal "Choose a Kraken connection before linking accounts.", flash[:alert] + end + + test "link existing account rejects non crypto accounts" do + account = @family.accounts.create!( + name: "Manual Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + kraken_account = @second_item.kraken_accounts.create!(name: "Kraken", account_id: "combined", account_type: "combined", currency: "USD") + + assert_no_difference "AccountProvider.count" do + post link_existing_account_kraken_items_url, params: { + kraken_item_id: @second_item.id, + account_id: account.id, + kraken_account_id: kraken_account.id + } + end + + assert_redirected_to account_path(account) + end + + test "link existing account rejects accounts with existing provider links" do + account = manual_crypto_exchange_account + linked_kraken_account = kraken_accounts(:one) + AccountProvider.create!(account: account, provider: linked_kraken_account) + kraken_account = @second_item.kraken_accounts.create!(name: "Kraken", account_id: "combined", account_type: "combined", currency: "USD") + + assert_no_difference "AccountProvider.count" do + post link_existing_account_kraken_items_url, params: { + kraken_item_id: @second_item.id, + account_id: account.id, + kraken_account_id: kraken_account.id + } + end + + assert_redirected_to account_path(account) + end + + test "link existing account rejects kraken accounts already linked elsewhere" do + linked_account = manual_crypto_exchange_account + available_account = manual_crypto_exchange_account + kraken_account = @second_item.kraken_accounts.create!(name: "Kraken", account_id: "combined", account_type: "combined", currency: "USD") + AccountProvider.create!(account: linked_account, provider: kraken_account) + + assert_no_difference "AccountProvider.count" do + post link_existing_account_kraken_items_url, params: { + kraken_item_id: @second_item.id, + account_id: available_account.id, + kraken_account_id: kraken_account.id + } + end + + assert_redirected_to account_path(available_account) + end + + test "select existing account renders selected kraken item id" do + account = manual_crypto_exchange_account + @second_item.kraken_accounts.create!(name: "Kraken", account_id: "combined", account_type: "combined", currency: "USD") + + get select_existing_account_kraken_items_url, params: { + kraken_item_id: @second_item.id, + account_id: account.id + } + + assert_response :success + assert_includes @response.body, %(name="kraken_item_id") + assert_includes @response.body, %(value="#{@second_item.id}") + end + + test "cannot access another family's kraken item" do + other_item = KrakenItem.create!( + family: families(:empty), + name: "Other Kraken", + api_key: "other_key", + api_secret: "other_secret" + ) + + get setup_accounts_kraken_item_url(other_item) + + assert_response :not_found + end + + private + + def manual_crypto_exchange_account + @family.accounts.create!( + name: "Manual Crypto", + balance: 0, + currency: "USD", + accountable: Crypto.create!(subtype: "exchange") + ) + end +end diff --git a/test/fixtures/kraken_accounts.yml b/test/fixtures/kraken_accounts.yml new file mode 100644 index 000000000..9038ee9dc --- /dev/null +++ b/test/fixtures/kraken_accounts.yml @@ -0,0 +1,8 @@ +one: + kraken_item: one + name: Kraken + account_id: combined + account_type: combined + currency: USD + current_balance: 1234.50 + extra: {} diff --git a/test/fixtures/kraken_items.yml b/test/fixtures/kraken_items.yml new file mode 100644 index 000000000..9181b1c75 --- /dev/null +++ b/test/fixtures/kraken_items.yml @@ -0,0 +1,20 @@ +one: + family: dylan_family + name: My Kraken + api_key: test_kraken_key_123 + api_secret: test_kraken_secret_456 + last_nonce: 0 + status: good + institution_name: Kraken + institution_domain: kraken.com + institution_url: https://www.kraken.com + institution_color: "#5841D8" + +requires_update: + family: dylan_family + name: Stale Kraken + api_key: old_kraken_key + api_secret: old_kraken_secret + last_nonce: 0 + status: requires_update + institution_name: Kraken diff --git a/test/models/kraken_account/asset_normalizer_test.rb b/test/models/kraken_account/asset_normalizer_test.rb new file mode 100644 index 000000000..540caf4d3 --- /dev/null +++ b/test/models/kraken_account/asset_normalizer_test.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require "test_helper" + +class KrakenAccount::AssetNormalizerTest < ActiveSupport::TestCase + test "normalizes kraken symbols through metadata and fallbacks" do + normalizer = KrakenAccount::AssetNormalizer.new( + "XXBT" => { "altname" => "XBT" }, + "XETH" => { "altname" => "ETH" }, + "ZUSD" => { "altname" => "USD" } + ) + + assert_equal "BTC", normalizer.normalize("XXBT")[:symbol] + assert_equal "ETH", normalizer.normalize("XETH")[:symbol] + assert_equal "USD", normalizer.normalize("ZUSD")[:symbol] + end + + test "preserves kraken suffix variants while pricing base asset" do + normalizer = KrakenAccount::AssetNormalizer.new("XETH" => { "altname" => "ETH" }) + + parsed = normalizer.normalize("XETH.F") + + assert_equal "ETH.F", parsed[:symbol] + assert_equal "ETH", parsed[:price_symbol] + assert_equal ".F", parsed[:suffix] + assert_equal "XETH", parsed[:raw_base] + end +end diff --git a/test/models/kraken_account/holdings_processor_test.rb b/test/models/kraken_account/holdings_processor_test.rb new file mode 100644 index 000000000..171e332bf --- /dev/null +++ b/test/models/kraken_account/holdings_processor_test.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require "test_helper" + +class KrakenAccount::HoldingsProcessorTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @family.update!(currency: "USD") + @item = KrakenItem.create!( + family: @family, + name: "Kraken", + api_key: "k", + api_secret: "s" + ) + @kraken_account = @item.kraken_accounts.create!( + name: "Kraken", + account_id: "combined", + account_type: "combined", + currency: "USD", + current_balance: 30_000, + raw_payload: { + "assets" => [ + { "symbol" => "BTC", "price_symbol" => "BTC", "balance" => "0.5", "price_usd" => "60000.0", "source" => "spot" } + ] + } + ) + @account = Account.create!( + family: @family, + name: "Kraken", + balance: 0, + currency: "USD", + accountable: Crypto.create!(subtype: "exchange") + ) + @account_provider = AccountProvider.create!(account: @account, provider: @kraken_account) + @security = Security.create!(ticker: "CRYPTO:BTC", name: "BTC", exchange_operating_mic: "XKRA", offline: true) + KrakenAccount::SecurityResolver.stubs(:resolve).returns(@security) + end + + test "imports holdings with account_provider_id" do + import_adapter = mock + import_adapter.expects(:import_holding).with( + has_entries( + security: @security, + quantity: 0.5.to_d, + amount: 30_000.to_d, + currency: "USD", + price: 60_000.to_d, + external_id: "kraken_BTC_spot_#{Date.current}", + account_provider_id: @account_provider.id, + source: "kraken" + ) + ) + Account::ProviderImportAdapter.stubs(:new).returns(import_adapter) + + KrakenAccount::HoldingsProcessor.new(@kraken_account).process + end + + test "does not overwrite a different provider holding with the same security/date/currency" do + binance_item = BinanceItem.create!(family: @family, name: "Binance", api_key: "b", api_secret: "s") + binance_account = binance_item.binance_accounts.create!(name: "Binance", account_type: "combined", currency: "USD") + binance_provider = AccountProvider.create!(account: @account, provider: binance_account) + existing = @account.holdings.create!( + security: @security, + qty: 0.25, + amount: 15_000, + currency: "USD", + date: Date.current, + price: 60_000, + account_provider_id: binance_provider.id + ) + + assert_no_difference -> { @account.holdings.count } do + KrakenAccount::HoldingsProcessor.new(@kraken_account).process + end + + assert_equal binance_provider.id, existing.reload.account_provider_id + assert_nil existing.external_id + assert_nil @account.holdings.find_by(external_id: "kraken_BTC_spot_#{Date.current}") + end + + test "does not log raw asset payloads when holding import fails" do + raw_asset = { + "symbol" => "BTC", + "price_symbol" => "BTC", + "balance" => "0.5", + "price_usd" => "60000.0", + "source" => "spot", + "account_balance_detail" => "sensitive payload" + } + @kraken_account.update!(raw_payload: { "assets" => [ raw_asset ] }) + failing_adapter = mock + failing_adapter.stubs(:import_holding).raises(StandardError, "boom") + Account::ProviderImportAdapter.stubs(:new).returns(failing_adapter) + + Rails.logger.expects(:error) + .with("KrakenAccount::HoldingsProcessor - failed asset symbol=BTC: boom") + + KrakenAccount::HoldingsProcessor.new(@kraken_account).process + end +end diff --git a/test/models/kraken_account/processor_test.rb b/test/models/kraken_account/processor_test.rb new file mode 100644 index 000000000..157b654a7 --- /dev/null +++ b/test/models/kraken_account/processor_test.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require "test_helper" + +class KrakenAccount::ProcessorTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @family.update!(currency: "USD") + @item = KrakenItem.create!(family: @family, name: "Kraken", api_key: "k", api_secret: "s") + @kraken_account = @item.kraken_accounts.create!( + name: "Kraken", + account_id: "combined", + account_type: "combined", + currency: "USD", + current_balance: 1000, + raw_payload: { + "asset_metadata" => { + "XXBT" => { "altname" => "XBT" }, + "ZUSD" => { "altname" => "USD" } + }, + "pair_metadata" => { + "XXBTZUSD" => { "altname" => "XBTUSD", "base" => "XXBT", "quote" => "ZUSD" } + } + }, + raw_transactions_payload: { + "trades" => { + "buy_tx" => trade_payload("buy", "0.001", "50.00", "0.10"), + "sell_tx" => trade_payload("sell", "0.002", "120.00", "0.20") + } + } + ) + @account = Account.create!( + family: @family, + name: "Kraken", + balance: 0, + currency: "USD", + accountable: Crypto.create!(subtype: "exchange") + ) + AccountProvider.create!(account: @account, provider: @kraken_account) + @security = Security.create!(ticker: "CRYPTO:BTC", name: "BTC", exchange_operating_mic: "XKRA", offline: true) + KrakenAccount::SecurityResolver.stubs(:resolve).returns(@security) + KrakenAccount::HoldingsProcessor.any_instance.stubs(:process).returns(nil) + end + + test "imports buy and sell spot fills as trade entries" do + assert_difference -> { @account.entries.where(source: "kraken").count }, 2 do + KrakenAccount::Processor.new(@kraken_account).process + end + + buy = @account.entries.find_by!(external_id: "kraken_trade_buy_tx", source: "kraken") + assert_equal(-50.to_d, buy.amount) + assert_equal "USD", buy.currency + assert_equal 0.001.to_d, buy.trade.qty + assert_equal 50_000.to_d, buy.trade.price + assert_equal 0.10.to_d, buy.trade.fee + assert_equal "Buy", buy.trade.investment_activity_label + + sell = @account.entries.find_by!(external_id: "kraken_trade_sell_tx", source: "kraken") + assert_equal 120.to_d, sell.amount + assert_equal(-0.002.to_d, sell.trade.qty) + assert_equal 0.20.to_d, sell.trade.fee + assert_equal "Sell", sell.trade.investment_activity_label + end + + test "trade import is idempotent by txid" do + assert_difference -> { @account.entries.where(source: "kraken").count }, 2 do + KrakenAccount::Processor.new(@kraken_account).process + end + + assert_no_difference -> { @account.entries.where(source: "kraken").count } do + KrakenAccount::Processor.new(@kraken_account).process + end + end + + test "updates linked crypto account balance without cash balance" do + KrakenAccount::Processor.new(@kraken_account).process + + @account.reload + assert_equal 1000.to_d, @account.balance + assert_equal 0.to_d, @account.cash_balance + assert_equal "USD", @account.currency + end + + private + + def trade_payload(type, volume, cost, fee) + price = volume.to_d.zero? ? 0.to_d : cost.to_d / volume.to_d + + { + "ordertxid" => "order_#{type}", + "pair" => "XBTUSD", + "time" => Time.current.to_f, + "type" => type, + "price" => price.to_s("F"), + "cost" => cost, + "fee" => fee, + "vol" => volume + } + end +end diff --git a/test/models/kraken_item/importer_test.rb b/test/models/kraken_item/importer_test.rb new file mode 100644 index 000000000..2da355bb9 --- /dev/null +++ b/test/models/kraken_item/importer_test.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require "test_helper" + +class KrakenItem::ImporterTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @item = KrakenItem.create!( + family: @family, + name: "Kraken", + api_key: "k", + api_secret: "s" + ) + @provider = mock + @provider.stubs(:get_api_key_info).returns({ "name" => "Sure read-only" }) + @provider.stubs(:get_asset_pairs).returns(pair_metadata) + @provider.stubs(:get_trades_history).returns({ "count" => 0, "trades" => {} }) + @provider.stubs(:get_ticker).returns(nil) + end + + test "creates a combined kraken account from BalanceEx" do + @provider.stubs(:get_asset_info).returns(asset_metadata) + @provider.stubs(:get_extended_balance).returns( + "XXBT" => { "balance" => "1.0", "credit" => "0", "credit_used" => "0", "hold_trade" => "0.25" }, + "ZUSD" => { "balance" => "50.0", "credit" => "0", "credit_used" => "0", "hold_trade" => "0" } + ) + @provider.stubs(:get_ticker).with("XBTUSD").returns("XXBTZUSD" => { "c" => [ "50000.00" ] }) + + assert_difference "@item.kraken_accounts.count", 1 do + KrakenItem::Importer.new(@item, kraken_provider: @provider).import + end + + account = @item.kraken_accounts.first + assert_equal "combined", account.account_id + assert_equal "combined", account.account_type + assert_equal "USD", account.currency + assert_in_delta 50_050, account.current_balance, 0.01 + + btc = account.raw_payload["assets"].find { |asset| asset["symbol"] == "BTC" } + assert_equal "0.75", btc["available"] + assert_equal "0.25", btc["hold_trade"] + end + + test "preserves suffix assets in metadata and marks missing prices" do + @provider.stubs(:get_asset_info).returns(asset_metadata) + @provider.stubs(:get_extended_balance).returns( + "XETH.F" => { "balance" => "2.0", "credit" => "0", "credit_used" => "0", "hold_trade" => "0" } + ) + + KrakenItem::Importer.new(@item, kraken_provider: @provider).import + + account = @item.kraken_accounts.first + eth = account.raw_payload["assets"].first + assert_equal "ETH.F", eth["symbol"] + assert_equal "ETH", eth["price_symbol"] + assert_equal ".F", eth["suffix"] + assert_equal "missing", eth["price_status"] + assert_includes account.extra.dig("kraken", "missing_prices"), "ETH.F" + end + + test "paginates TradesHistory in 50 fill pages" do + @provider.stubs(:get_asset_info).returns({}) + @provider.stubs(:get_extended_balance).returns({}) + first_page = 50.times.to_h { |i| [ "tx#{i}", trade_payload("tx#{i}") ] } + second_page = { "tx50" => trade_payload("tx50") } + + @provider.expects(:get_trades_history).with(start: nil, offset: 0).returns({ "count" => 51, "trades" => first_page }) + @provider.expects(:get_trades_history).with(start: nil, offset: 50).returns({ "count" => 51, "trades" => second_page }) + + result = KrakenItem::Importer.new(@item, kraken_provider: @provider).import + + assert_equal 51, result[:trades_imported] + assert_equal 51, @item.kraken_accounts.first.raw_transactions_payload["trades"].size + end + + test "marks item requires_update when required endpoint reports permission error" do + @provider.stubs(:get_asset_info).returns({}) + @provider.stubs(:get_asset_pairs).returns({}) + @provider.stubs(:get_extended_balance).raises(Provider::Kraken::PermissionError, "EGeneral:Permission denied") + + assert_raises(Provider::Kraken::PermissionError) do + KrakenItem::Importer.new(@item, kraken_provider: @provider).import + end + + assert @item.reload.requires_update? + end + + private + + def asset_metadata + { + "XXBT" => { "altname" => "XBT" }, + "XETH" => { "altname" => "ETH" }, + "ZUSD" => { "altname" => "USD" } + } + end + + def pair_metadata + { + "XXBTZUSD" => { "altname" => "XBTUSD", "base" => "XXBT", "quote" => "ZUSD" } + } + end + + def trade_payload(txid) + { + "ordertxid" => "order_#{txid}", + "pair" => "XBTUSD", + "time" => Time.current.to_f, + "type" => "buy", + "price" => "50000.0", + "cost" => "50.0", + "fee" => "0.1", + "vol" => "0.001" + } + end +end diff --git a/test/models/kraken_item_test.rb b/test/models/kraken_item_test.rb new file mode 100644 index 000000000..69c08d2f2 --- /dev/null +++ b/test/models/kraken_item_test.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require "test_helper" + +class KrakenItemTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @item = KrakenItem.create!( + family: @family, + name: "My Kraken", + api_key: "test_key", + api_secret: "test_secret" + ) + end + + test "belongs to family" do + assert_equal @family, @item.family + end + + test "has good status by default" do + assert_equal "good", @item.status + end + + test "strips credential whitespace before validation" do + item = KrakenItem.create!( + family: @family, + name: "Whitespace Kraken", + api_key: " key \n", + api_secret: " secret \n" + ) + + assert_equal "key", item.api_key + assert_equal "secret", item.api_secret + end + + test "rejects whitespace-only credentials" do + item = KrakenItem.new(family: @family, name: "Blank Kraken", api_key: " ", api_secret: "\n") + + assert_not item.valid? + assert_includes item.errors[:api_key], "can't be blank" + assert_includes item.errors[:api_secret], "can't be blank" + end + + test "credentials_configured rejects whitespace-only values" do + @item.update_columns(api_key: " ", api_secret: "secret") + + assert_not @item.reload.credentials_configured? + end + + test "next_nonce is monotonic even when stored nonce is ahead of clock" do + @item.update!(last_nonce: 9_000_000_000_000_000_000) + + first = @item.next_nonce!.to_i + second = @item.next_nonce!.to_i + + assert_equal 9_000_000_000_000_000_001, first + assert_equal 9_000_000_000_000_000_002, second + assert_equal second, @item.reload.last_nonce + end + + test "kraken provider uses item nonce generator" do + @item.update!(last_nonce: 9_000_000_000_000_000_000) + provider = @item.kraken_provider + + nonce = provider.send(:nonce_generator).call + + assert_equal "9000000000000000001", nonce + assert_equal 9_000_000_000_000_000_001, @item.reload.last_nonce + end + + test "duplicate combined account ids are scoped by kraken item" do + other_item = KrakenItem.create!( + family: @family, + name: "Other Kraken", + api_key: "other_key", + api_secret: "other_secret" + ) + + @item.kraken_accounts.create!(name: "Main", account_id: "combined", account_type: "combined", currency: "USD") + other_account = other_item.kraken_accounts.create!(name: "Other", account_id: "combined", account_type: "combined", currency: "USD") + + assert other_account.persisted? + end + + test "encrypts credentials when active record encryption is configured" do + skip "Encryption not configured" unless KrakenItem.encryption_ready? + + item = KrakenItem.create!( + family: @family, + name: "Encrypted Kraken", + api_key: "encrypted_key", + api_secret: "encrypted_secret" + ) + + quoted_id = KrakenItem.connection.quote(item.id) + raw = KrakenItem.connection.select_one("SELECT api_key, api_secret FROM kraken_items WHERE id = #{quoted_id}") + assert_not_equal "encrypted_key", raw["api_key"] + assert_not_equal "encrypted_secret", raw["api_secret"] + end +end diff --git a/test/models/provider/kraken_adapter_test.rb b/test/models/provider/kraken_adapter_test.rb new file mode 100644 index 000000000..a9ce06a08 --- /dev/null +++ b/test/models/provider/kraken_adapter_test.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +require "test_helper" +require "uri" + +class Provider::KrakenAdapterTest < ActiveSupport::TestCase + setup do + kraken_items(:requires_update).update!(scheduled_for_deletion: true) + end + + test "supports Crypto accounts only" do + assert_includes Provider::KrakenAdapter.supported_account_types, "Crypto" + assert_not_includes Provider::KrakenAdapter.supported_account_types, "Depository" + end + + test "returns fallback connection config when no credentials exist yet" do + family = families(:empty) + configs = Provider::KrakenAdapter.connection_configs(family: family) + + assert_equal 1, configs.length + assert_equal "kraken", configs.first[:key] + assert_equal I18n.t("kraken_items.provider_connection.default_name"), configs.first[:name] + assert configs.first[:can_connect] + end + + test "returns one connection config per credentialed kraken item" do + family = families(:dylan_family) + first_item = kraken_items(:one) + second_item = KrakenItem.create!( + family: family, + name: "Business Kraken", + api_key: "second_kraken_key", + api_secret: "second_kraken_secret" + ) + + configs = Provider::KrakenAdapter.connection_configs(family: family) + + assert_equal [ "kraken_#{second_item.id}", "kraken_#{first_item.id}" ], configs.map { |config| config[:key] } + assert_equal [ + I18n.t("kraken_items.provider_connection.name", name: second_item.name), + I18n.t("kraken_items.provider_connection.name", name: first_item.name) + ], configs.map { |config| config[:name] } + + new_account_uri = URI.parse(configs.first[:new_account_path].call("Crypto", "/accounts")) + assert_equal "/kraken_items/select_accounts", new_account_uri.path + assert_includes new_account_uri.query, "kraken_item_id=#{second_item.id}" + + existing_account_uri = URI.parse(configs.first[:existing_account_path].call(accounts(:crypto).id)) + assert_equal "/kraken_items/select_existing_account", existing_account_uri.path + assert_includes existing_account_uri.query, "kraken_item_id=#{second_item.id}" + end + + test "connection configs ignore whitespace-only credentials" do + family = families(:dylan_family) + blank_item = KrakenItem.create!( + family: family, + name: "Blank Kraken", + api_key: "temporary_key", + api_secret: "temporary_secret" + ) + blank_item.update_columns(api_key: " ", api_secret: " ") + + configs = Provider::KrakenAdapter.connection_configs(family: family) + + assert_equal [ "kraken_#{kraken_items(:one).id}" ], configs.map { |config| config[:key] } + end + + test "build_provider returns nil when family is nil" do + assert_nil Provider::KrakenAdapter.build_provider(family: nil) + end + + test "build_provider returns nil when family has no kraken items" do + assert_nil Provider::KrakenAdapter.build_provider(family: families(:empty)) + end + + test "build_provider returns Kraken provider when only one credentialed item exists" do + provider = Provider::KrakenAdapter.build_provider(family: families(:dylan_family)) + + assert_instance_of Provider::Kraken, provider + end + + test "build_provider requires explicit item when multiple credentialed items exist" do + family = families(:dylan_family) + KrakenItem.create!( + family: family, + name: "Second Kraken", + api_key: "second_kraken_key", + api_secret: "second_kraken_secret" + ) + + assert_nil Provider::KrakenAdapter.build_provider(family: family) + end + + test "build_provider uses explicit kraken item credentials" do + family = families(:dylan_family) + second_item = KrakenItem.create!( + family: family, + name: "Second Kraken", + api_key: " second_kraken_key \n", + api_secret: " second_kraken_secret \n" + ) + + provider = Provider::KrakenAdapter.build_provider(family: family, kraken_item_id: second_item.id) + + assert_instance_of Provider::Kraken, provider + assert_equal "second_kraken_key", provider.api_key + assert_equal "second_kraken_secret", provider.api_secret + end + + test "build_provider refuses kraken items outside the family" do + family = families(:dylan_family) + other_item = KrakenItem.create!( + family: families(:empty), + name: "Other Kraken", + api_key: "other_kraken_key", + api_secret: "other_kraken_secret" + ) + + assert_nil Provider::KrakenAdapter.build_provider(family: family, kraken_item_id: other_item.id) + end + + test "build_provider refuses explicit kraken item without usable credentials" do + family = families(:dylan_family) + blank_item = KrakenItem.create!( + family: family, + name: "Blank Kraken", + api_key: "temporary_key", + api_secret: "temporary_secret" + ) + blank_item.update_columns(api_key: " ", api_secret: " ") + + assert_nil Provider::KrakenAdapter.build_provider(family: family, kraken_item_id: blank_item.id) + end +end diff --git a/test/models/provider/kraken_test.rb b/test/models/provider/kraken_test.rb new file mode 100644 index 000000000..0d233420c --- /dev/null +++ b/test/models/provider/kraken_test.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +require "test_helper" +require "base64" + +class Provider::KrakenTest < ActiveSupport::TestCase + # Public Kraken docs signing sample, stored as bytes so secret scanners do + # not mistake the test vector for an accidentally committed credential. + OFFICIAL_SAMPLE_SECRET_BYTES = [ + 145, 1, 249, 29, 111, 252, 167, 91, 134, 57, 88, 219, 129, 96, 59, 22, + 233, 192, 152, 99, 188, 150, 196, 148, 92, 219, 46, 221, 234, 48, 239, + 171, 51, 243, 132, 53, 241, 245, 177, 159, 36, 115, 4, 112, 157, 222, + 151, 121, 156, 79, 106, 107, 223, 71, 1, 155, 110, 102, 232, 250, 23, + 88, 110, 94 + ].freeze + OFFICIAL_SAMPLE_SIGNATURE = "4/dpxb3iT4tp/ZCVEwSnEsLxx0bqyhLpdfOpc6fn7OR8+UClSV5n9E6aSS8MPtnRfp32bAb0nmbRn6H8ndwLUQ==" + + setup do + @provider = Provider::Kraken.new(api_key: "test_key", api_secret: official_sample_secret, nonce_generator: -> { "1616492376594" }) + end + + test "sign matches official Kraken Spot REST sample" do + params = { + "nonce" => "1616492376594", + "ordertype" => "limit", + "pair" => "XBTUSD", + "price" => "37500", + "type" => "buy", + "volume" => "1.25" + } + + signature = @provider.send(:sign, "/0/private/AddOrder", params) + + assert_equal OFFICIAL_SAMPLE_SIGNATURE, signature + end + + test "auth headers include api key and signature" do + headers = @provider.send(:auth_headers, "/0/private/BalanceEx", { "nonce" => "1616492376594" }) + + assert_equal "test_key", headers["API-Key"] + assert headers["API-Sign"].present? + assert_equal 64, Base64.strict_decode64(headers["API-Sign"]).bytesize + end + + test "private requests send signed post body and auth headers" do + response = mock_httparty_response(200, { "error" => [], "result" => { "name" => "Sure read-only" } }) + + Provider::Kraken.expects(:post) + .with( + "/0/private/GetApiKeyInfo", + has_entries( + body: "nonce=1616492376594", + headers: has_entries("API-Key" => "test_key", "Content-Type" => "application/x-www-form-urlencoded") + ) + ) + .returns(response) + + assert_equal({ "name" => "Sure read-only" }, @provider.get_api_key_info) + end + + test "handle response returns result on success" do + response = mock_httparty_response(200, { "error" => [], "result" => { "XXBT" => { "balance" => "1.0" } } }) + + assert_equal({ "XXBT" => { "balance" => "1.0" } }, @provider.send(:handle_response, response)) + end + + test "handle response raises api error for non 2xx" do + response = mock_httparty_response(500, { "error" => [ "EService:Unavailable" ] }) + + assert_raises(Provider::Kraken::ApiError) do + @provider.send(:handle_response, response) + end + end + + test "handle response rejects non-envelope payloads" do + response = mock_httparty_response(200, [ "not", "an", "envelope" ]) + + error = assert_raises(Provider::Kraken::ApiError) do + @provider.send(:handle_response, response) + end + + assert_equal "Malformed Kraken API response", error.message + end + + test "handle response requires error key" do + response = mock_httparty_response(200, { "result" => {} }) + + error = assert_raises(Provider::Kraken::ApiError) do + @provider.send(:handle_response, response) + end + + assert_equal "Malformed Kraken API response: missing error", error.message + end + + test "handle response requires result key" do + response = mock_httparty_response(200, { "error" => [] }) + + error = assert_raises(Provider::Kraken::ApiError) do + @provider.send(:handle_response, response) + end + + assert_equal "Malformed Kraken API response: missing result", error.message + end + + test "handle response maps invalid key errors" do + assert_raises(Provider::Kraken::AuthenticationError) do + @provider.send(:handle_response, kraken_error_response("EAPI:Invalid key")) + end + end + + test "handle response maps invalid signature errors" do + assert_raises(Provider::Kraken::AuthenticationError) do + @provider.send(:handle_response, kraken_error_response("EAPI:Invalid signature")) + end + end + + test "handle response maps permission errors" do + assert_raises(Provider::Kraken::PermissionError) do + @provider.send(:handle_response, kraken_error_response("EGeneral:Permission denied")) + end + end + + test "handle response maps rate limit errors" do + assert_raises(Provider::Kraken::RateLimitError) do + @provider.send(:handle_response, kraken_error_response("EAPI:Rate limit exceeded")) + end + end + + test "handle response maps throttled errors as rate limits" do + assert_raises(Provider::Kraken::RateLimitError) do + @provider.send(:handle_response, kraken_error_response("EService:Throttled: 1770000000")) + end + end + + test "handle response maps nonce errors" do + assert_raises(Provider::Kraken::NonceError) do + @provider.send(:handle_response, kraken_error_response("EAPI:Invalid nonce")) + end + end + + test "handle response maps otp required errors" do + assert_raises(Provider::Kraken::OTPRequiredError) do + @provider.send(:handle_response, kraken_error_response("EAPI:Invalid arguments:otp required")) + end + end + + private + + def official_sample_secret + Base64.strict_encode64(OFFICIAL_SAMPLE_SECRET_BYTES.pack("C*")) + end + + def kraken_error_response(error) + mock_httparty_response(200, { "error" => [ error ], "result" => nil }) + end + + def mock_httparty_response(code, body) + response = mock + response.stubs(:code).returns(code) + response.stubs(:parsed_response).returns(body) + response + end +end diff --git a/test/models/provider_connection_status_test.rb b/test/models/provider_connection_status_test.rb index 88ef50866..fa243a715 100644 --- a/test/models/provider_connection_status_test.rb +++ b/test/models/provider_connection_status_test.rb @@ -78,4 +78,15 @@ class ProviderConnectionStatusTest < ActiveSupport::TestCase assert_equal 1, status.dig(:accounts, :linked_count) assert_equal 1, status.dig(:accounts, :unlinked_count) end + + test "kraken provider status is included without credential fields" do + statuses = ProviderConnectionStatus.for_family(families(:dylan_family)) + kraken_status = statuses.find { |status| status[:provider] == "kraken" } + + assert kraken_status + assert_equal "KrakenItem", kraken_status[:provider_type] + refute_includes kraken_status.keys, :api_key + refute_includes kraken_status.keys, :api_secret + assert_equal true, kraken_status[:credentials_configured] + end end diff --git a/test/models/transaction_import_test.rb b/test/models/transaction_import_test.rb index c832b59f1..f3278497e 100644 --- a/test/models/transaction_import_test.rb +++ b/test/models/transaction_import_test.rb @@ -100,7 +100,7 @@ class TransactionImportTest < ActiveSupport::TestCase @import.publish end - assert_equal [ -100, 200, -300 ], @import.entries.map(&:amount) + assert_equal [ -100, 200, -300 ], @import.entries.order(:date).map(&:amount) end test "does not create duplicate when matching transaction exists with same name" do From 7c06fe6296ad244bc10d0b78c9e0ca473a2028ab Mon Sep 17 00:00:00 2001 From: Guillem Arias Fauste Date: Tue, 12 May 2026 00:37:47 +0200 Subject: [PATCH 10/31] feat(recurring): allow marking transfers as recurring (#895) (#1589) Refs #895, discussion #1224. Adds a "Mark as recurring" entry point on the transfer detail drawer that creates a `RecurringTransaction` carrying both source and destination accounts. The recurring index, settings toggle (`recurring_transactions_disabled`), and projected upcoming feed all light up automatically once the data shape is there. Schema: * `destination_account_id` nullable FK to accounts. `on_delete: :cascade` matches #20251030172500's precedent for accounts FKs. The existing `account_id` FK is widened to cascade in the same migration so Family destruction with a recurring transfer doesn't FK-violate. * Two predicate-partitioned partial unique indexes per shape: non-transfer rows (`destination_account_id IS NULL`, original 5-column shape preserved) and transfer rows (6-column shape including the destination). Postgres treats NULLs as distinct in unique indexes, so widening would have broken non-transfer dedupe. * Two CHECK constraints enforcing transfer invariants in PostgreSQL: `chk_recurring_txns_transfer_requires_source` (destination implies source) and `chk_recurring_txns_transfer_distinct_accounts` (destination cannot equal source). Per CLAUDE.md "Enforce null checks, unique indexes, and simple validations in the database schema for PostgreSQL". * `Account` gains an `inbound_recurring_transfers` inverse so the destroy chain reaches both ends. Controller / behaviour: * `transfers#mark_as_recurring` mirrors `transactions#mark_as_recurring`: i18n flashes (4 new keys: transfer_marked_as_recurring, transfer_already_exists, transfer_creation_failed, transfer_feature_disabled), `respond_to format.html`, `redirect_back_or_to transactions_path`, server-side gate on `recurring_transactions_disabled?`, and rescue both `RecordInvalid` and `RecordNotUnique` for the race window between the dedupe `find_by` and `create_from_transfer`. The `StandardError` rescue now logs the exception (class, message, transfer/family/user ids) before surfacing the generic flash so production failures aren't context-less. * `RecurringTransaction.accessible_by(user)` now requires destination_account_id (when present) to be in the user's accessible set, so a recurring transfer never leaks to a user without access to BOTH endpoints. * Model validation gains a `destination_account.blank?` branch in `transfer_endpoints_consistent` so a dangling `destination_account_id` (referenced row destroyed) surfaces as a normal validation error instead of an FK exception on save. * `Identifier` filter for transfer-kind transactions moved into SQL. UI: * Recurring index table and projected feed render transfer rows with the existing letter-avatar and the row's `name` field ("Transfer to {destination}"). No special pill or icon -- every row in `/recurring_transactions` is recurring by definition. Amount column on transfers uses `text-secondary` (muted-but-live) instead of the income/expense colour, since transfers are zero-net for the family. Out of scope (called out in the PR body): * Auto-creation of future Transfer rows on a schedule (discussion #1224's primary ask). Behaviour change vs the current projection-only model. * Auto-identification of recurring transfer pairs in `Identifier`. * Frequency model richer than `expected_day_of_month`. * `Cleaner` for recurring transfers (issue #1590 tracks this). Tests: * `RecurringTransaction#transfer?` predicate (with / without destination). * `transfer_endpoints_consistent`: rejects same source and destination, rejects dangling destination_account_id, rejects cross-family destination. * `RecurringTransaction.create_from_transfer` happy path; multi-currency variant stores source-side currency. * `projected_entry` exposes source / destination on transfer rows. * `Identifier` skips transfer-kind transactions; creates a pattern from expense halves while ignoring co-resident transfer halves. * Destroying the destination account cascades to inbound recurring transfers (FK + AR association). * Unique partial index still de-duplicates non-transfer rows after the destination_account_id widening. * `transfers#mark_as_recurring` happy path, idempotent on second call, rejected when `recurring_transactions_disabled`. Suite: 3261 / 0 / 0 / 24 on the latest upstream/main. Lint clean. Brakeman clean. Signed-off-by: Guillem Arias Fauste --- app/controllers/transfers_controller.rb | 72 +++- app/models/account.rb | 8 + app/models/recurring_transaction.rb | 87 ++++- app/models/recurring_transaction/cleaner.rb | 14 +- .../recurring_transaction/identifier.rb | 22 +- .../_projected_transaction.html.erb | 6 +- .../recurring_transactions/index.html.erb | 26 +- app/views/transfers/show.html.erb | 16 + .../views/recurring_transactions/en.yml | 4 + config/locales/views/transfers/en.yml | 4 + config/routes.rb | 6 +- ...on_account_id_to_recurring_transactions.rb | 107 ++++++ db/schema.rb | 17 +- test/controllers/transfers_controller_test.rb | 44 +++ test/models/recurring_transaction_test.rb | 322 ++++++++++++++++++ 15 files changed, 729 insertions(+), 26 deletions(-) create mode 100644 db/migrate/20260429120000_add_destination_account_id_to_recurring_transactions.rb diff --git a/app/controllers/transfers_controller.rb b/app/controllers/transfers_controller.rb index 255ffb86b..465891f2f 100644 --- a/app/controllers/transfers_controller.rb +++ b/app/controllers/transfers_controller.rb @@ -1,7 +1,7 @@ class TransfersController < ApplicationController include StreamExtensions - before_action :set_transfer, only: %i[show destroy update] + before_action :set_transfer, only: %i[show destroy update mark_as_recurring] before_action :set_accounts, only: %i[new create] def new @@ -11,6 +11,17 @@ class TransfersController < ApplicationController def show @categories = Current.family.categories.alphabetically + + # Whether the current user can hit `mark_as_recurring`: feature flag on, + # AND they have write access to BOTH transfer endpoints. Gating the + # view button on this avoids showing a CTA that the controller would + # reject via `require_account_permission!` for read-only sharers. + endpoint_ids = [ @transfer.from_account&.id, @transfer.to_account&.id ].compact + writable_endpoint_count = Account.writable_by(Current.user).where(id: endpoint_ids).distinct.count + @can_mark_as_recurring_transfer = + !Current.family.recurring_transactions_disabled? && + endpoint_ids.size == 2 && + writable_endpoint_count == 2 end def create @@ -75,6 +86,65 @@ class TransfersController < ApplicationController redirect_back_or_to transactions_url, notice: t(".success") end + def mark_as_recurring + if Current.family.recurring_transactions_disabled? + flash[:alert] = t("recurring_transactions.transfer_feature_disabled") + redirect_back_or_to transactions_path + return + end + + source_account = @transfer.from_account + destination_account = @transfer.to_account + + if source_account.nil? || destination_account.nil? + flash[:alert] = t("recurring_transactions.unexpected_error") + redirect_back_or_to transactions_path + return + end + + return unless require_account_permission!(source_account) + return unless require_account_permission!(destination_account) + + existing = Current.family.recurring_transactions.find_by( + account_id: source_account.id, + destination_account_id: destination_account.id, + amount: @transfer.outflow_transaction.entry.amount, + currency: @transfer.outflow_transaction.entry.currency + ) + + if existing + flash[:alert] = t("recurring_transactions.transfer_already_exists") + respond_to do |format| + format.html { redirect_back_or_to transactions_path } + end + return + end + + begin + RecurringTransaction.create_from_transfer(@transfer) + flash[:notice] = t("recurring_transactions.transfer_marked_as_recurring") + respond_to do |format| + format.html { redirect_back_or_to transactions_path } + end + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique + # RecordNotUnique covers the race window between `find_by` and `create!` + # (the partial unique index protects us at the DB level). + flash[:alert] = t("recurring_transactions.transfer_creation_failed") + respond_to do |format| + format.html { redirect_back_or_to transactions_path } + end + rescue StandardError => e + Rails.logger.error( + "transfers#mark_as_recurring failed: #{e.class} #{e.message} " \ + "(transfer=#{@transfer&.id} family=#{Current.family&.id} user=#{Current.user&.id})" + ) + flash[:alert] = t("recurring_transactions.unexpected_error") + respond_to do |format| + format.html { redirect_back_or_to transactions_path } + end + end + end + private def set_transfer # Finds the transfer and ensures the user has access to it diff --git a/app/models/account.rb b/app/models/account.rb index 0c50e1f70..c5363ced4 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -20,6 +20,14 @@ class Account < ApplicationRecord has_many :holdings, dependent: :destroy has_many :balances, dependent: :destroy has_many :recurring_transactions, dependent: :destroy + # Inverse for recurring transfers where this account is the destination. + # Account#recurring_transactions only matches account_id; without this + # association, destroying the destination account would hit the FK + # cascade silently and the AR cache wouldn't reflect the deletion. + has_many :inbound_recurring_transfers, + class_name: "RecurringTransaction", + foreign_key: :destination_account_id, + dependent: :destroy monetize :balance, :cash_balance diff --git a/app/models/recurring_transaction.rb b/app/models/recurring_transaction.rb index 0b30e6bd7..ffc2c75be 100644 --- a/app/models/recurring_transaction.rb +++ b/app/models/recurring_transaction.rb @@ -3,6 +3,7 @@ class RecurringTransaction < ApplicationRecord belongs_to :family belongs_to :account, optional: true + belongs_to :destination_account, optional: true, class_name: "Account" belongs_to :merchant, optional: true monetize :amount @@ -19,6 +20,7 @@ class RecurringTransaction < ApplicationRecord validates :occurrence_count, numericality: { only_integer: true, greater_than_or_equal_to: 0 } validate :merchant_or_name_present validate :amount_variance_consistency + validate :transfer_endpoints_consistent def merchant_or_name_present if merchant_id.blank? && name.blank? @@ -36,10 +38,50 @@ class RecurringTransaction < ApplicationRecord end end + # When this row represents a recurring transfer, both endpoints must be + # present, belong to the same family, and not be the same account. + def transfer_endpoints_consistent + return if destination_account_id.blank? + + if account_id.blank? + errors.add(:account, "must be present on a recurring transfer") + elsif account.blank? + # account_id references a row that was destroyed. Mirror the + # destination_account.blank? branch so the source side surfaces a + # normal validation error too. + errors.add(:account, "must exist") + elsif destination_account.blank? + # destination_account_id references a row that was destroyed (or never + # existed). Surface as a normal validation error instead of letting + # the FK fire on save. + errors.add(:destination_account, "must exist") + elsif account_id == destination_account_id + errors.add(:destination_account, "cannot be the same as the source account") + elsif account.family_id != destination_account.family_id + errors.add(:destination_account, "must belong to the same family as the source account") + end + end + + def transfer? + destination_account_id.present? + end + scope :for_family, ->(family) { where(family: family) } scope :expected_soon, -> { active.where("next_expected_date <= ?", 1.month.from_now) } scope :accessible_by, ->(user) { - where(account_id: Account.accessible_by(user).select(:id)).or(where(account_id: nil)) + accessible_account_ids = Account.accessible_by(user).select(:id) + # A recurring row is accessible when: + # * its account_id is in the user's accessible set or null (legacy rows + # with no account scoping survive), AND + # * its destination_account_id is also accessible OR null (so a recurring + # transfer never leaks into the list of a user without access to BOTH + # endpoints). + where(account_id: accessible_account_ids) + .or(where(account_id: nil)) + .merge( + where(destination_account_id: accessible_account_ids) + .or(where(destination_account_id: nil)) + ) } # Class methods for identification and cleanup @@ -58,6 +100,44 @@ class RecurringTransaction < ApplicationRecord Cleaner.new(family).cleanup_stale_transactions end + # Create a manual recurring transfer from an existing Transfer pair. + # Mirrors `create_from_transaction` but populates source + destination + # accounts and skips merchant / variance lookup -- transfers are + # account-pair-shaped, not merchant-shaped. + def self.create_from_transfer(transfer) + outflow_entry = transfer.outflow_transaction&.entry + inflow_entry = transfer.inflow_transaction&.entry + + raise ArgumentError, "transfer is missing one of its entries" unless outflow_entry && inflow_entry + + source_account = outflow_entry.account + destination_account = inflow_entry.account + family = source_account.family + + expected_day = outflow_entry.date.day + next_expected = calculate_next_expected_date_from_today(expected_day) + + create!( + family: family, + account: source_account, + destination_account: destination_account, + merchant_id: nil, + # Transfer#name yields "Payment to ..." for liability destinations + # and "Transfer to ..." otherwise, matching Transfer::Creator's + # name_prefix logic so the recurring row reads consistently with + # the originating Transfer. + name: transfer.name, + amount: outflow_entry.amount, # positive (outflow), per Sure sign convention + currency: outflow_entry.currency, + expected_day_of_month: expected_day, + last_occurrence_date: outflow_entry.date, + next_expected_date: next_expected, + status: "active", + occurrence_count: 1, + manual: true + ) + end + # Create a manual recurring transaction from an existing transaction # Automatically calculates amount variance from past 6 months of matching transactions def self.create_from_transaction(transaction, date_variance: 2) @@ -313,7 +393,10 @@ class RecurringTransaction < ApplicationRecord amount_min: expected_amount_min, amount_max: expected_amount_max, amount_avg: expected_amount_avg, - has_variance: has_amount_variance? + has_variance: has_amount_variance?, + transfer: transfer?, + source_account: account, + destination_account: destination_account ) end diff --git a/app/models/recurring_transaction/cleaner.rb b/app/models/recurring_transaction/cleaner.rb index 4aecbb343..dd22dcb89 100644 --- a/app/models/recurring_transaction/cleaner.rb +++ b/app/models/recurring_transaction/cleaner.rb @@ -7,11 +7,21 @@ class RecurringTransaction end # Mark recurring transactions as inactive if they haven't occurred recently - # Uses 2 months for automatic recurring, 6 months for manual recurring + # Uses 2 months for automatic recurring, 6 months for manual recurring. + # + # Transfer rows (destination_account_id present) are skipped: their + # `matching_transactions` helper looks at single-account name/amount + # which never matches a Transfer pair, so the Cleaner would + # incorrectly mark a still-recurring transfer inactive at the + # 6-month threshold. Issue #1590 tracks pair-detection-aware + # matching for recurring transfers. def cleanup_stale_transactions stale_count = 0 - family.recurring_transactions.active.find_each do |recurring_transaction| + family.recurring_transactions + .active + .where(destination_account_id: nil) + .find_each do |recurring_transaction| next unless recurring_transaction.should_be_inactive? # Determine threshold based on manual flag diff --git a/app/models/recurring_transaction/identifier.rb b/app/models/recurring_transaction/identifier.rb index 82834c26a..bcb6656d4 100644 --- a/app/models/recurring_transaction/identifier.rb +++ b/app/models/recurring_transaction/identifier.rb @@ -10,14 +10,20 @@ class RecurringTransaction def identify_recurring_patterns three_months_ago = 3.months.ago.to_date - # Get all transactions from the last 3 months + # Skip transfer-kind transactions: they're one half of a Transfer pair, so grouping them + # under their single account would produce incoherent recurring "patterns" that don't + # represent the underlying account-pair flow. Recurring transfers are tracked on a + # different shape (RecurringTransaction with destination_account_id). Filtering at the + # SQL level avoids loading and discarding transfer entries for a busy family. entries_with_transactions = family.entries + .joins("INNER JOIN transactions ON transactions.id = entries.entryable_id") .where(entryable_type: "Transaction") .where("entries.date >= ?", three_months_ago) + .where.not("transactions.kind": Transaction::TRANSFER_KINDS) .includes(:entryable) .to_a - # Group by merchant (if present) or name, along with amount (preserve sign) and currency + # Group by merchant (if present) or name, along with amount (preserve sign) and currency. grouped_transactions = entries_with_transactions .select { |entry| entry.entryable.is_a?(Transaction) } .group_by do |entry| @@ -140,9 +146,17 @@ class RecurringTransaction recurring_patterns.size end - # Update variance for existing manual recurring transactions + # Update variance for existing manual recurring transactions. + # + # Transfer rows (destination_account_id present) are skipped: their + # variance / occurrence tracking would need pair-detection across + # both endpoints rather than the single-account name/merchant match + # the helper performs. Issue #1590 tracks the proper Cleaner-aware + # matching for recurring transfers. def update_manual_recurring_transactions(since_date) - family.recurring_transactions.where(manual: true, status: "active").find_each do |recurring| + family.recurring_transactions + .where(manual: true, status: "active", destination_account_id: nil) + .find_each do |recurring| # Find matching transactions in the recent period matching_entries = RecurringTransaction.find_matching_transaction_entries( family: family, diff --git a/app/views/recurring_transactions/_projected_transaction.html.erb b/app/views/recurring_transactions/_projected_transaction.html.erb index 3659e3fe2..bc8778469 100644 --- a/app/views/recurring_transactions/_projected_transaction.html.erb +++ b/app/views/recurring_transactions/_projected_transaction.html.erb @@ -56,6 +56,10 @@
<% display_amount = recurring_transaction.manual? && recurring_transaction.expected_amount_avg.present? ? recurring_transaction.expected_amount_avg : recurring_transaction.amount %> - <%= content_tag :p, format_money(-Money.new(display_amount, recurring_transaction.currency)), class: ["font-medium", "privacy-sensitive", display_amount.negative? ? "text-success" : "text-subdued"] %> + <% if recurring_transaction.transfer? %> + <%= content_tag :p, format_money(Money.new(display_amount.abs, recurring_transaction.currency)), class: [ "font-medium", "privacy-sensitive", "text-secondary" ] %> + <% else %> + <%= content_tag :p, format_money(-Money.new(display_amount, recurring_transaction.currency)), class: ["font-medium", "privacy-sensitive", display_amount.negative? ? "text-success" : "text-subdued"] %> + <% end %>
diff --git a/app/views/recurring_transactions/index.html.erb b/app/views/recurring_transactions/index.html.erb index 7447cb317..96b29d4a1 100644 --- a/app/views/recurring_transactions/index.html.erb +++ b/app/views/recurring_transactions/index.html.erb @@ -128,16 +128,22 @@ <% end %>
- "> - <% if recurring_transaction.manual? && recurring_transaction.has_amount_variance? %> -
"> - ~ - <%= format_money(-recurring_transaction.expected_amount_avg_money) %> -
- <% else %> - <%= format_money(-recurring_transaction.amount_money) %> - <% end %> - + <% if recurring_transaction.transfer? %> + + <%= format_money(recurring_transaction.amount_money.abs) %> + + <% else %> + "> + <% if recurring_transaction.manual? && recurring_transaction.has_amount_variance? %> +
"> + ~ + <%= format_money(-recurring_transaction.expected_amount_avg_money) %> +
+ <% else %> + <%= format_money(-recurring_transaction.amount_money) %> + <% end %> + + <% end %> <%= t("recurring_transactions.day_of_month", day: recurring_transaction.expected_day_of_month) %> diff --git a/app/views/transfers/show.html.erb b/app/views/transfers/show.html.erb index 792899bbb..54705661f 100644 --- a/app/views/transfers/show.html.erb +++ b/app/views/transfers/show.html.erb @@ -73,6 +73,22 @@ <% end %> <% dialog.with_section(title: t(".settings")) do %>
+ <% if @can_mark_as_recurring_transfer %> +
+
+

<%= t(".mark_recurring_title") %>

+

<%= t(".mark_recurring_subtitle") %>

+
+ <%= render DS::Button.new( + text: t(".mark_recurring"), + variant: "outline", + icon: "repeat", + href: mark_as_recurring_transfer_path(@transfer), + method: :post, + frame: "_top" + ) %> +
+ <% end %>

<%= t(".delete_title") %>

diff --git a/config/locales/views/recurring_transactions/en.yml b/config/locales/views/recurring_transactions/en.yml index 504d321d9..5d57c9306 100644 --- a/config/locales/views/recurring_transactions/en.yml +++ b/config/locales/views/recurring_transactions/en.yml @@ -50,3 +50,7 @@ en: inactive: Inactive badges: manual: Manual + transfer_marked_as_recurring: Transfer marked as recurring + transfer_already_exists: A recurring transfer already exists for this account pair + transfer_creation_failed: Failed to create recurring transfer. Please check the transfer details and try again. + transfer_feature_disabled: Recurring transactions are disabled for this family diff --git a/config/locales/views/transfers/en.yml b/config/locales/views/transfers/en.yml index 9116f4f6b..2eda13585 100644 --- a/config/locales/views/transfers/en.yml +++ b/config/locales/views/transfers/en.yml @@ -31,6 +31,10 @@ en: transactions. delete_title: Remove transfer? details: Details + mark_recurring: Mark as recurring + mark_recurring_subtitle: Track this transfer as a recurring pattern in the upcoming + feed and recurring page. + mark_recurring_title: Mark transfer as recurring note_label: Notes note_placeholder: Add a note to this transfer overview: Overview diff --git a/config/routes.rb b/config/routes.rb index 1ab28a7c1..a9fee8dd1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -277,7 +277,11 @@ Rails.application.routes.draw do get :exchange_rate, to: "exchange_rates#show" - resources :transfers, only: %i[new create destroy show update] + resources :transfers, only: %i[new create destroy show update] do + member do + post :mark_as_recurring + end + end resources :imports, only: %i[index new show create update destroy] do member do diff --git a/db/migrate/20260429120000_add_destination_account_id_to_recurring_transactions.rb b/db/migrate/20260429120000_add_destination_account_id_to_recurring_transactions.rb new file mode 100644 index 000000000..ab5df02e3 --- /dev/null +++ b/db/migrate/20260429120000_add_destination_account_id_to_recurring_transactions.rb @@ -0,0 +1,107 @@ +class AddDestinationAccountIdToRecurringTransactions < ActiveRecord::Migration[7.2] + def up + add_reference :recurring_transactions, :destination_account, type: :uuid, null: true, + foreign_key: { to_table: :accounts, on_delete: :cascade } + + # Backfill cascade on the existing account_id FK while we're widening this + # table -- consistent with the cascading direction set by + # 20251030172500_add_cascade_on_account_deletes.rb. Without this, deleting + # an Account that has any recurring_transactions referencing it raises FK + # violations during Family destruction. + remove_foreign_key :recurring_transactions, :accounts, column: :account_id + add_foreign_key :recurring_transactions, :accounts, column: :account_id, on_delete: :cascade + + # Replace the partial unique indexes added by 20260326112218 with TWO + # predicate-partitioned variants: + # + # * non-transfer rows (destination_account_id IS NULL) keep the + # existing 5-column shape so behaviour is unchanged. + # * transfer rows (destination_account_id IS NOT NULL) get a + # 6-column variant that includes destination_account_id. + # + # We can't simply widen the index because Postgres treats NULLs as + # distinct in unique indexes, so two non-transfer rows with the same + # (family, account, merchant/name, amount, currency) but NULL + # destination would no longer collide. + remove_index :recurring_transactions, name: "idx_recurring_txns_acct_merchant", if_exists: true + remove_index :recurring_transactions, name: "idx_recurring_txns_acct_name", if_exists: true + + add_index :recurring_transactions, + [ :family_id, :account_id, :merchant_id, :amount, :currency ], + unique: true, + where: "merchant_id IS NOT NULL AND destination_account_id IS NULL", + name: "idx_recurring_txns_acct_merchant" + + add_index :recurring_transactions, + [ :family_id, :account_id, :name, :amount, :currency ], + unique: true, + where: "name IS NOT NULL AND merchant_id IS NULL AND destination_account_id IS NULL", + name: "idx_recurring_txns_acct_name" + + add_index :recurring_transactions, + [ :family_id, :account_id, :destination_account_id, :merchant_id, :amount, :currency ], + unique: true, + where: "destination_account_id IS NOT NULL AND merchant_id IS NOT NULL", + name: "idx_recurring_txns_pair_merchant" + + add_index :recurring_transactions, + [ :family_id, :account_id, :destination_account_id, :name, :amount, :currency ], + unique: true, + where: "destination_account_id IS NOT NULL AND name IS NOT NULL AND merchant_id IS NULL", + name: "idx_recurring_txns_pair_name" + + # Enforce transfer invariants in the database alongside the model + # validations. Per CLAUDE.md: "Enforce null checks, unique indexes, + # and simple validations in the database schema for PostgreSQL". + add_check_constraint :recurring_transactions, + "destination_account_id IS NULL OR account_id IS NOT NULL", + name: "chk_recurring_txns_transfer_requires_source" + + add_check_constraint :recurring_transactions, + "destination_account_id IS NULL OR destination_account_id <> account_id", + name: "chk_recurring_txns_transfer_distinct_accounts" + end + + def down + # Transfer rows depend on destination_account_id, which the legacy + # schema doesn't model. Re-adding the legacy unique indexes after + # transfer rows exist would violate uniqueness on otherwise-identical + # (family, account, merchant/name, amount, currency) tuples that + # differ only by destination, so we drop transfer rows before + # restoring the old shape. Down migrations are expected to lose + # feature-specific data; the column itself is removed below anyway. + execute <<~SQL + DELETE FROM recurring_transactions + WHERE destination_account_id IS NOT NULL + SQL + + remove_check_constraint :recurring_transactions, name: "chk_recurring_txns_transfer_requires_source" + remove_check_constraint :recurring_transactions, name: "chk_recurring_txns_transfer_distinct_accounts" + + remove_index :recurring_transactions, name: "idx_recurring_txns_pair_merchant", if_exists: true + remove_index :recurring_transactions, name: "idx_recurring_txns_pair_name", if_exists: true + remove_index :recurring_transactions, name: "idx_recurring_txns_acct_merchant", if_exists: true + remove_index :recurring_transactions, name: "idx_recurring_txns_acct_name", if_exists: true + + add_index :recurring_transactions, + [ :family_id, :account_id, :merchant_id, :amount, :currency ], + unique: true, + where: "merchant_id IS NOT NULL", + name: "idx_recurring_txns_acct_merchant" + + add_index :recurring_transactions, + [ :family_id, :account_id, :name, :amount, :currency ], + unique: true, + where: "name IS NOT NULL AND merchant_id IS NULL", + name: "idx_recurring_txns_acct_name" + + remove_foreign_key :recurring_transactions, :accounts, column: :account_id + add_foreign_key :recurring_transactions, :accounts, column: :account_id + + # Drop the destination_account_id FK by referencing the actual `accounts` + # table; Rails would otherwise infer the table as `destination_accounts` + # (the pluralised reference name) and fail with `no foreign key`. + remove_foreign_key :recurring_transactions, column: :destination_account_id + remove_reference :recurring_transactions, :destination_account + end +end diff --git a/db/schema.rb b/db/schema.rb index 1dbf42e98..ecd756f91 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1194,13 +1194,19 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do t.decimal "expected_amount_max", precision: 19, scale: 4 t.decimal "expected_amount_avg", precision: 19, scale: 4 t.uuid "account_id" + t.uuid "destination_account_id" t.index ["account_id"], name: "index_recurring_transactions_on_account_id" - t.index ["family_id", "account_id", "merchant_id", "amount", "currency"], name: "idx_recurring_txns_acct_merchant", unique: true, where: "(merchant_id IS NOT NULL)" - t.index ["family_id", "account_id", "name", "amount", "currency"], name: "idx_recurring_txns_acct_name", unique: true, where: "((name IS NOT NULL) AND (merchant_id IS NULL))" + t.index ["destination_account_id"], name: "index_recurring_transactions_on_destination_account_id" + t.index ["family_id", "account_id", "destination_account_id", "merchant_id", "amount", "currency"], name: "idx_recurring_txns_pair_merchant", unique: true, where: "((destination_account_id IS NOT NULL) AND (merchant_id IS NOT NULL))" + t.index ["family_id", "account_id", "destination_account_id", "name", "amount", "currency"], name: "idx_recurring_txns_pair_name", unique: true, where: "((destination_account_id IS NOT NULL) AND (name IS NOT NULL) AND (merchant_id IS NULL))" + t.index ["family_id", "account_id", "merchant_id", "amount", "currency"], name: "idx_recurring_txns_acct_merchant", unique: true, where: "((merchant_id IS NOT NULL) AND (destination_account_id IS NULL))" + t.index ["family_id", "account_id", "name", "amount", "currency"], name: "idx_recurring_txns_acct_name", unique: true, where: "((name IS NOT NULL) AND (merchant_id IS NULL) AND (destination_account_id IS NULL))" t.index ["family_id", "status"], name: "index_recurring_transactions_on_family_id_and_status" t.index ["family_id"], name: "index_recurring_transactions_on_family_id" t.index ["merchant_id"], name: "index_recurring_transactions_on_merchant_id" t.index ["next_expected_date"], name: "index_recurring_transactions_on_next_expected_date" + t.check_constraint "destination_account_id IS NULL OR account_id IS NOT NULL", name: "chk_recurring_txns_transfer_requires_source" + t.check_constraint "destination_account_id IS NULL OR destination_account_id <> account_id", name: "chk_recurring_txns_transfer_distinct_accounts" end create_table "rejected_transfers", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| @@ -1288,7 +1294,7 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do t.index ["kind"], name: "index_securities_on_kind" t.index ["price_provider", "offline_reason"], name: "index_securities_on_price_provider_and_offline_reason" t.index ["price_provider"], name: "index_securities_on_price_provider" - t.check_constraint "kind::text = ANY (ARRAY['standard'::character varying, 'cash'::character varying]::text[])", name: "chk_securities_kind" + t.check_constraint "kind::text = ANY (ARRAY['standard'::character varying::text, 'cash'::character varying::text])", name: "chk_securities_kind" end create_table "security_prices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| @@ -1447,8 +1453,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do t.datetime "updated_at", null: false t.boolean "manual_sync", default: false, null: false t.index ["account_id"], name: "index_sophtron_accounts_on_account_id" - t.index ["sophtron_item_id"], name: "index_sophtron_accounts_on_sophtron_item_id" t.index ["sophtron_item_id", "account_id"], name: "idx_unique_sophtron_accounts_per_item", unique: true + t.index ["sophtron_item_id"], name: "index_sophtron_accounts_on_sophtron_item_id" end create_table "sophtron_items", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| @@ -1772,7 +1778,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do add_foreign_key "oidc_identities", "users" add_foreign_key "plaid_accounts", "plaid_items" add_foreign_key "plaid_items", "families" - add_foreign_key "recurring_transactions", "accounts" + add_foreign_key "recurring_transactions", "accounts", column: "destination_account_id", on_delete: :cascade + add_foreign_key "recurring_transactions", "accounts", on_delete: :cascade add_foreign_key "recurring_transactions", "families" add_foreign_key "recurring_transactions", "merchants" add_foreign_key "rejected_transfers", "transactions", column: "inflow_transaction_id" diff --git a/test/controllers/transfers_controller_test.rb b/test/controllers/transfers_controller_test.rb index ca42a51fa..544b10057 100644 --- a/test/controllers/transfers_controller_test.rb +++ b/test/controllers/transfers_controller_test.rb @@ -198,4 +198,48 @@ class TransfersControllerTest < ActionDispatch::IntegrationTest transfer.reload end end + + test "mark_as_recurring creates a recurring transfer" do + transfer = transfers(:one) + family = users(:family_admin).family + family.recurring_transactions.destroy_all + + assert_difference -> { RecurringTransaction.where(family: family).count }, +1 do + post mark_as_recurring_transfer_url(transfer) + end + + rt = RecurringTransaction.where(family: family).last + assert rt.transfer? + assert_equal transfer.outflow_transaction.entry.account, rt.account + assert_equal transfer.inflow_transaction.entry.account, rt.destination_account + assert rt.manual? + assert_equal I18n.t("recurring_transactions.transfer_marked_as_recurring"), flash[:notice] + assert_redirected_to transactions_path + end + + test "mark_as_recurring is idempotent: second call flashes already-exists" do + transfer = transfers(:one) + family = users(:family_admin).family + family.recurring_transactions.destroy_all + + post mark_as_recurring_transfer_url(transfer) + assert_equal I18n.t("recurring_transactions.transfer_marked_as_recurring"), flash[:notice] + + assert_no_difference -> { RecurringTransaction.where(family: family).count } do + post mark_as_recurring_transfer_url(transfer) + end + assert_equal I18n.t("recurring_transactions.transfer_already_exists"), flash[:alert] + end + + test "mark_as_recurring is rejected when recurring_transactions_disabled" do + transfer = transfers(:one) + family = users(:family_admin).family + family.update!(recurring_transactions_disabled: true) + family.recurring_transactions.destroy_all + + assert_no_difference -> { RecurringTransaction.where(family: family).count } do + post mark_as_recurring_transfer_url(transfer) + end + assert_equal I18n.t("recurring_transactions.transfer_feature_disabled"), flash[:alert] + end end diff --git a/test/models/recurring_transaction_test.rb b/test/models/recurring_transaction_test.rb index 6c469eb83..20c8a28ae 100644 --- a/test/models/recurring_transaction_test.rb +++ b/test/models/recurring_transaction_test.rb @@ -738,4 +738,326 @@ class RecurringTransactionTest < ActiveSupport::TestCase assert recurring_b.present? assert_not_equal recurring_a, recurring_b end + + # ----- Recurring transfers (issue #895 / discussion #1224) ----- + + test "transfer? is false when destination_account is absent" do + rt = @family.recurring_transactions.create!( + account: @account, + name: "Spotify", + amount: 9.99, + currency: "USD", + expected_day_of_month: 5, + last_occurrence_date: Date.current, + next_expected_date: 5.days.from_now.to_date, + manual: true + ) + assert_not rt.transfer? + end + + test "transfer? is true when destination_account is present" do + destination = accounts(:credit_card) + rt = @family.recurring_transactions.create!( + account: @account, + destination_account: destination, + name: "Transfer to #{destination.name}", + amount: 500, + currency: "USD", + expected_day_of_month: 1, + last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, + manual: true + ) + assert rt.transfer? + end + + test "validation rejects same source and destination accounts" do + rt = @family.recurring_transactions.build( + account: @account, + destination_account: @account, + name: "Self-transfer", + amount: 100, + currency: "USD", + expected_day_of_month: 1, + last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, + manual: true + ) + assert_not rt.valid? + assert_includes rt.errors[:destination_account], "cannot be the same as the source account" + end + + test "validation rejects dangling source account_id (account does not exist)" do + rt = @family.recurring_transactions.build( + account_id: SecureRandom.uuid, # references nothing + destination_account: accounts(:credit_card), + name: "Phantom source", + amount: 100, + currency: "USD", + expected_day_of_month: 1, + last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, + manual: true + ) + assert_not rt.valid? + assert_includes rt.errors[:account], "must exist" + end + + test "validation rejects dangling destination_account_id (account does not exist)" do + rt = @family.recurring_transactions.build( + account: @account, + destination_account_id: SecureRandom.uuid, # references nothing + name: "Phantom transfer", + amount: 100, + currency: "USD", + expected_day_of_month: 1, + last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, + manual: true + ) + assert_not rt.valid? + assert_includes rt.errors[:destination_account], "must exist" + end + + test "validation rejects destination on different family" do + other_family = Family.create!(name: "Other", locale: "en", date_format: "%Y-%m-%d", currency: "USD") + other_account = other_family.accounts.create!(name: "Other depository", balance: 0, currency: "USD", accountable: Depository.new) + + rt = @family.recurring_transactions.build( + account: @account, + destination_account: other_account, + name: "Foreign transfer", + amount: 100, + currency: "USD", + expected_day_of_month: 1, + last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, + manual: true + ) + assert_not rt.valid? + assert_includes rt.errors[:destination_account], "must belong to the same family as the source account" + end + + test "create_from_transfer builds a recurring transfer with both endpoints" do + source = @account + destination = accounts(:credit_card) + + outflow_entry = source.entries.create!( + date: 5.days.ago.to_date, amount: 250, currency: "USD", + name: "Manual transfer", + entryable: Transaction.new(kind: "standard") + ) + inflow_entry = destination.entries.create!( + date: 5.days.ago.to_date, amount: -250, currency: "USD", + name: "Manual transfer", + entryable: Transaction.new(kind: "standard") + ) + transfer = Transfer.create!( + outflow_transaction: outflow_entry.entryable, + inflow_transaction: inflow_entry.entryable + ) + + rt = RecurringTransaction.create_from_transfer(transfer) + + assert rt.transfer? + assert_equal source, rt.account + assert_equal destination, rt.destination_account + assert_equal 250, rt.amount + assert_equal "USD", rt.currency + assert_equal 5.days.ago.to_date.day, rt.expected_day_of_month + assert rt.manual? + assert_equal "active", rt.status + end + + test "projected_entry exposes source and destination on a recurring transfer" do + destination = accounts(:credit_card) + rt = @family.recurring_transactions.create!( + account: @account, + destination_account: destination, + name: "Transfer to #{destination.name}", + amount: 500, + currency: "USD", + expected_day_of_month: 15, + last_occurrence_date: Date.current, + next_expected_date: 15.days.from_now.to_date, + manual: true + ) + + projected = rt.projected_entry + assert projected.transfer + assert_equal @account, projected.source_account + assert_equal destination, projected.destination_account + assert_equal 500, projected.amount + assert_equal "USD", projected.currency + end + + test "Identifier skips transfer-kind transactions" do + # Three depository transactions tagged as funds_movement (e.g. they're + # one half of a Transfer pair). Identifier shouldn't latch onto these + # as a single-account "pattern" because the underlying flow is two- + # account and is tracked on a different shape (destination_account_id). + [ 0, 1, 2 ].each do |months_ago| + transaction = Transaction.create!(merchant: @merchant, kind: "funds_movement") + @account.entries.create!( + date: months_ago.months.ago.beginning_of_month + 5.days, + amount: 50.00, + currency: "USD", + name: "Recurring transfer half", + entryable: transaction + ) + end + + assert_no_difference "@family.recurring_transactions.count" do + RecurringTransaction.identify_patterns_for!(@family) + end + end + + test "Identifier creates a pattern from expense halves while ignoring co-resident transfer halves" do + # Same merchant, amount, day-of-month: 3 standard expenses + 3 transfer halves. + # Without the TRANSFER_KINDS filter, the identifier would either double-count + # (six occurrences) or surface a weird pattern. With the filter, only the + # expense pattern is created. + [ 0, 1, 2 ].each do |months_ago| + base_date = months_ago.months.ago.beginning_of_month + 5.days + + @account.entries.create!( + date: base_date, amount: 50.00, currency: "USD", name: "Coffee", + entryable: Transaction.create!(merchant: @merchant, kind: "standard") + ) + @account.entries.create!( + date: base_date, amount: 50.00, currency: "USD", name: "Half of transfer", + entryable: Transaction.create!(merchant: @merchant, kind: "funds_movement") + ) + end + + assert_difference "@family.recurring_transactions.count", 1 do + RecurringTransaction.identify_patterns_for!(@family) + end + assert_nil @family.recurring_transactions.last.destination_account_id + end + + test "create_from_transfer name reflects Transfer#name (Payment vs Transfer based on destination)" do + # Transfer#name returns "Payment to ..." for liability destinations + # and "Transfer to ..." otherwise, mirroring Transfer::Creator's + # name_prefix logic. The recurring row should pick that up rather + # than hard-coding "Transfer to ...". + source = @account + cc_destination = accounts(:credit_card) # liability + outflow = source.entries.create!( + date: 5.days.ago.to_date, amount: 100, currency: "USD", + name: "raw", entryable: Transaction.new(kind: "standard") + ) + inflow = cc_destination.entries.create!( + date: 5.days.ago.to_date, amount: -100, currency: "USD", + name: "raw", entryable: Transaction.new(kind: "standard") + ) + transfer = Transfer.create!( + outflow_transaction: outflow.entryable, inflow_transaction: inflow.entryable + ) + + rt = RecurringTransaction.create_from_transfer(transfer) + assert_equal "Payment to #{cc_destination.name}", rt.name + end + + test "create_from_transfer stores source-side currency on multi-currency transfers" do + source = @account # USD depository + destination = @family.accounts.create!( + name: "EUR cash", balance: 0, currency: "EUR", accountable: Depository.new + ) + outflow_entry = source.entries.create!( + date: 5.days.ago.to_date, amount: 100, currency: "USD", + name: "FX transfer", entryable: Transaction.new(kind: "standard") + ) + inflow_entry = destination.entries.create!( + date: 5.days.ago.to_date, amount: -92, currency: "EUR", + name: "FX transfer", entryable: Transaction.new(kind: "standard") + ) + transfer = Transfer.create!( + outflow_transaction: outflow_entry.entryable, + inflow_transaction: inflow_entry.entryable + ) + + rt = RecurringTransaction.create_from_transfer(transfer) + assert_equal "USD", rt.currency, "stores source-side currency" + assert_equal 100, rt.amount, "stores source-side amount" + end + + test "destroying the destination account cascades to inbound recurring transfers" do + source = @account + destination = accounts(:credit_card) + rt = @family.recurring_transactions.create!( + account: source, destination_account: destination, + name: "Transfer to CC", amount: 250, currency: "USD", + expected_day_of_month: 1, last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, manual: true + ) + + assert_difference -> { RecurringTransaction.count }, -1 do + destination.destroy + end + assert_not RecurringTransaction.exists?(rt.id) + end + + test "Cleaner skips recurring transfers so they aren't mistakenly marked inactive" do + # `matching_transactions` is single-account name/amount-based and never + # matches a Transfer pair, so without the skip the recurring transfer + # would flip to inactive at the 6-month threshold even when the user + # is still doing the transfer monthly. Issue #1590 tracks the proper + # pair-detection fix. + rt = @family.recurring_transactions.create!( + account: @account, destination_account: accounts(:credit_card), + name: "Transfer to CC", amount: 250, currency: "USD", + expected_day_of_month: 5, + last_occurrence_date: 7.months.ago.to_date, + next_expected_date: 5.days.from_now.to_date, + manual: true + ) + assert rt.should_be_inactive?, "guard sanity: row would be marked inactive without the skip" + + RecurringTransaction.cleanup_stale_for(@family) + assert_equal "active", rt.reload.status + end + + test "Identifier#update_manual_recurring_transactions skips recurring transfers" do + # Same reasoning as the Cleaner skip. Without the guard, the helper + # would call find_matching_transaction_entries (single-account, by + # name) on a transfer row and silently overwrite its variance / + # occurrence_count with []. The variance fields should stay nil. + rt = @family.recurring_transactions.create!( + account: @account, destination_account: accounts(:credit_card), + name: "Transfer to CC", amount: 500, currency: "USD", + expected_day_of_month: 1, + last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, + manual: true, + occurrence_count: 7 + ) + + RecurringTransaction.identify_patterns_for!(@family) + + rt.reload + assert_nil rt.expected_amount_min + assert_nil rt.expected_amount_max + assert_nil rt.expected_amount_avg + assert_equal 7, rt.occurrence_count, "occurrence_count must not be overwritten by the manual-recurring update path" + end + + test "unique partial index still de-duplicates non-transfer recurring rows after destination widening" do + base_attrs = { + account: @account, + merchant: @merchant, + amount: 15.99, + currency: "USD", + expected_day_of_month: 5, + last_occurrence_date: Date.current, + next_expected_date: 1.month.from_now.to_date, + manual: false, + occurrence_count: 3 + } + @family.recurring_transactions.create!(base_attrs) + + assert_raises(ActiveRecord::RecordNotUnique) do + @family.recurring_transactions.create!(base_attrs) + end + end end From c1678181f0e63360787bd3404add3f1d01f739e8 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 15:41:05 -0700 Subject: [PATCH 11/31] fix(imports): import raw balance records (#1724) * fix(imports): import raw balance records * fix(imports): preserve partial balance components --- app/models/family/data_importer.rb | 48 ++++- test/models/family/data_importer_test.rb | 218 +++++++++++++++++++++++ 2 files changed, 264 insertions(+), 2 deletions(-) diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb index 171caf620..7601516ce 100644 --- a/app/models/family/data_importer.rb +++ b/app/models/family/data_importer.rb @@ -1,7 +1,7 @@ require "set" class Family::DataImporter - SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Transfer RejectedTransfer Trade Holding Valuation Budget BudgetCategory Rule].freeze + SUPPORTED_TYPES = %w[Account Balance Category Tag Merchant RecurringTransaction Transaction Transfer RejectedTransfer Trade Holding Valuation Budget BudgetCategory Rule].freeze ACCOUNTABLE_TYPES = Accountable::TYPES.freeze def initialize(family, ndjson_content) @@ -30,6 +30,7 @@ class Family::DataImporter Import.transaction do # Import in dependency order import_accounts(records["Account"] || []) + import_balances(records["Balance"] || []) import_categories(records["Category"] || []) import_tags(records["Tag"] || []) import_merchants(records["Merchant"] || []) @@ -128,6 +129,49 @@ class Family::DataImporter status.to_s.in?(%w[active disabled draft]) ? status.to_s : "active" end + def import_balances(records) + records.each do |record| + data = record["data"] || {} + new_account_id = @id_mappings[:accounts][data["account_id"]] + balance_date = parse_import_date(data["date"]) + next if new_account_id.blank? || balance_date.blank? || data["balance"].blank? + + account = @family.accounts.find(new_account_id) + currency = data["currency"].presence || account.currency + balance = account.balances.find_or_initialize_by(date: balance_date, currency: currency) + + balance.assign_attributes(imported_balance_attributes(data)) + balance.save! + end + end + + def imported_balance_attributes(data) + attributes = { + balance: data["balance"].to_d, + cash_balance: optional_decimal(data["cash_balance"]), + start_cash_balance: optional_decimal(data["start_cash_balance"]), + start_non_cash_balance: optional_decimal(data["start_non_cash_balance"]), + cash_inflows: optional_decimal(data["cash_inflows"]), + cash_outflows: optional_decimal(data["cash_outflows"]), + non_cash_inflows: optional_decimal(data["non_cash_inflows"]), + non_cash_outflows: optional_decimal(data["non_cash_outflows"]), + net_market_flows: optional_decimal(data["net_market_flows"]), + cash_adjustments: optional_decimal(data["cash_adjustments"]), + non_cash_adjustments: optional_decimal(data["non_cash_adjustments"]) + }.compact + + attributes[:flows_factor] = balance_flows_factor_for(data["flows_factor"]) if data["flows_factor"].present? + attributes + end + + def optional_decimal(value) + value.presence&.to_d + end + + def balance_flows_factor_for(value) + value.to_i.in?([ -1, 1 ]) ? value.to_i : 1 + end + def import_categories(records) # First pass: create all categories without parent relationships parent_mappings = {} @@ -472,7 +516,7 @@ class Family::DataImporter # Account-level opening balances must precede every imported account # activity, including standalone valuation snapshots. - %w[Transaction Trade Holding Valuation].each do |type| + %w[Balance Transaction Trade Holding Valuation].each do |type| records[type].to_a.each do |record| data = record["data"] || {} account_id = data["account_id"] diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb index 22691648c..0d8efd0be 100644 --- a/test/models/family/data_importer_test.rb +++ b/test/models/family/data_importer_test.rb @@ -77,6 +77,176 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal "active", account.status end + test "imports raw balance history records" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Balance History Checking", + balance: "1200.00", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-01-31", + balance: "1200.00", + currency: "USD", + cash_balance: "1100.00", + start_cash_balance: "1000.00", + start_non_cash_balance: "0.00", + cash_inflows: "300.00", + cash_outflows: "200.00", + non_cash_inflows: "0.00", + non_cash_outflows: "0.00", + net_market_flows: "0.00", + cash_adjustments: "0.00", + non_cash_adjustments: "0.00", + flows_factor: 1 + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Balance History Checking") + balance = account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + + assert_equal 1200.0, balance.balance.to_f + assert_equal 1100.0, balance.cash_balance.to_f + assert_equal 1000.0, balance.start_cash_balance.to_f + assert_equal 300.0, balance.cash_inflows.to_f + assert_equal 200.0, balance.cash_outflows.to_f + assert_equal 1, balance.flows_factor + end + + test "imports duplicate raw balance records idempotently by account date and currency" do + balance_record = { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-01-31", + balance: "1200.00", + currency: "USD", + cash_balance: "1100.00", + flows_factor: 1 + } + } + + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Idempotent Balance Checking", + balance: "1200.00", + currency: "USD", + accountable_type: "Depository" + } + }, + balance_record, + balance_record.deep_merge(data: { id: "balance-1-duplicate", balance: "1300.00", cash_balance: "1250.00" }) + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Idempotent Balance Checking") + assert_equal 1, account.balances.where(date: Date.parse("2024-01-31"), currency: "USD").count + + balance = account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + assert_equal 1300.0, balance.balance.to_f + assert_equal 1250.0, balance.cash_balance.to_f + end + + test "preserves omitted raw balance components on duplicate records" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Partial Balance Checking", + balance: "1200.00", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-01-31", + balance: "1200.00", + currency: "USD", + cash_balance: "1100.00", + cash_inflows: "300.00", + cash_outflows: "200.00", + flows_factor: -1 + } + }, + { + type: "Balance", + data: { + id: "balance-1-partial", + account_id: "acct-1", + date: "2024-01-31", + balance: "1300.00", + currency: "USD" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Partial Balance Checking") + balance = account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + + assert_equal 1300.0, balance.balance.to_f + assert_equal 1100.0, balance.cash_balance.to_f + assert_equal 300.0, balance.cash_inflows.to_f + assert_equal 200.0, balance.cash_outflows.to_f + assert_equal(-1, balance.flows_factor) + end + + test "dates synthesized account opening balance before imported balance history" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Balance Anchored Checking", + balance: "500.00", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-02-01", + balance: "500.00", + currency: "USD" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Balance Anchored Checking") + opening_anchor = account.valuations.opening_anchor.first + + assert_not_nil opening_anchor + assert_equal Date.parse("2024-01-31"), opening_anchor.entry.date + end + test "dates synthesized account opening balance before oldest imported activity" do ndjson = build_ndjson([ { @@ -798,6 +968,54 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert imported_holding.security_locked end + test "round trips raw balance history through full export" do + source_family = Family.create!( + name: "Source Balance Family", + currency: "USD", + locale: "en", + date_format: "%Y-%m-%d" + ) + source_account = source_family.accounts.create!( + name: "Round Trip Balance Checking", + accountable: Depository.new, + balance: 1_500, + currency: "USD" + ) + source_account.balances.create!( + date: Date.parse("2024-01-31"), + balance: 1_500, + cash_balance: 1_450, + currency: "USD", + start_cash_balance: 1_000, + start_non_cash_balance: 0, + cash_inflows: 700, + cash_outflows: 250, + non_cash_inflows: 0, + non_cash_outflows: 0, + net_market_flows: 0, + cash_adjustments: 0, + non_cash_adjustments: 0, + flows_factor: 1 + ) + + zip_data = Family::DataExporter.new(source_family).generate_export + ndjson = nil + Zip::File.open_buffer(zip_data) do |zip| + ndjson = zip.read("all.ndjson") + end + + Family::DataImporter.new(@family, ndjson).import! + + imported_account = @family.accounts.find_by!(name: "Round Trip Balance Checking") + imported_balance = imported_account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + + assert_equal 1500.0, imported_balance.balance.to_f + assert_equal 1450.0, imported_balance.cash_balance.to_f + assert_equal 1000.0, imported_balance.start_cash_balance.to_f + assert_equal 700.0, imported_balance.cash_inflows.to_f + assert_equal 250.0, imported_balance.cash_outflows.to_f + end + test "imports holding snapshots with ticker fallback when exchange mic is missing" do existing_security = Security.create!( ticker: "VTI", From d943e32b1522bc22bda32fd23e9f667c05a0632b Mon Sep 17 00:00:00 2001 From: "Sure Admin (bot)" Date: Tue, 12 May 2026 00:55:46 +0200 Subject: [PATCH 12/31] fix: correct SnapTrade cash activity signs (#1634) * fix: correct snaptrade cash activity signs * test: update snaptrade withdrawal sign expectation --------- Co-authored-by: SureBot --- .../snaptrade_account/activities_processor.rb | 4 +- .../activities_processor_test.rb | 42 ++++++++++++++++--- .../snaptrade_account_processor_test.rb | 4 +- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/app/models/snaptrade_account/activities_processor.rb b/app/models/snaptrade_account/activities_processor.rb index 141a24496..96b35cbb4 100644 --- a/app/models/snaptrade_account/activities_processor.rb +++ b/app/models/snaptrade_account/activities_processor.rb @@ -247,9 +247,9 @@ class SnaptradeAccount::ActivitiesProcessor def normalize_cash_amount(amount, activity_type) case activity_type when "WITHDRAWAL", "TRANSFER_OUT", "FEE", "TAX" - -amount.abs # These should be negative (money out) + amount.abs # Money out should be positive in Sure when "CONTRIBUTION", "TRANSFER_IN", "DIVIDEND", "DIV", "INTEREST", "CASH" - amount.abs # These should be positive (money in) + -amount.abs # Money in should be negative in Sure else amount end diff --git a/test/models/snaptrade_account/activities_processor_test.rb b/test/models/snaptrade_account/activities_processor_test.rb index dedde9de5..2c5639952 100644 --- a/test/models/snaptrade_account/activities_processor_test.rb +++ b/test/models/snaptrade_account/activities_processor_test.rb @@ -70,7 +70,7 @@ class SnaptradeAccount::ActivitiesProcessorTest < ActiveSupport::TestCase assert_equal "Sell", trade.investment_activity_label end - test "processes dividend cash activity" do + test "processes dividend cash activity as negative inflow" do @snaptrade_account.update!(raw_activities_payload: [ build_cash_activity( id: "div_001", @@ -89,10 +89,11 @@ class SnaptradeAccount::ActivitiesProcessorTest < ActiveSupport::TestCase assert entry.entryable.is_a?(Transaction), "Entry should be a Transaction" transaction = entry.entryable + assert_equal(-25.50, entry.amount.to_f) assert_equal "Dividend", transaction.investment_activity_label end - test "processes contribution with positive amount" do + test "processes contribution with negative inflow amount" do @snaptrade_account.update!(raw_activities_payload: [ build_cash_activity( id: "contrib_001", @@ -107,12 +108,11 @@ class SnaptradeAccount::ActivitiesProcessorTest < ActiveSupport::TestCase entry = @account.entries.find_by(external_id: "contrib_001", source: "snaptrade") assert_not_nil entry - # Amount is on entry, not transaction - assert_equal 500.00, entry.amount.to_f # Positive for contributions + assert_equal(-500.00, entry.amount.to_f) assert_equal "Contribution", entry.entryable.investment_activity_label end - test "processes withdrawal with negative amount" do + test "processes withdrawal with positive outflow amount" do @snaptrade_account.update!(raw_activities_payload: [ build_cash_activity( id: "withdraw_001", @@ -127,10 +127,40 @@ class SnaptradeAccount::ActivitiesProcessorTest < ActiveSupport::TestCase entry = @account.entries.find_by(external_id: "withdraw_001", source: "snaptrade") assert_not_nil entry - assert_equal(-200.00, entry.amount.to_f) # Negative for withdrawals + assert_equal 200.00, entry.amount.to_f assert_equal "Withdrawal", entry.entryable.investment_activity_label end + test "processes transfers with Sure sign convention" do + @snaptrade_account.update!(raw_activities_payload: [ + build_cash_activity( + id: "transfer_in_001", + type: "TRANSFER_IN", + amount: 300.00, + settlement_date: Date.current.to_s + ), + build_cash_activity( + id: "transfer_out_001", + type: "TRANSFER_OUT", + amount: 125.00, + settlement_date: Date.current.to_s + ) + ]) + + processor = SnaptradeAccount::ActivitiesProcessor.new(@snaptrade_account) + processor.process + + transfer_in = @account.entries.find_by(external_id: "transfer_in_001", source: "snaptrade") + transfer_out = @account.entries.find_by(external_id: "transfer_out_001", source: "snaptrade") + + assert_not_nil transfer_in + assert_not_nil transfer_out + assert_equal(-300.00, transfer_in.amount.to_f) + assert_equal 125.00, transfer_out.amount.to_f + assert_equal "Transfer", transfer_in.entryable.investment_activity_label + assert_equal "Transfer", transfer_out.entryable.investment_activity_label + end + test "maps all known activity types correctly" do type_mappings = { "BUY" => "Buy", diff --git a/test/models/snaptrade_account_processor_test.rb b/test/models/snaptrade_account_processor_test.rb index c2127c0e2..154b0690c 100644 --- a/test/models/snaptrade_account_processor_test.rb +++ b/test/models/snaptrade_account_processor_test.rb @@ -210,7 +210,7 @@ class SnaptradeAccountProcessorTest < ActiveSupport::TestCase assert_equal "Dividend", tx_entry.entryable.investment_activity_label end - test "activities processor normalizes withdrawal as negative amount" do + test "activities processor normalizes withdrawal as positive outflow amount" do @snaptrade_account.update!( raw_activities_payload: [ { @@ -228,7 +228,7 @@ class SnaptradeAccountProcessorTest < ActiveSupport::TestCase assert_equal 1, result[:transactions] tx_entry = @account.entries.find_by(external_id: "activity_withdraw_1") - assert tx_entry.amount.negative? + assert_equal 1000.00, tx_entry.amount.to_f end test "activities processor skips activities without external_id" do From 5ceb55be036313778f75701e21173df4667fc200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Tue, 12 May 2026 12:17:00 +0200 Subject: [PATCH 13/31] Scope SnapTrade orphan cleanup to current family (#1769) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Scope SnapTrade orphan cleanup to current family Restrict orphaned user listing and deletion to SnapTrade user IDs that belong to the current family namespace. Add model tests to prevent cross-family enumeration/deletion regressions. * Update test/models/snaptrade_item_test.rb Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Juan José Mata * test: fix snaptrade orphaned users assertion * style: fix snaptrade test array spacing --------- Signed-off-by: Juan José Mata Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: KiloClaw --- app/models/snaptrade_item/provided.rb | 3 ++- test/models/snaptrade_item_test.rb | 38 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/app/models/snaptrade_item/provided.rb b/app/models/snaptrade_item/provided.rb index 5ffb43906..c8d104f43 100644 --- a/app/models/snaptrade_item/provided.rb +++ b/app/models/snaptrade_item/provided.rb @@ -160,13 +160,14 @@ module SnaptradeItem::Provided return [] unless credentials_configured? && user_registered? all_users = list_all_users - all_users.reject { |uid| uid == snaptrade_user_id } + all_users.select { |uid| uid != snaptrade_user_id && uid.start_with?("family_#{family_id}_") } end # Delete an orphaned SnapTrade user and all their connections def delete_orphaned_user(user_id) return false unless credentials_configured? return false if user_id == snaptrade_user_id # Don't delete current user + return false unless user_id.start_with?("family_#{family_id}_") snaptrade_provider.delete_user(user_id: user_id) true diff --git a/test/models/snaptrade_item_test.rb b/test/models/snaptrade_item_test.rb index 79e6c6689..6b31c9895 100644 --- a/test/models/snaptrade_item_test.rb +++ b/test/models/snaptrade_item_test.rb @@ -75,4 +75,42 @@ class SnaptradeItemTest < ActiveSupport::TestCase provider = item.snaptrade_provider assert_instance_of Provider::Snaptrade, provider end + + test "orphaned_users only includes users for the same family" do + item = SnaptradeItem.new( + family: @family, + name: "Test", + client_id: "test", + consumer_key: "test", + snaptrade_user_id: "family_#{@family.id}_111", + snaptrade_user_secret: "secret" + ) + + item.stubs(:list_all_users).returns([ + "family_#{@family.id}_111", + "family_#{@family.id}_222", + "family_999_333", + "legacy_user_444" + ]) + + assert_equal([ "family_#{@family.id}_222" ], item.orphaned_users) + end + + test "delete_orphaned_user rejects users outside the current family namespace" do + item = SnaptradeItem.new( + family: @family, + name: "Test", + client_id: "test", + consumer_key: "test", + snaptrade_user_id: "family_#{@family.id}_111", + snaptrade_user_secret: "secret" + ) + + provider = mock + provider.expects(:delete_user).never + item.stubs(:snaptrade_provider).returns(provider) + + assert_not item.delete_orphaned_user("family_999_222") + assert_not item.delete_orphaned_user("legacy_user_333") + end end From 73b6077ac386c7ae7fd87a82bcb1cfc81bf27b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Tue, 12 May 2026 12:18:17 +0200 Subject: [PATCH 14/31] Constrain Lunchflow base URL to trusted endpoint (#1768) * Constrain Lunchflow base URL to trusted endpoint Prevent SSRF by ignoring user-provided Lunchflow base_url values unless they match the canonical Lunchflow HTTPS endpoint. Add model tests covering invalid host/scheme and valid canonicalization behavior. * Linter --- app/models/lunchflow_item.rb | 15 ++++++++++++++- test/models/lunchflow_item_test.rb | 31 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 test/models/lunchflow_item_test.rb diff --git a/app/models/lunchflow_item.rb b/app/models/lunchflow_item.rb index ba9830c6b..94cf078e7 100644 --- a/app/models/lunchflow_item.rb +++ b/app/models/lunchflow_item.rb @@ -1,6 +1,8 @@ class LunchflowItem < ApplicationRecord include Syncable, Provided, Unlinking, Encryptable + DEFAULT_BASE_URL = "https://lunchflow.app/api/v1".freeze + enum :status, { good: "good", requires_update: "requires_update" }, default: :good # Encrypt sensitive credentials and raw payloads if ActiveRecord encryption is configured @@ -154,6 +156,17 @@ class LunchflowItem < ApplicationRecord end def effective_base_url - base_url.presence || "https://lunchflow.app/api/v1" + return DEFAULT_BASE_URL if base_url.blank? + + uri = URI.parse(base_url) + return DEFAULT_BASE_URL unless uri.is_a?(URI::HTTPS) + return DEFAULT_BASE_URL unless uri.host == "lunchflow.app" + return DEFAULT_BASE_URL unless [ "", "/", "/api/v1", "/api/v1/" ].include?(uri.path) + return DEFAULT_BASE_URL unless uri.query.blank? + return DEFAULT_BASE_URL unless uri.fragment.blank? + + DEFAULT_BASE_URL + rescue URI::InvalidURIError + DEFAULT_BASE_URL end end diff --git a/test/models/lunchflow_item_test.rb b/test/models/lunchflow_item_test.rb new file mode 100644 index 000000000..13b06f2fd --- /dev/null +++ b/test/models/lunchflow_item_test.rb @@ -0,0 +1,31 @@ +require "test_helper" + +class LunchflowItemTest < ActiveSupport::TestCase + def setup + @lunchflow_item = lunchflow_items(:one) + end + + test "effective_base_url returns default when base_url blank" do + @lunchflow_item.base_url = nil + + assert_equal LunchflowItem::DEFAULT_BASE_URL, @lunchflow_item.effective_base_url + end + + test "effective_base_url returns default for non-lunchflow host" do + @lunchflow_item.base_url = "https://169.254.169.254/latest/meta-data" + + assert_equal LunchflowItem::DEFAULT_BASE_URL, @lunchflow_item.effective_base_url + end + + test "effective_base_url returns default for non-https scheme" do + @lunchflow_item.base_url = "http://lunchflow.app/api/v1" + + assert_equal LunchflowItem::DEFAULT_BASE_URL, @lunchflow_item.effective_base_url + end + + test "effective_base_url returns canonical default for valid lunchflow url" do + @lunchflow_item.base_url = "https://lunchflow.app/api/v1/" + + assert_equal LunchflowItem::DEFAULT_BASE_URL, @lunchflow_item.effective_base_url + end +end From fdffcd0dfdba7f8e53b0bc44064951fa3d267c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Tue, 12 May 2026 16:08:31 +0200 Subject: [PATCH 15/31] Add libvips to devcontainer Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan José Mata --- .devcontainer/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b871287d4..3d4e5bf25 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update -qq \ libyaml-dev \ libyaml-0-2 \ openssh-client \ + libvips \ postgresql-client \ vim \ procps \ From 12d799e0b87f0fc0a4e497970954091432617c17 Mon Sep 17 00:00:00 2001 From: plind <59729252+plind-junior@users.noreply.github.com> Date: Tue, 12 May 2026 10:41:58 -0700 Subject: [PATCH 16/31] fix(binance): support CRYPTO: prefix and USD stablecoins (#1771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(binance): support CRYPTO: prefix and USD stablecoins Holdings processors (CoinStats, Coinbase, Kraken, SimpleFIN, Lunchflow, Binance) store crypto securities with a "CRYPTO:" prefix, but Provider::BinancePublic#parse_ticker only accepted Binance-search-style tickers like "BTCUSD". As a result, every fetched price for tickers like CRYPTO:USDT, CRYPTO:USDC, CRYPTO:SOL, CRYPTO:TRUMP, CRYPTO:KAITO failed with "Unsupported Binance ticker". - Strip the CRYPTO: prefix in parse_ticker. - Short-circuit USD-pegged stablecoins (USDT, USDC, BUSD, DAI, FDUSD, TUSD, USDP, PYUSD) to a synthetic flat 1.0 USD price. Binance has no self-pair (USDTUSDT is invalid), and the few stablecoin/USDT pairs that do exist hover at ~1.0 with sub-cent noise. - Default prefixed bare base assets (CRYPTO:SOL etc.) to the …USDT pair (USD). Only when prefixed, so unprefixed garbage like BTCBNB / BTCGBP still returns nil and the existing rejection tests still pass. - fetch_security_info returns links: nil for stablecoins rather than a broken /trade/ URL. Closes #1441. * fix(binance): strip CRYPTO: prefix in search_securities Security::Resolver calls search_provider with the raw holdings-processor symbol (CRYPTO:SOL, CRYPTO:USDT) before any price fetch. Without prefix handling here, first-time crypto imports never resolve to an online Binance security and the new stablecoin/prefix paths in parse_ticker were unreachable for that flow. - Strip CRYPTO: from the search query. - Short-circuit USD stablecoins to a synthetic search result (no exchangeInfo call, no Binance self-pair to find). - Teach parse_ticker the "{stablecoin}USD" form produced by the synthetic result so price fetches route to stablecoin_prices. --------- Co-authored-by: plind-junior --- app/models/provider/binance_public.rb | 90 +++++++++++++-- test/models/provider/binance_public_test.rb | 121 ++++++++++++++++++++ 2 files changed, 204 insertions(+), 7 deletions(-) diff --git a/app/models/provider/binance_public.rb b/app/models/provider/binance_public.rb index f335030a9..cc4a71f7c 100644 --- a/app/models/provider/binance_public.rb +++ b/app/models/provider/binance_public.rb @@ -35,6 +35,16 @@ class Provider::BinancePublic < Provider MS_PER_DAY = 24 * 60 * 60 * 1000 SEARCH_LIMIT = 25 + # USD-pegged stablecoins. Binance has no self-pair (USDTUSDT is invalid) and + # the few stablecoin/USDT pairs that do exist (USDCUSDT, etc.) hover at ~1.0 + # with sub-cent noise — synthesizing a flat 1.0 USD price is both accurate + # enough and avoids surfacing transient depeg ticks from market data. + USD_STABLECOINS = %w[USDT USDC BUSD DAI FDUSD TUSD USDP PYUSD].freeze + + # Symbol prefix applied by holdings processors (CoinStats, Coinbase, Kraken, + # Binance, SimpleFIN, Lunchflow) to distinguish crypto from stock tickers. + CRYPTO_PREFIX = "CRYPTO:".freeze + def initialize # No API key required — public market data only. end @@ -58,9 +68,13 @@ class Provider::BinancePublic < Provider def search_securities(symbol, country_code: nil, exchange_operating_mic: nil) with_provider_response do - query = symbol.to_s.strip.upcase + query = symbol.to_s.strip.upcase.delete_prefix(CRYPTO_PREFIX) next [] if query.empty? + if USD_STABLECOINS.include?(query) + next [ stablecoin_search_result(query) ] + end + symbols = exchange_info_symbols matches = symbols.select do |s| @@ -128,10 +142,12 @@ class Provider::BinancePublic < Provider # logo_url is intentionally nil — crypto logos are set at save time by # Security#generate_logo_url_from_brandfetch via the /crypto/{base} # route, not returned from this provider. + links = parsed[:binance_pair] ? "https://www.binance.com/en/trade/#{parsed[:binance_pair]}" : nil + SecurityInfo.new( symbol: symbol, name: parsed[:base], - links: "https://www.binance.com/en/trade/#{parsed[:binance_pair]}", + links: links, logo_url: nil, description: nil, kind: "crypto", @@ -161,6 +177,10 @@ class Provider::BinancePublic < Provider parsed = parse_ticker(symbol) raise InvalidSecurityPriceError, "Unsupported Binance ticker: #{symbol}" if parsed.nil? + if parsed[:stablecoin] + next stablecoin_prices(symbol, parsed, start_date, end_date, exchange_operating_mic) + end + binance_pair = parsed[:binance_pair] display_currency = parsed[:display_currency] prices = [] @@ -220,6 +240,36 @@ class Provider::BinancePublic < Provider end private + # Synthetic search hit for a USD-pegged stablecoin. Binance has no self-pair + # (USDTUSDT etc. don't exist), so we manufacture a result instead of letting + # the resolver fall back to an offline CRYPTO:* row. The downstream price + # path short-circuits via parse_ticker -> stablecoin_prices. + def stablecoin_search_result(base) + Security.new( + symbol: "#{base}USD", + name: base, + logo_url: ::Security.brandfetch_crypto_url(base), + exchange_operating_mic: BINANCE_MIC, + country_code: nil, + currency: "USD" + ) + end + + # Synthesize flat 1.0 USD prices for USD-pegged stablecoins across the + # requested range. Avoids a Binance round-trip (there is no self-pair like + # USDTUSDT) and produces stable values for portfolio aggregation. + def stablecoin_prices(symbol, parsed, start_date, end_date, exchange_operating_mic) + (start_date..end_date).map do |date| + Price.new( + symbol: symbol, + date: date, + price: 1.0, + currency: parsed[:display_currency], + exchange_operating_mic: exchange_operating_mic + ) + end + end + def base_url ENV["BINANCE_PUBLIC_URL"] || "https://data-api.binance.vision" end @@ -247,11 +297,24 @@ class Provider::BinancePublic < Provider end end - # Maps a user-visible ticker (e.g. "BTCUSD", "ETHEUR") to the Binance pair - # symbol, base asset, and display currency. Returns nil if the ticker does - # not end with a supported quote currency. + # Maps a user-visible ticker to the Binance pair symbol, base asset, and + # display currency. Accepts: + # - "BTCUSD"/"ETHEUR" — fiat suffix from search_securities output + # - "CRYPTO:BTCUSD" — prefixed form stored by holdings processors + # - "CRYPTO:SOL"/"SOL" — bare base asset; defaults to the USDT pair (USD) + # - "CRYPTO:USDT"/"USDT" — USD-pegged stablecoin; binance_pair is nil and + # callers short-circuit to a synthetic 1.0 USD price + # Returns nil only when the input is empty after stripping the prefix. def parse_ticker(ticker) - ticker_up = ticker.to_s.upcase + raw = ticker.to_s.upcase + prefixed = raw.start_with?(CRYPTO_PREFIX) + ticker_up = raw.delete_prefix(CRYPTO_PREFIX) + return nil if ticker_up.empty? + + if USD_STABLECOINS.include?(ticker_up) + return { binance_pair: nil, base: ticker_up, display_currency: "USD", stablecoin: true } + end + SUPPORTED_QUOTES.each do |quote| display_currency = QUOTE_TO_CURRENCY[quote] next unless ticker_up.end_with?(display_currency) @@ -259,9 +322,22 @@ class Provider::BinancePublic < Provider base = ticker_up.delete_suffix(display_currency) next if base.empty? + # "{stablecoin}USD" form (e.g. "USDTUSD" produced by search_securities) + # routes to synthetic 1.0 USD pricing — there is no Binance self-pair. + if display_currency == "USD" && USD_STABLECOINS.include?(base) + return { binance_pair: nil, base: base, display_currency: "USD", stablecoin: true } + end + return { binance_pair: "#{base}#{quote}", base: base, display_currency: display_currency } end - nil + + # No fiat suffix matched. Only treat the input as a bare base asset when + # it arrived with the CRYPTO: prefix from a holdings processor — that + # tells us it really is a single coin symbol (SOL, TRUMP, KAITO), not a + # malformed pair like "BTCBNB" or "BTCGBP" that we want to reject. + return nil unless prefixed + + { binance_pair: "#{ticker_up}USDT", base: ticker_up, display_currency: "USD" } end # Cached for 24h — exchangeInfo returns the full symbol universe (thousands diff --git a/test/models/provider/binance_public_test.rb b/test/models/provider/binance_public_test.rb index 340e4cc83..bf0476043 100644 --- a/test/models/provider/binance_public_test.rb +++ b/test/models/provider/binance_public_test.rb @@ -68,6 +68,38 @@ class Provider::BinancePublicTest < ActiveSupport::TestCase assert_equal [ "BTCUSD" ], response.data.map(&:symbol) end + test "search_securities strips CRYPTO: prefix from holdings-processor symbols" do + @provider.stubs(:exchange_info_symbols).returns(sample_exchange_info) + + response = @provider.search_securities("CRYPTO:BTC") + + assert response.success? + assert_includes response.data.map(&:symbol), "BTCUSD" + end + + test "search_securities returns a synthetic stablecoin result without hitting exchangeInfo" do + @provider.expects(:exchange_info_symbols).never + + response = @provider.search_securities("CRYPTO:USDT") + + assert response.success? + assert_equal 1, response.data.size + row = response.data.first + assert_equal "USDTUSD", row.symbol + assert_equal "USDT", row.name + assert_equal "USD", row.currency + assert_equal "BNCX", row.exchange_operating_mic + assert_nil row.country_code + end + + test "parse_ticker treats stablecoin/USD search-result form as stablecoin" do + parsed = @provider.send(:parse_ticker, "USDTUSD") + assert parsed[:stablecoin] + assert_nil parsed[:binance_pair] + assert_equal "USDT", parsed[:base] + assert_equal "USD", parsed[:display_currency] + end + test "search_securities returns empty array when query does not match" do @provider.stubs(:exchange_info_symbols).returns(sample_exchange_info) @@ -162,6 +194,95 @@ class Provider::BinancePublicTest < ActiveSupport::TestCase assert_nil @provider.send(:parse_ticker, "GIBBERISH") end + test "parse_ticker strips CRYPTO: prefix from holdings processors" do + parsed = @provider.send(:parse_ticker, "CRYPTO:BTCUSD") + assert_equal "BTCUSDT", parsed[:binance_pair] + assert_equal "BTC", parsed[:base] + assert_equal "USD", parsed[:display_currency] + end + + test "parse_ticker flags USD stablecoins for synthetic pricing" do + %w[USDT USDC BUSD DAI FDUSD TUSD USDP PYUSD].each do |stable| + parsed = @provider.send(:parse_ticker, "CRYPTO:#{stable}") + assert parsed[:stablecoin], "expected #{stable} to be flagged as stablecoin" + assert_nil parsed[:binance_pair] + assert_equal stable, parsed[:base] + assert_equal "USD", parsed[:display_currency] + end + end + + test "parse_ticker defaults prefixed bare base assets to the USDT pair" do + parsed = @provider.send(:parse_ticker, "CRYPTO:SOL") + assert_equal "SOLUSDT", parsed[:binance_pair] + assert_equal "SOL", parsed[:base] + assert_equal "USD", parsed[:display_currency] + end + + test "parse_ticker still rejects unprefixed malformed tickers" do + # No CRYPTO: prefix → behaves like a Binance-search ticker (must end in a + # supported fiat). Protects against false defaults like "BTCBNB" → "BTCBNBUSDT". + assert_nil @provider.send(:parse_ticker, "SOL") + assert_nil @provider.send(:parse_ticker, "BTCBNB") + end + + test "fetch_security_prices returns synthetic 1.0 USD prices for stablecoins" do + # No HTTP call expected — short-circuited entirely. + @provider.expects(:client).never + + response = @provider.fetch_security_prices( + symbol: "CRYPTO:USDT", + exchange_operating_mic: "BNCX", + start_date: Date.parse("2026-01-01"), + end_date: Date.parse("2026-01-03") + ) + + assert response.success? + assert_equal 3, response.data.size + assert response.data.all? { |p| p.price == 1.0 && p.currency == "USD" } + assert_equal Date.parse("2026-01-01"), response.data.first.date + assert_equal Date.parse("2026-01-03"), response.data.last.date + end + + test "fetch_security_price returns 1.0 USD for a stablecoin single day" do + @provider.expects(:client).never + + response = @provider.fetch_security_price( + symbol: "CRYPTO:USDT", + exchange_operating_mic: "BNCX", + date: Date.parse("2026-01-15") + ) + + assert response.success? + assert_equal 1.0, response.data.price + assert_equal "USD", response.data.currency + end + + test "fetch_security_info handles stablecoin (no Binance pair link)" do + response = @provider.fetch_security_info(symbol: "CRYPTO:USDT", exchange_operating_mic: "BNCX") + + assert response.success? + assert_equal "USDT", response.data.name + assert_equal "crypto", response.data.kind + assert_nil response.data.links + end + + test "fetch_security_prices resolves a bare CRYPTO: ticker against the USDT pair" do + rows = [ kline_row("2026-01-15", "150.25") ] + mock_client_returning_klines(rows) + + response = @provider.fetch_security_prices( + symbol: "CRYPTO:SOL", + exchange_operating_mic: "BNCX", + start_date: Date.parse("2026-01-15"), + end_date: Date.parse("2026-01-15") + ) + + assert response.success? + assert_equal 1, response.data.size + assert_equal "USD", response.data.first.currency + assert_in_delta 150.25, response.data.first.price + end + # ================================ # Single price # ================================ From 04b0122dbfe1f3a53a2d594ef2c6cf2f93601d1f Mon Sep 17 00:00:00 2001 From: Tristan the Katana <50181095+felixmuinde@users.noreply.github.com> Date: Tue, 12 May 2026 20:57:13 +0300 Subject: [PATCH 17/31] feat(mobile): add suggested questions to empty chat screen (#1773) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(mobile): add suggested questions to empty chat screen - New constants file (lib/constants/suggested_questions.dart) for the 4 suggested question chips, kept separate from screen logic with a clear l10n upgrade path noted in comments - Empty chat screen now shows a personalised greeting and tappable OutlinedButton chips; tapping one pre-fills and sends the message - Optimistic message insertion in ChatProvider.sendMessage so the user message and typing indicator appear instantly on tap, with rollback on failure - Full AI response revealed only once polling detects stable content (2 consecutive polls with no growth), preventing partial responses from flashing on screen - fetchChat stops any in-progress polling before fetching so a manual refresh always shows the authoritative server response - Fixed updateChatTitle silently wiping messages when the title-update API response omits the messages array Co-Authored-By: Claude Sonnet 4.6 * fix(mobile): address PR review comments - Extract _rollbackOptimisticMessage helper to eliminate duplicated rollback logic in sendMessage failure and catch branches - Replace raw 'Error: ${e.toString()}' user-facing strings with a generic message; retain technical details via debugPrint in each catch block - Replace inline ternary in updateChatTitle with explicit if/else for readability while preserving message-preservation behaviour - Fix non-reactive AuthProvider read inside Consumer builder (listen: false → listen: true) so greeting updates when user's firstName changes Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- mobile/lib/constants/suggested_questions.dart | 14 +++ mobile/lib/providers/chat_provider.dart | 108 +++++++++++++---- .../lib/screens/chat_conversation_screen.dart | 112 +++++++++++++++--- 3 files changed, 194 insertions(+), 40 deletions(-) create mode 100644 mobile/lib/constants/suggested_questions.dart diff --git a/mobile/lib/constants/suggested_questions.dart b/mobile/lib/constants/suggested_questions.dart new file mode 100644 index 000000000..818be7652 --- /dev/null +++ b/mobile/lib/constants/suggested_questions.dart @@ -0,0 +1,14 @@ +import 'package:flutter/material.dart'; + +/// Suggested questions shown on the empty chat screen. +/// +/// l10n upgrade path: when Flutter localisation is added to the mobile app, +/// replace this const list with a function that accepts [BuildContext] and +/// returns localised strings via AppLocalizations. The call site in +/// _EmptyState requires only a one-line change. +const List<({IconData icon, String text})> suggestedQuestions = [ + (icon: Icons.account_balance_wallet_outlined, text: 'What is my current net worth?'), + (icon: Icons.show_chart, text: 'How has my spending changed this month?'), + (icon: Icons.savings_outlined, text: 'How can I improve my savings rate?'), + (icon: Icons.receipt_long_outlined, text: 'What are my biggest expenses lately?'), +]; diff --git a/mobile/lib/providers/chat_provider.dart b/mobile/lib/providers/chat_provider.dart index d68caac28..317535fdb 100644 --- a/mobile/lib/providers/chat_provider.dart +++ b/mobile/lib/providers/chat_provider.dart @@ -23,6 +23,11 @@ class ChatProvider with ChangeNotifier { /// Used to detect when the LLM has finished writing (no growth between polls). int? _lastAssistantContentLength; + /// Number of consecutive polls with no content growth. + /// Requires 2 consecutive stable polls before declaring the response complete, + /// to avoid prematurely stopping on a brief server-side generation pause. + int _stablePollingCount = 0; + List get chats => _chats; Chat? get currentChat => _currentChat; bool get isLoading => _isLoading; @@ -55,7 +60,8 @@ class ChatProvider with ChangeNotifier { _errorMessage = result['error'] ?? 'Failed to fetch chats'; } } catch (e) { - _errorMessage = 'Error: ${e.toString()}'; + debugPrint('fetchChats error: $e'); + _errorMessage = 'Something went wrong. Please try again.'; } finally { _isLoading = false; notifyListeners(); @@ -67,6 +73,10 @@ class ChatProvider with ChangeNotifier { required String accessToken, required String chatId, }) async { + // Stop any in-progress polling — the server response is the source of truth + // when explicitly fetching a chat. This prevents a stale poll from + // overwriting the freshly fetched data and ensures the message filter lifts. + _stopPolling(); _isLoading = true; _errorMessage = null; notifyListeners(); @@ -84,7 +94,8 @@ class ChatProvider with ChangeNotifier { _errorMessage = result['error'] ?? 'Failed to fetch chat'; } } catch (e) { - _errorMessage = 'Error: ${e.toString()}'; + debugPrint('fetchChat error: $e'); + _errorMessage = 'Something went wrong. Please try again.'; } finally { _isLoading = false; notifyListeners(); @@ -142,13 +153,25 @@ class ChatProvider with ChangeNotifier { return null; } } catch (e) { - _errorMessage = 'Error: ${e.toString()}'; + debugPrint('createChat error: $e'); + _errorMessage = 'Something went wrong. Please try again.'; _isLoading = false; notifyListeners(); return null; } } + void _rollbackOptimisticMessage(String optimisticId, String chatId) { + if (_currentChat != null && _currentChat!.id == chatId) { + _currentChat = _currentChat!.copyWith( + messages: _currentChat!.messages + .where((m) => m.id != optimisticId) + .toList(), + ); + } + _isWaitingForResponse = false; + } + /// Send a message to the current chat. /// Returns true if delivery succeeded, false otherwise. Future sendMessage({ @@ -158,6 +181,26 @@ class ChatProvider with ChangeNotifier { }) async { _isSendingMessage = true; _errorMessage = null; + + // Optimistically add the user message so it appears immediately — before + // the network round-trip completes. This makes the empty-state disappear + // and the typing indicator show at the same instant. + final now = DateTime.now(); + final optimisticId = 'pending-${now.millisecondsSinceEpoch}'; + final optimisticMessage = Message( + id: optimisticId, + type: 'text', + role: 'user', + content: content, + createdAt: now, + updatedAt: now, + ); + if (_currentChat != null && _currentChat!.id == chatId) { + _currentChat = _currentChat!.copyWith( + messages: [..._currentChat!.messages, optimisticMessage], + ); + } + _isWaitingForResponse = true; notifyListeners(); try { @@ -170,11 +213,13 @@ class ChatProvider with ChangeNotifier { if (result['success'] == true) { final message = result['message'] as Message; - // Add the message to current chat if it's loaded + // Replace the optimistic message with the confirmed one from the server. if (_currentChat != null && _currentChat!.id == chatId) { - _currentChat = _currentChat!.copyWith( - messages: [..._currentChat!.messages, message], - ); + final updated = _currentChat!.messages + .where((m) => m.id != optimisticMessage.id) + .toList() + ..add(message); + _currentChat = _currentChat!.copyWith(messages: updated); } _errorMessage = null; @@ -183,11 +228,16 @@ class ChatProvider with ChangeNotifier { _startPolling(accessToken, chatId); return true; } else { + // Roll back the optimistic message on failure. + _rollbackOptimisticMessage(optimisticId, chatId); _errorMessage = result['error'] ?? 'Failed to send message'; return false; } } catch (e) { - _errorMessage = 'Error: ${e.toString()}'; + // Roll back the optimistic message on error. + _rollbackOptimisticMessage(optimisticId, chatId); + debugPrint('sendMessage error: $e'); + _errorMessage = 'Something went wrong. Please try again.'; return false; } finally { _isSendingMessage = false; @@ -217,15 +267,23 @@ class ChatProvider with ChangeNotifier { _chats[index] = updatedChat; } - // Update current chat if it's the same + // Update current chat if it's the same. + // Preserve existing messages — the title-update response may omit them. if (_currentChat != null && _currentChat!.id == chatId) { - _currentChat = updatedChat; + final Chat newChat; + if (updatedChat.messages.isEmpty) { + newChat = updatedChat.copyWith(messages: _currentChat!.messages); + } else { + newChat = updatedChat; + } + _currentChat = newChat; } notifyListeners(); } } catch (e) { - _errorMessage = 'Error: ${e.toString()}'; + debugPrint('updateChatTitle error: $e'); + _errorMessage = 'Something went wrong. Please try again.'; notifyListeners(); } } @@ -256,7 +314,8 @@ class ChatProvider with ChangeNotifier { return false; } } catch (e) { - _errorMessage = 'Error: ${e.toString()}'; + debugPrint('deleteChat error: $e'); + _errorMessage = 'Something went wrong. Please try again.'; notifyListeners(); return false; } @@ -266,6 +325,7 @@ class ChatProvider with ChangeNotifier { void _startPolling(String accessToken, String chatId) { _pollingTimer?.cancel(); _lastAssistantContentLength = null; + _stablePollingCount = 0; _isWaitingForResponse = true; _pollingStartTime = DateTime.now(); notifyListeners(); @@ -289,6 +349,7 @@ class ChatProvider with ChangeNotifier { _isPollingRequestInFlight = false; _isWaitingForResponse = false; _lastAssistantContentLength = null; + _stablePollingCount = 0; } /// Poll for updates @@ -335,13 +396,6 @@ class ChatProvider with ChangeNotifier { if (shouldUpdate) { _currentChat = updatedChat; - // Hide thinking indicator as soon as the first assistant content arrives. - if (_isWaitingForResponse) { - final lastMsg = updatedChat.messages.lastOrNull; - if (lastMsg != null && lastMsg.isAssistant && lastMsg.content.isNotEmpty) { - _isWaitingForResponse = false; - } - } notifyListeners(); } @@ -362,6 +416,7 @@ class ChatProvider with ChangeNotifier { if (newLen > (previousLen ?? -1)) { _lastAssistantContentLength = newLen; + _stablePollingCount = 0; if (newLen > 0) { // Content is growing — reset the inactivity clock. _pollingStartTime = DateTime.now(); @@ -369,11 +424,16 @@ class ChatProvider with ChangeNotifier { } // newLen == 0: empty placeholder, keep polling } else if (newLen > 0) { - // Content stable and non-empty: no growth since last poll — done. - _stopPolling(); - _lastAssistantContentLength = null; - notifyListeners(); - return; + // Content stable and non-empty. + // Require 2 consecutive stable polls before declaring done, to avoid + // stopping prematurely on a brief server-side generation pause. + _stablePollingCount++; + if (_stablePollingCount >= 2) { + _stopPolling(); + _lastAssistantContentLength = null; + notifyListeners(); + return; + } } // newLen == 0 with previousLen already 0: still empty, keep polling } diff --git a/mobile/lib/screens/chat_conversation_screen.dart b/mobile/lib/screens/chat_conversation_screen.dart index 9f9b3c73c..a18aafb5b 100644 --- a/mobile/lib/screens/chat_conversation_screen.dart +++ b/mobile/lib/screens/chat_conversation_screen.dart @@ -6,6 +6,7 @@ import '../models/chat.dart'; import '../providers/auth_provider.dart'; import '../providers/chat_provider.dart'; import '../models/message.dart'; +import '../constants/suggested_questions.dart'; import '../widgets/typing_indicator.dart'; class _SendMessageIntent extends Intent { @@ -86,6 +87,14 @@ class _ChatConversationScreenState extends State { } } + Future _sendSuggestedQuestion(String question) async { + if (!mounted) return; + final chatProvider = Provider.of(context, listen: false); + if (chatProvider.isSendingMessage || chatProvider.isWaitingForResponse) return; + _messageController.text = question; + await _sendMessage(); + } + Future _loadChat({bool forceRefresh = false}) async { if (_chatId == null) return; @@ -322,26 +331,45 @@ class _ChatConversationScreenState extends State { ); } - final messages = chatProvider.currentChat?.messages ?? []; + final allMessages = chatProvider.currentChat?.messages ?? []; + // While waiting for the AI response, hide the last (partial/streaming) + // assistant message so the typing indicator shows instead of partial content. + // The full response is revealed once polling detects stable content. + final messages = chatProvider.isWaitingForResponse + ? allMessages.where((m) { + return !(m.isAssistant && m == allMessages.lastOrNull); + }).toList() + : allMessages; + final firstName = + Provider.of(context, listen: true).user?.firstName; return Column( children: [ Expanded( - child: ListView.builder( - controller: _scrollController, - padding: const EdgeInsets.all(16), - itemCount: messages.length + - (chatProvider.isWaitingForResponse ? 1 : 0), - itemBuilder: (context, index) { - if (index == messages.length) { - return const _TypingIndicatorBubble(); - } - return _MessageBubble( - message: messages[index], - formatTime: _formatTime, - ); - }, - ), + child: messages.isEmpty && + !chatProvider.isLoading && + !chatProvider.isSendingMessage && + !chatProvider.isWaitingForResponse + ? _EmptyState( + firstName: firstName, + isSending: false, + onQuestionTap: _sendSuggestedQuestion, + ) + : ListView.builder( + controller: _scrollController, + padding: const EdgeInsets.all(16), + itemCount: messages.length + + (chatProvider.isWaitingForResponse ? 1 : 0), + itemBuilder: (context, index) { + if (index == messages.length) { + return const _TypingIndicatorBubble(); + } + return _MessageBubble( + message: messages[index], + formatTime: _formatTime, + ); + }, + ), ), // Message input @@ -551,6 +579,58 @@ class _MessageBubble extends StatelessWidget { } } +class _EmptyState extends StatelessWidget { + final String? firstName; + final bool isSending; + final void Function(String) onQuestionTap; + + const _EmptyState({ + required this.firstName, + required this.isSending, + required this.onQuestionTap, + }); + + @override + Widget build(BuildContext context) { + final colorScheme = Theme.of(context).colorScheme; + final name = (firstName ?? '').trim(); + final greeting = name.isNotEmpty ? 'Hi $name, how can I help?' : 'How can I help?'; + + return ListView( + padding: const EdgeInsets.all(24), + children: [ + const SizedBox(height: 32), + Text( + greeting, + style: Theme.of(context).textTheme.headlineSmall?.copyWith( + fontWeight: FontWeight.bold, + ), + textAlign: TextAlign.center, + ), + const SizedBox(height: 32), + ...suggestedQuestions.map( + (q) => Padding( + padding: const EdgeInsets.only(bottom: 12), + child: OutlinedButton.icon( + onPressed: isSending ? null : () => onQuestionTap(q.text), + icon: Icon(q.icon, size: 20), + label: Text(q.text, textAlign: TextAlign.left), + style: OutlinedButton.styleFrom( + alignment: Alignment.centerLeft, + padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 14), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(12), + ), + foregroundColor: colorScheme.onSurface, + ), + ), + ), + ), + ], + ); + } +} + class _TypingIndicatorBubble extends StatelessWidget { const _TypingIndicatorBubble(); From 18f0718f6304e87ea97435f1661203bc3d8e30ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Tue, 12 May 2026 20:26:43 +0200 Subject: [PATCH 18/31] Fixed on a PR about to be merged --- .devcontainer/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3d4e5bf25..b871287d4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,7 +15,6 @@ RUN apt-get update -qq \ libyaml-dev \ libyaml-0-2 \ openssh-client \ - libvips \ postgresql-client \ vim \ procps \ From f6fee24f99fc60e7829abbcfb0b2af528f3e9ed7 Mon Sep 17 00:00:00 2001 From: plind <59729252+plind-junior@users.noreply.github.com> Date: Tue, 12 May 2026 12:19:03 -0700 Subject: [PATCH 19/31] fix(ds/dialog): use existing i18n namespace for close button label (#1776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DS::Dialog#close_button called I18n.t("common.close") but no `common.close` key exists in any locale file, so every modal rendered the literal string "Translation missing: en.common.close" as both the `title` and `aria-label` of the X close button — visible to screen readers and as a hover tooltip. Switch to `ds.dialog.close` to mirror the existing `ds.alert.*` namespace under config/locales/views/components/*.yml, and add the English string. Other locales fall back to English (fallbacks=true in config/application.rb) until translated. Closes #1763. Co-authored-by: plind-junior --- app/components/DS/dialog.rb | 4 ++-- config/locales/views/components/en.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/components/DS/dialog.rb b/app/components/DS/dialog.rb index e9c4eb3ae..1027f6db0 100644 --- a/app/components/DS/dialog.rb +++ b/app/components/DS/dialog.rb @@ -133,8 +133,8 @@ class DS::Dialog < DesignSystemComponent variant: "icon", class: classes, icon: "x", - title: I18n.t("common.close"), - aria_label: I18n.t("common.close"), + title: I18n.t("ds.dialog.close"), + aria_label: I18n.t("ds.dialog.close"), data: { action: "DS--dialog#close" } ) end diff --git a/config/locales/views/components/en.yml b/config/locales/views/components/en.yml index b17124e37..785343aba 100644 --- a/config/locales/views/components/en.yml +++ b/config/locales/views/components/en.yml @@ -8,6 +8,8 @@ en: warning: Warning error: Error destructive: Error + dialog: + close: Close provider_sync_summary: title: Sync summary last_sync: "Last sync: %{time_ago} ago" From 2a0fcd4faead9e2dc9adf19dfc5face5cea7f7b3 Mon Sep 17 00:00:00 2001 From: Tao Chen <42793494+IamTaoChen@users.noreply.github.com> Date: Wed, 13 May 2026 03:28:00 +0800 Subject: [PATCH 20/31] feat: opening_balance_date and opening_balance(i18n) (#1377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * make default of opening_balance_date_label is TODAY * feat(i18n): add multi-language support for opening balance label - Use `t("valuations.show.opening_balance")` for all opening balance display (list and detail views) - Add or update `opening_balance` translation in all major languages under `config/locales/views/valuations/` - Now "Opening balance" will be localized in all supported languages * revert -2.years * Update config/locales/views/valuations/es.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Juan José Mata * Update config/locales/views/valuations/pt-BR.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Juan José Mata * Fix indentation for opening_balance in ro.yml Signed-off-by: Juan José Mata * Fix indentation for opening_balance in Turkish locale Signed-off-by: Juan José Mata * Update zh-TW.yml Signed-off-by: Juan José Mata --------- Signed-off-by: Juan José Mata Co-authored-by: Juan José Mata Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- app/views/valuations/_header.html.erb | 3 ++- app/views/valuations/_valuation.html.erb | 10 ++++++---- config/locales/views/valuations/de.yml | 1 + config/locales/views/valuations/en.yml | 1 + config/locales/views/valuations/es.yml | 1 + config/locales/views/valuations/fr.yml | 1 + config/locales/views/valuations/nb.yml | 3 ++- config/locales/views/valuations/nl.yml | 1 + config/locales/views/valuations/pt-BR.yml | 1 + config/locales/views/valuations/ro.yml | 1 + config/locales/views/valuations/tr.yml | 3 ++- config/locales/views/valuations/zh-CN.yml | 1 + config/locales/views/valuations/zh-TW.yml | 1 + 13 files changed, 21 insertions(+), 7 deletions(-) diff --git a/app/views/valuations/_header.html.erb b/app/views/valuations/_header.html.erb index 2db8c8739..d4a376876 100644 --- a/app/views/valuations/_header.html.erb +++ b/app/views/valuations/_header.html.erb @@ -2,7 +2,8 @@ <%= tag.header class: "mb-4 space-y-1", id: dom_id(entry, :header) do %> - <%= entry.name %> + <% valuation = entry.entryable %> + <%= valuation.respond_to?(:opening_anchor?) && valuation.opening_anchor? ? t("valuations.show.opening_balance") : entry.name %>
diff --git a/app/views/valuations/_valuation.html.erb b/app/views/valuations/_valuation.html.erb index e377eda0f..003faf5fe 100644 --- a/app/views/valuations/_valuation.html.erb +++ b/app/views/valuations/_valuation.html.erb @@ -17,10 +17,12 @@ <%= render DS::FilledIcon.new(icon: icon, size: "md", hex_color: color, rounded: true) %>
- <%= link_to entry.name, - entry_path(entry), - data: { turbo_frame: "drawer", turbo_prefetch: false }, - class: "hover:underline" %> + <%= link_to( + valuation.opening_anchor? ? t("valuations.show.opening_balance") : entry.name, + entry_path(entry), + data: { turbo_frame: "drawer", turbo_prefetch: false }, + class: "hover:underline" + ) %>
diff --git a/config/locales/views/valuations/de.yml b/config/locales/views/valuations/de.yml index cf7a48000..fc799ab44 100644 --- a/config/locales/views/valuations/de.yml +++ b/config/locales/views/valuations/de.yml @@ -28,3 +28,4 @@ de: note_placeholder: Füge zusätzliche Details zu diesem Eintrag hinzu overview: Übersicht settings: Einstellungen + opening_balance: Eröffnungssaldo diff --git a/config/locales/views/valuations/en.yml b/config/locales/views/valuations/en.yml index f4a55767c..6a8e964a2 100644 --- a/config/locales/views/valuations/en.yml +++ b/config/locales/views/valuations/en.yml @@ -30,3 +30,4 @@ en: note_placeholder: Add any additional details about this entry overview: Overview settings: Settings + opening_balance: "Opening balance" diff --git a/config/locales/views/valuations/es.yml b/config/locales/views/valuations/es.yml index 9f1e6312c..cf0816266 100644 --- a/config/locales/views/valuations/es.yml +++ b/config/locales/views/valuations/es.yml @@ -30,3 +30,4 @@ es: note_placeholder: Añade cualquier detalle adicional sobre esta entrada overview: Resumen settings: Configuración + opening_balance: Saldo inicial diff --git a/config/locales/views/valuations/fr.yml b/config/locales/views/valuations/fr.yml index d8d59a104..ccd22d4af 100644 --- a/config/locales/views/valuations/fr.yml +++ b/config/locales/views/valuations/fr.yml @@ -28,3 +28,4 @@ fr: note_placeholder: Ajoutez tout détail supplémentaire à ce bilan overview: Aperçu settings: Paramètres + opening_balance: Solde d'ouverture diff --git a/config/locales/views/valuations/nb.yml b/config/locales/views/valuations/nb.yml index 5125b0071..f2d311307 100644 --- a/config/locales/views/valuations/nb.yml +++ b/config/locales/views/valuations/nb.yml @@ -27,4 +27,5 @@ nb: note_label: Notater note_placeholder: Legg til eventuelle tilleggsdetaljer om denne oppføringen overview: Oversikt - settings: Innstillinger \ No newline at end of file + settings: Innstillinger + opening_balance: Startsaldo \ No newline at end of file diff --git a/config/locales/views/valuations/nl.yml b/config/locales/views/valuations/nl.yml index 7bc507bc7..e84eb6865 100644 --- a/config/locales/views/valuations/nl.yml +++ b/config/locales/views/valuations/nl.yml @@ -28,3 +28,4 @@ nl: note_placeholder: Voeg eventuele aanvullende details toe over deze invoer overview: Overzicht settings: Instellingen + opening_balance: Beginsaldo diff --git a/config/locales/views/valuations/pt-BR.yml b/config/locales/views/valuations/pt-BR.yml index 41d90cad1..32b6504ab 100644 --- a/config/locales/views/valuations/pt-BR.yml +++ b/config/locales/views/valuations/pt-BR.yml @@ -28,3 +28,4 @@ pt-BR: note_placeholder: Adicione detalhes adicionais sobre esta entrada overview: Visão geral settings: Configurações + opening_balance: Saldo inicial diff --git a/config/locales/views/valuations/ro.yml b/config/locales/views/valuations/ro.yml index b607ba206..29c406faa 100644 --- a/config/locales/views/valuations/ro.yml +++ b/config/locales/views/valuations/ro.yml @@ -24,3 +24,4 @@ ro: note_placeholder: Adaugă orice detalii suplimentare despre această înregistrare overview: Prezentare generală settings: Setări + opening_balance: Sold inițial diff --git a/config/locales/views/valuations/tr.yml b/config/locales/views/valuations/tr.yml index 1eb75757b..461876e95 100644 --- a/config/locales/views/valuations/tr.yml +++ b/config/locales/views/valuations/tr.yml @@ -27,4 +27,5 @@ tr: note_label: Notlar note_placeholder: Bu girişle ilgili ek detaylar ekleyin overview: Genel Bakış - settings: Ayarlar \ No newline at end of file + settings: Ayarlar + opening_balance: Açılış bakiyesi diff --git a/config/locales/views/valuations/zh-CN.yml b/config/locales/views/valuations/zh-CN.yml index e6a3a37ec..2cc6bfc63 100644 --- a/config/locales/views/valuations/zh-CN.yml +++ b/config/locales/views/valuations/zh-CN.yml @@ -30,3 +30,4 @@ zh-CN: note_placeholder: 添加此记录的其他详细信息 overview: 概览 settings: 设置 + opening_balance: 初始余额 diff --git a/config/locales/views/valuations/zh-TW.yml b/config/locales/views/valuations/zh-TW.yml index b1a374afe..af39427d9 100644 --- a/config/locales/views/valuations/zh-TW.yml +++ b/config/locales/views/valuations/zh-TW.yml @@ -28,3 +28,4 @@ zh-TW: note_placeholder: 加入關於此紀錄的額外資訊 overview: 概覽 settings: 設定 + opening_balance: 初始餘額 From 0ab3b0b6988b3982895aefb6280a98ea8a86ef82 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Tue, 12 May 2026 12:29:29 -0700 Subject: [PATCH 21/31] feat(exports): add rule operand references (#1726) * feat(exports): add rule operand references * fix(exports): preserve rule operand references * refactor(exports): simplify rule operand branches * refactor(validation): centralize UUID format checks * fix(imports): preserve false rule operands --- app/controllers/api/v1/base_controller.rb | 5 +- .../api/v1/valuations_controller.rb | 4 - app/models/family/data_exporter.rb | 78 ++++++++---- app/models/family/data_importer.rb | 19 ++- lib/uuid_format.rb | 11 ++ test/lib/uuid_format_test.rb | 17 +++ test/models/family/data_exporter_test.rb | 48 ++++++++ test/models/family/data_importer_test.rb | 113 ++++++++++++++++++ 8 files changed, 261 insertions(+), 34 deletions(-) create mode 100644 lib/uuid_format.rb create mode 100644 test/lib/uuid_format_test.rb diff --git a/app/controllers/api/v1/base_controller.rb b/app/controllers/api/v1/base_controller.rb index ef175bf17..f1631ec1c 100644 --- a/app/controllers/api/v1/base_controller.rb +++ b/app/controllers/api/v1/base_controller.rb @@ -3,14 +3,11 @@ class Api::V1::BaseController < ApplicationController include Doorkeeper::Rails::Helpers - UUID_PATTERN = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i - private_constant :UUID_PATTERN - InvalidFilterError = Class.new(StandardError) class << self def valid_uuid?(value) - value.to_s.match?(UUID_PATTERN) + UuidFormat.valid?(value) end end diff --git a/app/controllers/api/v1/valuations_controller.rb b/app/controllers/api/v1/valuations_controller.rb index d5ce1842b..24c9cfd21 100644 --- a/app/controllers/api/v1/valuations_controller.rb +++ b/app/controllers/api/v1/valuations_controller.rb @@ -269,10 +269,6 @@ class Api::V1::ValuationsController < Api::V1::BaseController raise InvalidFilterError, "#{key} must be an ISO 8601 date" end - def valid_uuid?(value) - value.to_s.match?(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i) - end - def safe_page_param page = params[:page].to_i page > 0 ? page : 1 diff --git a/app/models/family/data_exporter.rb b/app/models/family/data_exporter.rb index 34b6beba5..da3f8ac36 100644 --- a/app/models/family/data_exporter.rb +++ b/app/models/family/data_exporter.rb @@ -493,11 +493,14 @@ class Family::DataExporter end def serialize_condition(condition) + operand = resolve_condition_operand(condition) data = { condition_type: condition.condition_type, operator: condition.operator, - value: resolve_condition_value(condition) + value: operand[:value] } + value_ref = operand[:value_ref] + data[:value_ref] = value_ref if value_ref.present? if condition.compound? && condition.sub_conditions.any? data[:sub_conditions] = condition.sub_conditions.map { |sub| serialize_condition(sub) } @@ -507,52 +510,79 @@ class Family::DataExporter end def serialize_action(action) - { + operand = resolve_action_operand(action) + data = { action_type: action.action_type, - value: resolve_action_value(action) + value: operand[:value] } + value_ref = operand[:value_ref] + data[:value_ref] = value_ref if value_ref.present? + + data end - def resolve_condition_value(condition) - return condition.value unless condition.value.present? + def resolve_condition_operand(condition) + return rule_operand(condition.value) unless condition.value.present? # Map category UUIDs to names for portability - if condition.condition_type == "transaction_category" && condition.value.present? - category = @family.categories.find_by(id: condition.value) - return category&.name || condition.value + if condition.condition_type == "transaction_category" + return rule_operand(condition.value, type: "Category", relation: @family.categories) end # Map merchant UUIDs to names for portability - if condition.condition_type == "transaction_merchant" && condition.value.present? - merchant = @family.merchants.find_by(id: condition.value) - return merchant&.name || condition.value + if condition.condition_type == "transaction_merchant" + return rule_operand(condition.value, type: "Merchant", relation: @family.merchants) end - condition.value + rule_operand(condition.value) end - def resolve_action_value(action) - return action.value unless action.value.present? + def resolve_action_operand(action) + return rule_operand(action.value) unless action.value.present? # Map category UUIDs to names for portability - if action.action_type == "set_transaction_category" && action.value.present? - category = @family.categories.find_by(id: action.value) || @family.categories.find_by(name: action.value) - return category&.name || action.value + if action.action_type == "set_transaction_category" + return rule_operand(action.value, type: "Category", relation: @family.categories, fallback_to_name: true) end # Map merchant UUIDs to names for portability - if action.action_type == "set_transaction_merchant" && action.value.present? - merchant = @family.merchants.find_by(id: action.value) || @family.merchants.find_by(name: action.value) - return merchant&.name || action.value + if action.action_type == "set_transaction_merchant" + return rule_operand(action.value, type: "Merchant", relation: @family.merchants, fallback_to_name: true) end # Map tag UUIDs to names for portability - if action.action_type == "set_transaction_tags" && action.value.present? - tag = @family.tags.find_by(id: action.value) || @family.tags.find_by(name: action.value) - return tag&.name || action.value + if action.action_type == "set_transaction_tags" + return rule_operand(action.value, type: "Tag", relation: @family.tags, fallback_to_name: true) end - action.value + rule_operand(action.value) + end + + def rule_operand(value, type: nil, relation: nil, fallback_to_name: false) + record = relation && resolve_rule_operand_record(relation, value, fallback_to_name: fallback_to_name) + + { + value: record&.name || value, + value_ref: record ? rule_value_ref(type, record) : nil + } + end + + def resolve_rule_operand_record(relation, value, fallback_to_name:) + return relation.find_by(id: value) if uuid_like?(value) + + relation.find_by(name: value) if fallback_to_name + end + + def rule_value_ref(type, record) + { + type: type, + id: record.id, + name: record.name + } + end + + def uuid_like?(value) + UuidFormat.valid?(value) end def serialize_conditions_for_csv(conditions) diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb index 7601516ce..8ee468dc8 100644 --- a/app/models/family/data_importer.rb +++ b/app/models/family/data_importer.rb @@ -671,7 +671,7 @@ class Family::DataImporter def resolve_rule_condition_value(condition_data) condition_type = condition_data["condition_type"] - value = condition_data["value"] + value = rule_operand_value(condition_data) return value unless value.present? @@ -699,7 +699,7 @@ class Family::DataImporter def resolve_rule_action_value(action_data) action_type = action_data["action_type"] - value = action_data["value"] + value = rule_operand_value(action_data) return value unless value.present? @@ -732,6 +732,21 @@ class Family::DataImporter value end + def rule_operand_value(data) + raw_value = data["value"] + value = raw_value.is_a?(String) ? raw_value.presence : raw_value + value_ref_name = data.dig("value_ref", "name") + + return value_ref_name if value.is_a?(String) && uuid_like?(value) && value_ref_name.present? + return value unless value.nil? + + value_ref_name + end + + def uuid_like?(value) + UuidFormat.valid?(value) + end + def importable_cost_basis_source(value) source = value.to_s Holding::COST_BASIS_SOURCES.include?(source) ? source : nil diff --git a/lib/uuid_format.rb b/lib/uuid_format.rb new file mode 100644 index 000000000..6e4890f9c --- /dev/null +++ b/lib/uuid_format.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module UuidFormat + PATTERN = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i.freeze + + module_function + + def valid?(value) + PATTERN.match?(value.to_s) + end +end diff --git a/test/lib/uuid_format_test.rb b/test/lib/uuid_format_test.rb new file mode 100644 index 000000000..0d7389b97 --- /dev/null +++ b/test/lib/uuid_format_test.rb @@ -0,0 +1,17 @@ +require "test_helper" + +class UuidFormatTest < ActiveSupport::TestCase + test "valid matches canonical UUID values" do + uuid = SecureRandom.uuid + + assert UuidFormat.valid?(uuid) + assert UuidFormat.valid?(uuid.upcase) + end + + test "valid rejects non UUID values" do + refute UuidFormat.valid?(nil) + refute UuidFormat.valid?("") + refute UuidFormat.valid?("not-a-uuid") + refute UuidFormat.valid?("#{SecureRandom.uuid}-extra") + end +end diff --git a/test/models/family/data_exporter_test.rb b/test/models/family/data_exporter_test.rb index 03376e08e..5a8ff650a 100644 --- a/test/models/family/data_exporter_test.rb +++ b/test/models/family/data_exporter_test.rb @@ -318,9 +318,56 @@ class Family::DataExporterTest < ActiveSupport::TestCase assert_equal "set_transaction_category", actions[0]["action_type"] # Should export category name instead of UUID assert_equal "Test Category", actions[0]["value"] + assert_equal({ "type" => "Category", "id" => @category.id, "name" => "Test Category" }, actions[0]["value_ref"]) end end + test "exports rule condition value refs for mapped operands" do + category_rule = @family.rules.build( + name: "Category Condition Rule", + resource_type: "transaction", + active: true + ) + category_rule.conditions.build( + condition_type: "transaction_category", + operator: "=", + value: @category.id + ) + category_rule.actions.build(action_type: "auto_categorize") + category_rule.save! + + zip_data = @exporter.generate_export + + Zip::File.open_buffer(zip_data) do |zip| + rule_data = zip.read("all.ndjson").split("\n").filter_map do |line| + parsed = JSON.parse(line) + parsed if parsed["type"] == "Rule" && parsed["data"]["name"] == "Category Condition Rule" + end.first + + condition = rule_data["data"]["conditions"].first + assert_equal "Test Category", condition["value"] + assert_equal({ "type" => "Category", "id" => @category.id, "name" => "Test Category" }, condition["value_ref"]) + end + end + + test "rule operand lookup skips name fallback for stale UUID values" do + stale_uuid = SecureRandom.uuid + relation = mock + relation.expects(:find_by).with(id: stale_uuid).once.returns(nil) + relation.expects(:find_by).with(name: stale_uuid).never + + operand = @exporter.send( + :rule_operand, + stale_uuid, + type: "Category", + relation: relation, + fallback_to_name: true + ) + + assert_equal stale_uuid, operand[:value] + assert_nil operand[:value_ref] + end + test "exports rule actions and maps tag UUIDs to names" do # Create a rule with a tag action tag_rule = @family.rules.build( @@ -359,6 +406,7 @@ class Family::DataExporterTest < ActiveSupport::TestCase assert_equal "set_transaction_tags", actions[0]["action_type"] # Should export tag name instead of UUID assert_equal "Test Tag", actions[0]["value"] + assert_equal({ "type" => "Tag", "id" => @tag.id, "name" => "Test Tag" }, actions[0]["value_ref"]) end end diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb index 0d8efd0be..c6c75dda1 100644 --- a/test/models/family/data_importer_test.rb +++ b/test/models/family/data_importer_test.rb @@ -1498,6 +1498,119 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal category.id, action.value end + test "imports rules from normalized operand value refs" do + ndjson = build_ndjson([ + { + type: "Rule", + version: 1, + data: { + name: "Map Merchant To Dining", + resource_type: "transaction", + active: true, + conditions: [ + { + condition_type: "transaction_merchant", + operator: "=", + value_ref: { + type: "Merchant", + id: "source-merchant-id", + name: "Coffee Bar" + } + } + ], + actions: [ + { + action_type: "set_transaction_category", + value_ref: { + type: "Category", + id: "source-category-id", + name: "Dining" + } + } + ] + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + rule = @family.rules.find_by!(name: "Map Merchant To Dining") + merchant = @family.merchants.find_by!(name: "Coffee Bar") + category = @family.categories.find_by!(name: "Dining") + + assert_equal merchant.id, rule.conditions.first.value + assert_equal category.id, rule.actions.first.value + end + + test "imports rule value refs when legacy operand values are stale UUIDs" do + stale_merchant_id = SecureRandom.uuid + stale_category_id = SecureRandom.uuid + ndjson = build_ndjson([ + { + type: "Rule", + version: 1, + data: { + name: "Map Stale UUID Operands", + resource_type: "transaction", + active: true, + conditions: [ + { + condition_type: "transaction_merchant", + operator: "=", + value: stale_merchant_id, + value_ref: { + type: "Merchant", + id: stale_merchant_id, + name: "Coffee Bar" + } + } + ], + actions: [ + { + action_type: "set_transaction_category", + value: stale_category_id, + value_ref: { + type: "Category", + id: stale_category_id, + name: "Dining" + } + } + ] + } + } + ]) + + importer = Family::DataImporter.new(@family, ndjson) + importer.import! + + rule = @family.rules.find_by!(name: "Map Stale UUID Operands") + merchant = @family.merchants.find_by!(name: "Coffee Bar") + category = @family.categories.find_by!(name: "Dining") + + assert_equal merchant.id, rule.conditions.first.value + assert_equal category.id, rule.actions.first.value + assert_not @family.merchants.exists?(name: stale_merchant_id) + assert_not @family.categories.exists?(name: stale_category_id) + end + + test "preserves explicit false rule operand values" do + importer = Family::DataImporter.new(@family, "") + + value = importer.send( + :rule_operand_value, + { + "value" => false, + "value_ref" => { + "type" => "Category", + "name" => "Fallback" + } + } + ) + + assert_equal false, value + end + test "imports rules with compound conditions" do ndjson = build_ndjson([ { From 6402f1dd08927ce88a12c707bf628ad630a4a038 Mon Sep 17 00:00:00 2001 From: plind <59729252+plind-junior@users.noreply.github.com> Date: Tue, 12 May 2026 12:55:22 -0700 Subject: [PATCH 22/31] fix(sso): preserve user-edited name across OIDC logins (#1777) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OidcIdentity#sync_user_attributes! runs on every SSO sign-in and overwrote user.first_name / user.last_name with whatever the IdP sent, because the precedence was `auth.info.* || user.*` — the IdP always won when it supplied a value. A user who edited their first name to "Adam" inside Sure had it reset to the IdP value "Ben" on the next login, while the last name only "stuck" when the IdP happened not to return a last_name (#1103). Swap the precedence to `user.* || auth.info.*` so the IdP fills only when Sure has nothing on file (first link or admin-blanked field). Edits inside Sure are then authoritative for every subsequent login. The audit copy on the OidcIdentity record itself is unchanged, so the IdP-reported name is still available for debugging. Closes #1103. Co-authored-by: plind-junior --- app/models/oidc_identity.rb | 9 +++++--- test/models/oidc_identity_test.rb | 35 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/app/models/oidc_identity.rb b/app/models/oidc_identity.rb index fe85a1fdf..a95daf10e 100644 --- a/app/models/oidc_identity.rb +++ b/app/models/oidc_identity.rb @@ -25,10 +25,13 @@ class OidcIdentity < ApplicationRecord groups: groups }) - # Sync name to user if provided (keep existing if IdP doesn't provide) + # Sync name to user only when Sure has nothing on file (first link, or an + # admin blanked the field). Edits made inside Sure must survive subsequent + # SSO logins — previously the IdP value won unconditionally and clobbered + # any manually-edited name on every login (#1103). user.update!( - first_name: auth.info&.first_name.presence || user.first_name, - last_name: auth.info&.last_name.presence || user.last_name + first_name: user.first_name.presence || auth.info&.first_name.presence, + last_name: user.last_name.presence || auth.info&.last_name.presence ) # Apply role mapping based on group membership diff --git a/test/models/oidc_identity_test.rb b/test/models/oidc_identity_test.rb index 43342964b..52a807a8f 100644 --- a/test/models/oidc_identity_test.rb +++ b/test/models/oidc_identity_test.rb @@ -57,6 +57,41 @@ class OidcIdentityTest < ActiveSupport::TestCase end end + test "sync_user_attributes! preserves user-edited first and last name" do + # Regression for #1103: every SSO login used to overwrite user.first_name + # with the IdP value, clobbering edits the user made inside Sure. + @user.update!(first_name: "Adam", last_name: "Smith") + auth = OmniAuth::AuthHash.new( + provider: @oidc_identity.provider, + uid: @oidc_identity.uid, + info: { email: @user.email, first_name: "Ben", last_name: "Jones" } + ) + + @oidc_identity.sync_user_attributes!(auth) + + @user.reload + assert_equal "Adam", @user.first_name + assert_equal "Smith", @user.last_name + # Identity record itself still mirrors the IdP for audit / debugging. + assert_equal "Ben", @oidc_identity.info["first_name"] + assert_equal "Jones", @oidc_identity.info["last_name"] + end + + test "sync_user_attributes! fills blank user names from the IdP" do + @user.update!(first_name: nil, last_name: nil) + auth = OmniAuth::AuthHash.new( + provider: @oidc_identity.provider, + uid: @oidc_identity.uid, + info: { email: @user.email, first_name: "Ben", last_name: "Jones" } + ) + + @oidc_identity.sync_user_attributes!(auth) + + @user.reload + assert_equal "Ben", @user.first_name + assert_equal "Jones", @user.last_name + end + test "creates from omniauth hash" do auth = OmniAuth::AuthHash.new({ provider: "google_oauth2", From d5bfccc4c752fa151bd918ccc37e01070e448731 Mon Sep 17 00:00:00 2001 From: Tristan the Katana <50181095+felixmuinde@users.noreply.github.com> Date: Wed, 13 May 2026 00:13:11 +0300 Subject: [PATCH 23/31] feat(mobile): add mass delete for chats (#1779) * feat(mobile): add mass delete for chats Long-press any chat to enter selection mode, tap items to select/deselect, use Select All to toggle all, then delete with a single confirmation. Swipe-to-delete continues to work outside selection mode. * fix(mobile): address PR review comments on mass delete - Wrap each deleteChat call in its own try-catch so a single network failure doesn't abort the entire Future.wait operation - Add null-safe casting for deletedCount and failedIds in provider - Fix misleading error snackbar copy ("Some chats could not be deleted" implied partial failure; provider only returns false on total failure) Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- mobile/lib/providers/chat_provider.dart | 36 +++++ mobile/lib/screens/chat_list_screen.dart | 163 ++++++++++++++++++++--- mobile/lib/services/chat_service.dart | 28 ++++ 3 files changed, 209 insertions(+), 18 deletions(-) diff --git a/mobile/lib/providers/chat_provider.dart b/mobile/lib/providers/chat_provider.dart index 317535fdb..2eb6ed212 100644 --- a/mobile/lib/providers/chat_provider.dart +++ b/mobile/lib/providers/chat_provider.dart @@ -321,6 +321,42 @@ class ChatProvider with ChangeNotifier { } } + /// Delete multiple chats + Future deleteMultipleChats({ + required String accessToken, + required List chatIds, + }) async { + try { + final result = await _chatService.deleteMultipleChats( + accessToken: accessToken, + chatIds: chatIds, + ); + + final deletedCount = (result['deletedCount'] as int?) ?? 0; + if (result['success'] == true || deletedCount > 0) { + final failedIds = ((result['failedIds'] as List?) ?? []).cast().toSet(); + final deleted = chatIds.toSet().difference(failedIds); + _chats.removeWhere((c) => deleted.contains(c.id)); + + if (_currentChat != null && deleted.contains(_currentChat!.id)) { + _currentChat = null; + } + + notifyListeners(); + return true; + } + + _errorMessage = 'Failed to delete chats'; + notifyListeners(); + return false; + } catch (e) { + debugPrint('deleteMultipleChats error: $e'); + _errorMessage = 'Something went wrong. Please try again.'; + notifyListeners(); + return false; + } + } + /// Start polling for new messages (AI responses) void _startPolling(String accessToken, String chatId) { _pollingTimer?.cancel(); diff --git a/mobile/lib/screens/chat_list_screen.dart b/mobile/lib/screens/chat_list_screen.dart index 3c1606ecd..7868a30f5 100644 --- a/mobile/lib/screens/chat_list_screen.dart +++ b/mobile/lib/screens/chat_list_screen.dart @@ -12,6 +12,9 @@ class ChatListScreen extends StatefulWidget { } class _ChatListScreenState extends State { + bool _isSelectionMode = false; + final Set _selectedChatIds = {}; + @override void initState() { super.initState(); @@ -35,6 +38,89 @@ class _ChatListScreenState extends State { await _loadChats(); } + void _toggleSelectionMode() { + setState(() { + _isSelectionMode = !_isSelectionMode; + _selectedChatIds.clear(); + }); + } + + void _toggleSelectAll(List allIds) { + setState(() { + if (_selectedChatIds.length == allIds.length) { + _selectedChatIds.clear(); + } else { + _selectedChatIds + ..clear() + ..addAll(allIds); + } + }); + } + + void _toggleChatSelection(String id) { + setState(() { + if (_selectedChatIds.contains(id)) { + _selectedChatIds.remove(id); + } else { + _selectedChatIds.add(id); + } + }); + } + + Future _deleteSelectedChats() async { + final authProvider = Provider.of(context, listen: false); + final chatProvider = Provider.of(context, listen: false); + + final confirmed = await showDialog( + context: context, + builder: (context) => AlertDialog( + title: const Text('Delete Chats'), + content: Text( + 'Delete ${_selectedChatIds.length} chat(s)? This cannot be undone.', + ), + actions: [ + TextButton( + onPressed: () => Navigator.pop(context, false), + child: const Text('Cancel'), + ), + TextButton( + onPressed: () => Navigator.pop(context, true), + child: const Text('Delete', style: TextStyle(color: Colors.red)), + ), + ], + ), + ); + + if (confirmed != true || !mounted) return; + + final accessToken = await authProvider.getValidAccessToken(); + if (accessToken == null) { + await authProvider.logout(); + return; + } + + final success = await chatProvider.deleteMultipleChats( + accessToken: accessToken, + chatIds: _selectedChatIds.toList(), + ); + + if (!mounted) return; + + setState(() { + _isSelectionMode = false; + _selectedChatIds.clear(); + }); + + ScaffoldMessenger.of(context).showSnackBar( + SnackBar( + content: Text( + success ? 'Chats deleted' : 'Failed to delete chats', + ), + backgroundColor: success ? Colors.green : Colors.red, + ), + ); + } + Future _openNewChat() async { if (!mounted) return; @@ -74,17 +160,38 @@ class _ChatListScreenState extends State { title: const Text('Chats'), centerTitle: false, actions: [ - Padding( - padding: const EdgeInsets.only(top: 12, right: 12), - child: InkWell( - onTap: _handleRefresh, - child: const SizedBox( - width: 36, - height: 36, - child: Icon(Icons.refresh), + if (_isSelectionMode) ...[ + IconButton( + icon: const Icon(Icons.delete), + onPressed: _selectedChatIds.isNotEmpty ? _deleteSelectedChats : null, + ), + IconButton( + icon: const Icon(Icons.select_all), + onPressed: () { + final allIds = Provider.of(context, listen: false) + .chats + .map((c) => c.id) + .toList(); + _toggleSelectAll(allIds); + }, + ), + IconButton( + icon: const Icon(Icons.close), + onPressed: _toggleSelectionMode, + ), + ] else ...[ + Padding( + padding: const EdgeInsets.only(top: 12, right: 12), + child: InkWell( + onTap: _handleRefresh, + child: const SizedBox( + width: 36, + height: 36, + child: Icon(Icons.refresh), + ), ), ), - ), + ], ], ), body: Consumer( @@ -166,9 +273,12 @@ class _ChatListScreenState extends State { itemCount: chatProvider.chats.length, itemBuilder: (context, index) { final chat = chatProvider.chats[index]; + final isSelected = _selectedChatIds.contains(chat.id); return Dismissible( key: Key(chat.id), - direction: DismissDirection.endToStart, + direction: _isSelectionMode + ? DismissDirection.none + : DismissDirection.endToStart, background: Container( color: Colors.red, alignment: Alignment.centerRight, @@ -208,13 +318,18 @@ class _ChatListScreenState extends State { } }, child: ListTile( - leading: CircleAvatar( - backgroundColor: colorScheme.primaryContainer, - child: Icon( - Icons.chat, - color: colorScheme.onPrimaryContainer, - ), - ), + leading: _isSelectionMode + ? Checkbox( + value: isSelected, + onChanged: (_) => _toggleChatSelection(chat.id), + ) + : CircleAvatar( + backgroundColor: colorScheme.primaryContainer, + child: Icon( + Icons.chat, + color: colorScheme.onPrimaryContainer, + ), + ), title: Text( chat.title, maxLines: 1, @@ -223,7 +338,7 @@ class _ChatListScreenState extends State { subtitle: chat.lastMessageAt != null ? Text(_formatDateTime(chat.lastMessageAt!)) : null, - trailing: chat.messageCount != null + trailing: chat.messageCount != null && !_isSelectionMode ? Container( padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), decoration: BoxDecoration( @@ -241,6 +356,10 @@ class _ChatListScreenState extends State { ) : null, onTap: () async { + if (_isSelectionMode) { + _toggleChatSelection(chat.id); + return; + } await Navigator.push( context, MaterialPageRoute( @@ -249,6 +368,14 @@ class _ChatListScreenState extends State { ); _loadChats(); }, + onLongPress: _isSelectionMode + ? null + : () { + setState(() { + _isSelectionMode = true; + _selectedChatIds.add(chat.id); + }); + }, ), ); }, diff --git a/mobile/lib/services/chat_service.dart b/mobile/lib/services/chat_service.dart index 1e55e57da..37b927f4c 100644 --- a/mobile/lib/services/chat_service.dart +++ b/mobile/lib/services/chat_service.dart @@ -331,6 +331,34 @@ class ChatService { } } + /// Delete multiple chats in parallel + Future> deleteMultipleChats({ + required String accessToken, + required List chatIds, + }) async { + final results = await Future.wait( + chatIds.map((id) async { + try { + return await deleteChat(accessToken: accessToken, chatId: id); + } catch (_) { + return {'success': false}; + } + }), + eagerError: false, + ); + final failedIds = chatIds + .asMap() + .entries + .where((e) => results[e.key]['success'] != true) + .map((e) => e.value) + .toList(); + return { + 'success': failedIds.isEmpty, + 'deletedCount': chatIds.length - failedIds.length, + 'failedIds': failedIds, + }; + } + /// Retry the last assistant response in a chat Future> retryMessage({ required String accessToken, From 3c4c32584aadeb94f75f7e3f6a75e212ddda71aa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 12 May 2026 21:37:59 +0000 Subject: [PATCH 24/31] Bump version to next iteration after v0.7.1-alpha.6 release --- .sure-version | 2 +- charts/sure/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.sure-version b/.sure-version index 57984f54d..81fe56438 100644 --- a/.sure-version +++ b/.sure-version @@ -1 +1 @@ -0.7.1-alpha.6 +0.7.1-alpha.7 diff --git a/charts/sure/Chart.yaml b/charts/sure/Chart.yaml index c0859d4b0..b9e521afb 100644 --- a/charts/sure/Chart.yaml +++ b/charts/sure/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: sure description: Official Helm chart for deploying the Sure Rails app (web + Sidekiq) on Kubernetes with optional HA PostgreSQL (CloudNativePG) and Redis. type: application -version: 0.7.1-alpha.6 -appVersion: "0.7.1-alpha.6" +version: 0.7.1-alpha.7 +appVersion: "0.7.1-alpha.7" kubeVersion: ">=1.25.0-0" From ce5d7dd736ad2613463e84def6699099f2ce4e61 Mon Sep 17 00:00:00 2001 From: Gian-Reto Tarnutzer Date: Tue, 12 May 2026 23:45:19 +0200 Subject: [PATCH 25/31] Add Interactive Brokers Provider (#1722) * Display multi-currency holdings correctly * Implement IBKR provider * Fix: Use historical exchange rate for historical prices * Add brokerage exchange rate for trades * Sync historical balances from IBKR * Add logos in activity history * Fix privacy mode blur in account view * Improve IBKR XML Flex report parser errors --- .../UI/account/activity_date.html.erb | 2 +- app/controllers/accounts_controller.rb | 4 +- app/controllers/ibkr_items_controller.rb | 236 +++++++++++++++ .../settings/providers_controller.rb | 6 + app/controllers/transactions_controller.rb | 1 + app/helpers/settings_helper.rb | 3 + .../time_series_chart_controller.js | 2 +- app/models/account.rb | 50 +++- app/models/account/opening_balance_manager.rb | 6 +- app/models/account/provider_import_adapter.rb | 10 +- app/models/account/syncer.rb | 13 + app/models/balance/sync_cache.rb | 5 +- app/models/data_enrichment.rb | 15 +- app/models/family.rb | 2 +- app/models/family/ibkr_connectable.rb | 22 ++ app/models/holding.rb | 10 +- app/models/holding/materializer.rb | 21 +- app/models/holding/portfolio_cache.rb | 2 +- app/models/ibkr_account.rb | 78 +++++ .../ibkr_account/activities_processor.rb | 221 ++++++++++++++ app/models/ibkr_account/data_helpers.rb | 78 +++++ .../ibkr_account/historical_balances_sync.rb | 79 +++++ app/models/ibkr_account/holdings_processor.rb | 105 +++++++ app/models/ibkr_account/processor.rb | 56 ++++ app/models/ibkr_item.rb | 124 ++++++++ app/models/ibkr_item/importer.rb | 33 ++ app/models/ibkr_item/provided.rb | 9 + app/models/ibkr_item/report_parser.rb | 143 +++++++++ app/models/ibkr_item/sync_complete_event.rb | 22 ++ app/models/ibkr_item/syncer.rb | 68 +++++ app/models/ibkr_item/unlinking.rb | 37 +++ app/models/provider/ibkr_adapter.rb | 59 ++++ app/models/provider/ibkr_flex.rb | 144 +++++++++ app/models/provider/metadata.rb | 1 + app/models/provider_connection_status.rb | 1 + app/models/snaptrade_account/processor.rb | 25 ++ app/models/trade.rb | 32 ++ app/models/transaction.rb | 29 +- .../activity_security_preloader.rb | 36 +++ app/views/accounts/index.html.erb | 6 +- app/views/holdings/_cash.html.erb | 2 +- app/views/holdings/_cost_basis_cell.html.erb | 4 +- app/views/holdings/_holding.html.erb | 2 +- app/views/ibkr_items/_ibkr_item.html.erb | 114 +++++++ .../select_existing_account.html.erb | 39 +++ app/views/ibkr_items/setup_accounts.html.erb | 174 +++++++++++ .../settings/providers/_ibkr_panel.html.erb | 152 ++++++++++ app/views/trades/_trade.html.erb | 21 +- app/views/transactions/_transaction.html.erb | 11 +- config/locales/views/ibkr_items/en.yml | 84 ++++++ config/locales/views/settings/en.yml | 56 ++++ config/routes.rb | 14 + ...12210600_create_ibkr_items_and_accounts.rb | 41 +++ .../20260512211000_add_extra_to_trades.rb | 6 + ...equity_summary_payload_to_ibkr_accounts.rb | 5 + db/schema.rb | 44 ++- .../controllers/ibkr_items_controller_test.rb | 100 +++++++ .../settings/providers_controller_test.rb | 31 ++ test/fixtures/files/ibkr/flex_statement.xml | 45 +++ test/fixtures/ibkr_accounts.yml | 27 ++ test/fixtures/ibkr_items.yml | 15 + .../account/opening_balance_manager_test.rb | 69 +++++ .../account/provider_import_adapter_test.rb | 57 +++- test/models/account/syncer_test.rb | 31 ++ test/models/account_ibkr_creation_test.rb | 30 ++ test/models/account_test.rb | 51 ++++ test/models/balance/sync_cache_test.rb | 56 ++++ test/models/holding/materializer_test.rb | 80 +++++ test/models/holding/portfolio_cache_test.rb | 36 +++ test/models/holding_test.rb | 16 + .../historical_balances_sync_test.rb | 116 ++++++++ test/models/ibkr_account_processor_test.rb | 281 ++++++++++++++++++ .../ibkr_item/sync_complete_event_test.rb | 23 ++ test/models/ibkr_item/syncer_test.rb | 26 ++ test/models/ibkr_item_importer_test.rb | 42 +++ test/models/ibkr_item_report_parser_test.rb | 58 ++++ test/models/ibkr_item_test.rb | 20 ++ .../snaptrade_account_processor_test.rb | 30 ++ test/models/trade_test.rb | 24 ++ .../activity_security_preloader_test.rb | 28 ++ test/models/transaction_test.rb | 40 +++ 81 files changed, 3838 insertions(+), 59 deletions(-) create mode 100644 app/controllers/ibkr_items_controller.rb create mode 100644 app/models/family/ibkr_connectable.rb create mode 100644 app/models/ibkr_account.rb create mode 100644 app/models/ibkr_account/activities_processor.rb create mode 100644 app/models/ibkr_account/data_helpers.rb create mode 100644 app/models/ibkr_account/historical_balances_sync.rb create mode 100644 app/models/ibkr_account/holdings_processor.rb create mode 100644 app/models/ibkr_account/processor.rb create mode 100644 app/models/ibkr_item.rb create mode 100644 app/models/ibkr_item/importer.rb create mode 100644 app/models/ibkr_item/provided.rb create mode 100644 app/models/ibkr_item/report_parser.rb create mode 100644 app/models/ibkr_item/sync_complete_event.rb create mode 100644 app/models/ibkr_item/syncer.rb create mode 100644 app/models/ibkr_item/unlinking.rb create mode 100644 app/models/provider/ibkr_adapter.rb create mode 100644 app/models/provider/ibkr_flex.rb create mode 100644 app/models/transaction/activity_security_preloader.rb create mode 100644 app/views/ibkr_items/_ibkr_item.html.erb create mode 100644 app/views/ibkr_items/select_existing_account.html.erb create mode 100644 app/views/ibkr_items/setup_accounts.html.erb create mode 100644 app/views/settings/providers/_ibkr_panel.html.erb create mode 100644 config/locales/views/ibkr_items/en.yml create mode 100644 db/migrate/20260512210600_create_ibkr_items_and_accounts.rb create mode 100644 db/migrate/20260512211000_add_extra_to_trades.rb create mode 100644 db/migrate/20260512211200_add_raw_equity_summary_payload_to_ibkr_accounts.rb create mode 100644 test/controllers/ibkr_items_controller_test.rb create mode 100644 test/fixtures/files/ibkr/flex_statement.xml create mode 100644 test/fixtures/ibkr_accounts.yml create mode 100644 test/fixtures/ibkr_items.yml create mode 100644 test/models/account/syncer_test.rb create mode 100644 test/models/account_ibkr_creation_test.rb create mode 100644 test/models/ibkr_account/historical_balances_sync_test.rb create mode 100644 test/models/ibkr_account_processor_test.rb create mode 100644 test/models/ibkr_item/sync_complete_event_test.rb create mode 100644 test/models/ibkr_item/syncer_test.rb create mode 100644 test/models/ibkr_item_importer_test.rb create mode 100644 test/models/ibkr_item_report_parser_test.rb create mode 100644 test/models/ibkr_item_test.rb create mode 100644 test/models/transaction/activity_security_preloader_test.rb diff --git a/app/components/UI/account/activity_date.html.erb b/app/components/UI/account/activity_date.html.erb index a1a330588..f563801ab 100644 --- a/app/components/UI/account/activity_date.html.erb +++ b/app/components/UI/account/activity_date.html.erb @@ -20,7 +20,7 @@
- <%= end_balance_money.format %> + <%= end_balance_money.format %> <%= render DS::Tooltip.new(text: "The end of day balance, after all transactions and adjustments", placement: "left", size: "sm") %>
<%= helpers.icon "chevron-down", class: "group-open:rotate-180" %> diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index efe9bec08..4b4f34a31 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -20,6 +20,7 @@ class AccountsController < ApplicationController @mercury_items = visible_provider_items(family.mercury_items.ordered.includes(:syncs, :mercury_accounts)) @coinbase_items = visible_provider_items(family.coinbase_items.ordered.includes(:coinbase_accounts, :accounts, :syncs)) @snaptrade_items = visible_provider_items(family.snaptrade_items.ordered.includes(:syncs, :snaptrade_accounts)) + @ibkr_items = visible_provider_items(family.ibkr_items.ordered.includes(:syncs, :ibkr_accounts)) @indexa_capital_items = visible_provider_items(family.indexa_capital_items.ordered.includes(:syncs, :indexa_capital_accounts)) @sophtron_items = visible_provider_items(family.sophtron_items.ordered.includes(:syncs, :sophtron_accounts)) @@ -47,13 +48,14 @@ class AccountsController < ApplicationController @chart_view = params[:chart_view] || "balance" @tab = params[:tab] @q = params.fetch(:q, {}).permit(:search, status: []) - entries = @account.entries.where(excluded: false).search(@q).reverse_chronological + entries = @account.entries.where(excluded: false).search(@q).reverse_chronological.includes(:entryable) @pagy, @entries = pagy( entries, limit: safe_per_page, params: request.query_parameters.except("tab").merge("tab" => "activity") ) + Transaction::ActivitySecurityPreloader.new(@entries).preload @activity_feed_data = Account::ActivityFeedData.new(@account, @entries) end diff --git a/app/controllers/ibkr_items_controller.rb b/app/controllers/ibkr_items_controller.rb new file mode 100644 index 000000000..34936995b --- /dev/null +++ b/app/controllers/ibkr_items_controller.rb @@ -0,0 +1,236 @@ +class IbkrItemsController < ApplicationController + before_action :set_ibkr_item, only: [ :update, :destroy, :sync, :setup_accounts, :complete_account_setup ] + before_action :require_admin!, only: [ :create, :select_accounts, :select_existing_account, :link_existing_account, :update, :destroy, :sync, :setup_accounts, :complete_account_setup ] + + def create + @ibkr_item = Current.family.ibkr_items.build(ibkr_item_params) + @ibkr_item.name ||= t("ibkr_items.defaults.name") + + if @ibkr_item.save + @ibkr_item.sync_later + + if turbo_frame_request? + flash.now[:notice] = t(".success") + render turbo_stream: [ + turbo_stream.replace( + "ibkr-providers-panel", + partial: "settings/providers/ibkr_panel" + ), + *flash_notification_stream_items + ] + else + redirect_to accounts_path, notice: t(".success"), status: :see_other + end + else + @error_message = @ibkr_item.errors.full_messages.join(", ") + + if turbo_frame_request? + render turbo_stream: turbo_stream.replace( + "ibkr-providers-panel", + partial: "settings/providers/ibkr_panel", + locals: { error_message: @error_message } + ), status: :unprocessable_entity + else + redirect_to settings_providers_path, alert: @error_message, status: :see_other + end + end + end + + def update + attrs = ibkr_item_params.to_h + attrs["query_id"] = @ibkr_item.query_id if attrs["query_id"].blank? + attrs["token"] = @ibkr_item.token if attrs["token"].blank? + + if @ibkr_item.update(attrs.merge(status: :good)) + @ibkr_item.sync_later unless @ibkr_item.syncing? + + if turbo_frame_request? + flash.now[:notice] = t(".success") + render turbo_stream: [ + turbo_stream.replace( + "ibkr-providers-panel", + partial: "settings/providers/ibkr_panel" + ), + *flash_notification_stream_items + ] + else + redirect_to accounts_path, notice: t(".success"), status: :see_other + end + else + @error_message = @ibkr_item.errors.full_messages.join(", ") + + if turbo_frame_request? + render turbo_stream: turbo_stream.replace( + "ibkr-providers-panel", + partial: "settings/providers/ibkr_panel", + locals: { error_message: @error_message } + ), status: :unprocessable_entity + else + redirect_to settings_providers_path, alert: @error_message, status: :see_other + end + end + end + + def destroy + begin + @ibkr_item.unlink_all!(dry_run: false) + rescue => e + Rails.logger.warn("IBKR unlink during destroy failed: #{e.class} - #{e.message}") + end + + @ibkr_item.destroy_later + redirect_to settings_providers_path, notice: t(".success"), status: :see_other + end + + def sync + @ibkr_item.sync_later unless @ibkr_item.syncing? + + respond_to do |format| + format.html { redirect_back_or_to accounts_path } + format.json { head :ok } + end + end + + def select_accounts + ibkr_item = current_ibkr_item + unless ibkr_item + redirect_to settings_providers_path, alert: t(".not_configured") + return + end + + redirect_to setup_accounts_ibkr_item_path(ibkr_item) + end + + def select_existing_account + @account = Current.family.accounts.find(params[:account_id]) + @available_ibkr_accounts = Current.family.ibkr_items + .includes(ibkr_accounts: { account_provider: :account }) + .flat_map(&:ibkr_accounts) + .select { |ibkr_account| ibkr_account.account_provider.nil? } + .sort_by { |ibkr_account| ibkr_account.updated_at || ibkr_account.created_at } + .reverse + + render :select_existing_account, layout: false + end + + def link_existing_account + account = Current.family.accounts.find_by(id: params[:account_id]) + ibkr_account = Current.family.ibkr_items + .joins(:ibkr_accounts) + .where(ibkr_accounts: { id: params[:ibkr_account_id] }) + .first + &.ibkr_accounts + &.find_by(id: params[:ibkr_account_id]) + + if account.blank? || ibkr_account.blank? + redirect_to settings_providers_path, alert: t(".not_found") + return + end + + if account.accountable_type != "Investment" || account.account_providers.any? || account.plaid_account_id.present? || account.simplefin_account_id.present? + redirect_to account_path(account), alert: t(".only_manual_investment") + return + end + + provider = nil + + ibkr_account.with_lock do + if ibkr_account.current_account.present? + redirect_to account_path(account), alert: t(".already_linked") + return + end + + provider = ibkr_account.ensure_account_provider!(account) + end + + raise "Failed to create AccountProvider link" unless provider + + begin + IbkrAccount::Processor.new(ibkr_account.reload).process + rescue => e + Rails.logger.error("Failed to process linked IBKR account #{ibkr_account.id}: #{e.class} - #{e.message}") + end + + ibkr_account.ibkr_item.sync_later unless ibkr_account.ibkr_item.syncing? + redirect_to account_path(account), notice: t(".success"), status: :see_other + rescue => e + Rails.logger.error("Failed to link existing IBKR account: #{e.class} - #{e.message}") + redirect_to settings_providers_path, alert: t(".failed"), status: :see_other + end + + def setup_accounts + @ibkr_accounts = @ibkr_item.ibkr_accounts.includes(account_provider: :account) + @linked_accounts = @ibkr_accounts.select { |ibkr_account| ibkr_account.current_account.present? } + @unlinked_accounts = @ibkr_accounts.reject { |ibkr_account| ibkr_account.current_account.present? } + + no_accounts = @linked_accounts.blank? && @unlinked_accounts.blank? + latest_sync = @ibkr_item.syncs.ordered.first + should_sync = latest_sync.nil? || !latest_sync.completed? + + if no_accounts && !@ibkr_item.syncing? && should_sync + @ibkr_item.sync_later + end + + @linkable_accounts = Current.family.accounts + .visible + .where(accountable_type: "Investment") + .left_joins(:account_providers) + .where(account_providers: { id: nil }) + .order(:name) + + @syncing = @ibkr_item.syncing? + @waiting_for_sync = no_accounts && @syncing + @no_accounts_found = no_accounts && !@syncing && @ibkr_item.last_synced_at.present? + end + + def complete_account_setup + selected_accounts = Array(params[:account_ids]).reject(&:blank?) + created_accounts = [] + + selected_accounts.each do |ibkr_account_id| + ibkr_account = @ibkr_item.ibkr_accounts.find_by(id: ibkr_account_id) + next unless ibkr_account + + ibkr_account.with_lock do + next if ibkr_account.current_account.present? + + account = Account.create_from_ibkr_account(ibkr_account) + ibkr_account.ensure_account_provider!(account) + created_accounts << account + end + + begin + IbkrAccount::Processor.new(ibkr_account.reload).process + rescue => e + Rails.logger.error("Failed to process IBKR account #{ibkr_account.id} after setup: #{e.class} - #{e.message}") + end + end + + @ibkr_item.update!(pending_account_setup: @ibkr_item.unlinked_accounts_count.positive?) + @ibkr_item.sync_later if created_accounts.any? + + if created_accounts.any? + redirect_to accounts_path, notice: t(".success", count: created_accounts.count), status: :see_other + elsif selected_accounts.empty? + redirect_to setup_accounts_ibkr_item_path(@ibkr_item), alert: t(".none_selected"), status: :see_other + else + redirect_to setup_accounts_ibkr_item_path(@ibkr_item), alert: t(".none_created"), status: :see_other + end + end + + private + + def set_ibkr_item + @ibkr_item = Current.family.ibkr_items.find(params[:id]) + end + + def current_ibkr_item + active_items = Current.family.ibkr_items.active + + active_items.syncable.ordered.first || active_items.ordered.first + end + + def ibkr_item_params + params.require(:ibkr_item).permit(:name, :query_id, :token) + end +end diff --git a/app/controllers/settings/providers_controller.rb b/app/controllers/settings/providers_controller.rb index a3feda136..b8852065f 100644 --- a/app/controllers/settings/providers_controller.rb +++ b/app/controllers/settings/providers_controller.rb @@ -191,6 +191,7 @@ class Settings::ProvidersController < ApplicationController { key: "binance", title: "Binance", turbo_id: "binance", partial: "binance_panel" }, { key: "kraken", title: "Kraken", turbo_id: "kraken", partial: "kraken_panel" }, { key: "snaptrade", title: "SnapTrade", turbo_id: "snaptrade", partial: "snaptrade_panel", auto_open: "manage" }, + { key: "ibkr", title: "Interactive Brokers", turbo_id: "ibkr", partial: "ibkr_panel" }, { key: "indexa_capital", title: "Indexa Capital", turbo_id: "indexa_capital", partial: "indexa_capital_panel" }, { key: "sophtron", title: "Sophtron", turbo_id: "sophtron", partial: "sophtron_panel" } ].freeze @@ -208,6 +209,7 @@ class Settings::ProvidersController < ApplicationController "binance" => "BinanceItem", "kraken" => "KrakenItem", "snaptrade" => "SnaptradeItem", + "ibkr" => "IbkrItem", "indexa_capital" => "IndexaCapitalItem", "sophtron" => "SophtronItem" }.freeze @@ -232,6 +234,8 @@ class Settings::ProvidersController < ApplicationController @kraken_items = Current.family.kraken_items.active.ordered when "snaptrade" @snaptrade_items = Current.family.snaptrade_items.includes(:snaptrade_accounts).ordered + when "ibkr" + @ibkr_items = Current.family.ibkr_items.ordered when "indexa_capital" @indexa_capital_items = Current.family.indexa_capital_items.ordered when "sophtron" @@ -257,6 +261,7 @@ class Settings::ProvidersController < ApplicationController @mercury_items = Current.family.mercury_items.active.ordered @coinbase_items = Current.family.coinbase_items.ordered # Coinbase panel needs name and sync info for status display @snaptrade_items = Current.family.snaptrade_items.ordered + @ibkr_items = Current.family.ibkr_items.ordered.select(:id) @indexa_capital_items = Current.family.indexa_capital_items.ordered.select(:id) @binance_items = Current.family.binance_items.active.ordered @kraken_items = Current.family.kraken_items.active.ordered @@ -286,6 +291,7 @@ class Settings::ProvidersController < ApplicationController "binance" => @binance_items, "kraken" => @kraken_items, "snaptrade" => @snaptrade_items, + "ibkr" => @ibkr_items, "indexa_capital" => @indexa_capital_items, "sophtron" => @sophtron_items } diff --git a/app/controllers/transactions_controller.rb b/app/controllers/transactions_controller.rb index d88ee1bcf..8ba56d8e1 100644 --- a/app/controllers/transactions_controller.rb +++ b/app/controllers/transactions_controller.rb @@ -27,6 +27,7 @@ class TransactionsController < ApplicationController ) @pagy, @transactions = pagy(base_scope, limit: safe_per_page) + Transaction::ActivitySecurityPreloader.new(@transactions).preload # Preload split parent data entry_ids = @transactions.map { |t| t.entry.id } diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index 84aba4b58..b4e927292 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -102,6 +102,9 @@ module SettingsHelper return { status: :warn, meta: t("settings.providers.meta.registration_needed") } end sync_based_summary(key) + when "ibkr" + return { status: :off } unless @ibkr_items&.any? + sync_based_summary(key) when "indexa_capital" return { status: :off } unless @indexa_capital_items&.any? sync_based_summary(key) diff --git a/app/javascript/controllers/time_series_chart_controller.js b/app/javascript/controllers/time_series_chart_controller.js index 98baffc17..d4f4988af 100644 --- a/app/javascript/controllers/time_series_chart_controller.js +++ b/app/javascript/controllers/time_series_chart_controller.js @@ -289,7 +289,7 @@ export default class extends Controller { .append("div") .attr( "class", - "bg-container text-sm font-sans absolute p-2 border border-secondary rounded-lg pointer-events-none opacity-0 top-0", + "bg-container text-sm font-sans absolute p-2 border border-secondary rounded-lg pointer-events-none opacity-0 top-0 privacy-sensitive", ); } diff --git a/app/models/account.rb b/app/models/account.rb index c5363ced4..13c06c58e 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -274,6 +274,29 @@ class Account < ApplicationRecord create_and_sync(attributes, skip_initial_sync: true) end + def create_from_ibkr_account(ibkr_account) + family = ibkr_account.ibkr_item.family + default_name = if ibkr_account.ibkr_account_id.present? + "Interactive Brokers (#{ibkr_account.ibkr_account_id})" + else + "Interactive Brokers" + end + + attributes = { + family: family, + name: default_name, + balance: 0, + cash_balance: 0, + currency: ibkr_account.currency.presence || family.currency, + accountable_type: "Investment", + accountable_attributes: { + subtype: "brokerage" + } + } + + create_and_sync(attributes, skip_initial_sync: true) + end + def create_from_kraken_account(kraken_account) family = kraken_account.kraken_item.family @@ -293,7 +316,6 @@ class Account < ApplicationRecord create_and_sync(attributes, skip_initial_sync: true) end - private def build_simplefin_accountable_attributes(simplefin_account, account_type, subtype) @@ -363,15 +385,23 @@ class Account < ApplicationRecord end def current_holdings - holdings - .where(currency: currency) - .where.not(qty: 0) - .where( - id: holdings.select("DISTINCT ON (security_id) id") - .where(currency: currency) - .order(:security_id, date: :desc) - ) - .order(amount: :desc) + if (provider_snapshot_date = latest_provider_holdings_snapshot_date) + holdings + .where.not(account_provider_id: nil) + .where(date: provider_snapshot_date) + .where.not(qty: 0) + .order(amount: :desc) + else + holdings + .where(currency: currency) + .where.not(qty: 0) + .where( + id: holdings.select("DISTINCT ON (security_id) id") + .where(currency: currency) + .order(:security_id, date: :desc) + ) + .order(amount: :desc) + end end def latest_provider_holdings_snapshot_date diff --git a/app/models/account/opening_balance_manager.rb b/app/models/account/opening_balance_manager.rb index 95597cdaa..3ad6818e0 100644 --- a/app/models/account/opening_balance_manager.rb +++ b/app/models/account/opening_balance_manager.rb @@ -51,7 +51,11 @@ class Account::OpeningBalanceManager end def oldest_entry_date - @oldest_entry_date ||= account.entries.minimum(:date) + if opening_anchor_valuation&.entry + account.entries.where.not(id: opening_anchor_valuation.entry.id).minimum(:date) + else + account.entries.minimum(:date) + end end def default_date diff --git a/app/models/account/provider_import_adapter.rb b/app/models/account/provider_import_adapter.rb index 633c26f1b..a207468aa 100644 --- a/app/models/account/provider_import_adapter.rb +++ b/app/models/account/provider_import_adapter.rb @@ -551,8 +551,9 @@ class Account::ProviderImportAdapter # @param external_id [String, nil] Provider's unique ID (optional, for deduplication) # @param source [String] Provider name # @param activity_label [String, nil] Investment activity label (e.g., "Buy", "Sell", "Reinvestment") + # @param exchange_rate [BigDecimal, Numeric, nil] Optional provider-supplied FX rate into the account currency # @return [Entry] The created entry with trade - def import_trade(security:, quantity:, price:, amount:, currency:, date:, name: nil, external_id: nil, source:, activity_label: nil) + def import_trade(security:, quantity:, price:, amount:, currency:, date:, name: nil, external_id: nil, source:, activity_label: nil, exchange_rate: nil) raise ArgumentError, "security is required" if security.nil? raise ArgumentError, "source is required" if source.blank? @@ -585,13 +586,16 @@ class Account::ProviderImportAdapter end # Always update Trade attributes (works for both new and existing records) - entry.entryable.assign_attributes( + trade_attributes = { security: security, qty: quantity, price: price, currency: currency, investment_activity_label: activity_label || (quantity > 0 ? "Buy" : "Sell") - ) + } + trade_attributes[:exchange_rate] = exchange_rate unless exchange_rate.nil? + + entry.entryable.assign_attributes(trade_attributes) entry.assign_attributes( date: date, diff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rb index 3b6a4c9b0..874091f81 100644 --- a/app/models/account/syncer.rb +++ b/app/models/account/syncer.rb @@ -9,6 +9,7 @@ class Account::Syncer Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})") import_market_data materialize_balances(window_start_date: sync.window_start_date) + apply_provider_balance_overrides end def perform_post_sync @@ -34,4 +35,16 @@ class Account::Syncer Rails.logger.error("Error syncing market data for account #{account.id}: #{e.message}") Sentry.capture_exception(e) end + + def apply_provider_balance_overrides + return unless account.linked_to?("IbkrAccount") + + ibkr_account = account.account_providers.find_by(provider_type: "IbkrAccount")&.provider + return unless ibkr_account + + IbkrAccount::HistoricalBalancesSync.new(ibkr_account).sync! + rescue => e + Rails.logger.error("Error syncing IBKR historical balances for account #{account.id}: #{e.class} - #{e.message}") + Sentry.capture_exception(e) + end end diff --git a/app/models/balance/sync_cache.rb b/app/models/balance/sync_cache.rb index 1f1a93a68..58caf70ad 100644 --- a/app/models/balance/sync_cache.rb +++ b/app/models/balance/sync_cache.rb @@ -37,10 +37,7 @@ class Balance::SyncCache @converted_entries ||= account.entries.excluding_split_parents.includes(:entryable).order(:date).to_a.map do |e| converted_entry = e.dup - # Extract custom exchange rate if present on Transaction - custom_rate = if e.entryable.is_a?(Transaction) - e.entryable.extra&.dig("exchange_rate") - end + custom_rate = e.entryable.exchange_rate if e.entryable.respond_to?(:exchange_rate) # Use Money#exchange_to with custom rate if available, standard lookup otherwise converted_entry.amount = converted_entry.amount_money.exchange_to( diff --git a/app/models/data_enrichment.rb b/app/models/data_enrichment.rb index db09f81b5..817b05fb2 100644 --- a/app/models/data_enrichment.rb +++ b/app/models/data_enrichment.rb @@ -1,5 +1,18 @@ class DataEnrichment < ApplicationRecord belongs_to :enrichable, polymorphic: true - enum :source, { rule: "rule", plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats", mercury: "mercury", indexa_capital: "indexa_capital", sophtron: "sophtron" } + enum :source, { + rule: "rule", + plaid: "plaid", + simplefin: "simplefin", + lunchflow: "lunchflow", + synth: "synth", + ai: "ai", + enable_banking: "enable_banking", + coinstats: "coinstats", + mercury: "mercury", + indexa_capital: "indexa_capital", + sophtron: "sophtron", + ibkr: "ibkr" + } end diff --git a/app/models/family.rb b/app/models/family.rb index fa7d1222d..7ebec6c5d 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -2,7 +2,7 @@ class Family < ApplicationRecord include Syncable, AutoTransferMatchable, Subscribeable, VectorSearchable include PlaidConnectable, SimplefinConnectable, LunchflowConnectable, EnableBankingConnectable include CoinbaseConnectable, BinanceConnectable, KrakenConnectable, CoinstatsConnectable, SnaptradeConnectable, MercuryConnectable, SophtronConnectable - include IndexaCapitalConnectable + include IndexaCapitalConnectable, IbkrConnectable DATE_FORMATS = [ [ "MM-DD-YYYY", "%m-%d-%Y" ], diff --git a/app/models/family/ibkr_connectable.rb b/app/models/family/ibkr_connectable.rb new file mode 100644 index 000000000..bebfcb1e8 --- /dev/null +++ b/app/models/family/ibkr_connectable.rb @@ -0,0 +1,22 @@ +module Family::IbkrConnectable + extend ActiveSupport::Concern + + included do + has_many :ibkr_items, dependent: :destroy + end + + def can_connect_ibkr? + true + end + + def create_ibkr_item!(query_id:, token:, item_name: nil) + ibkr_item = ibkr_items.create!( + name: item_name.presence || "Interactive Brokers", + query_id: query_id, + token: token + ) + + ibkr_item.sync_later + ibkr_item + end +end diff --git a/app/models/holding.rb b/app/models/holding.rb index 0b18b3e9c..ac8c37c26 100644 --- a/app/models/holding.rb +++ b/app/models/holding.rb @@ -38,7 +38,7 @@ class Holding < ApplicationRecord return nil unless amount return 0 if amount.zero? - account.balance.zero? ? 1 : amount / account.balance * 100 + account.balance.zero? ? 1 : amount_in_account_currency / account.balance * 100 end # Returns average cost per share, or nil if unknown. @@ -256,6 +256,14 @@ class Holding < ApplicationRecord end private + def amount_in_account_currency + return amount if currency == account.currency + + Money.new(amount, currency).exchange_to(account.currency, date: date).amount + rescue Money::ConversionError + amount + end + def calculate_trend return nil unless amount_money return nil if avg_cost.nil? # Can't calculate trend without cost basis (0 is valid for airdrops) diff --git a/app/models/holding/materializer.rb b/app/models/holding/materializer.rb index 522508599..5832e2033 100644 --- a/app/models/holding/materializer.rb +++ b/app/models/holding/materializer.rb @@ -22,10 +22,10 @@ class Holding::Materializer # securities are still needed to derive sane balance charts between sync snapshots. cleanup_shadowed_calculated_holdings - # Also remove calculated rows on the provider's latest snapshot date when those - # securities are no longer present in the provider payload. This keeps "current" - # holdings/balance composition aligned with the provider snapshot while preserving - # older calculated history. + # Also remove non-provider rows on the provider's latest snapshot date for securities + # that appear in the provider snapshot. The provider snapshot is authoritative for + # those securities on that day, even when it is denominated in a different currency + # than the account or the reverse-calculated holdings. cleanup_stale_calculated_rows_on_latest_provider_snapshot # Reload holdings association to clear any cached stale data @@ -152,17 +152,12 @@ class Holding::Materializer .where(date: provider_snapshot_date) .distinct .pluck(:security_id) + return if provider_security_ids.empty? - scope = account.holdings - .where(account_provider_id: nil, date: provider_snapshot_date) + deleted_count = account.holdings + .where(account_provider_id: nil, date: provider_snapshot_date, security_id: provider_security_ids) + .delete_all - scope = if provider_security_ids.any? - scope.where.not(security_id: provider_security_ids) - else - scope - end - - deleted_count = scope.delete_all Rails.logger.info("Cleaned up #{deleted_count} stale calculated holdings on latest provider snapshot date") if deleted_count > 0 end diff --git a/app/models/holding/portfolio_cache.rb b/app/models/holding/portfolio_cache.rb index ca6febe2e..4ea52920e 100644 --- a/app/models/holding/portfolio_cache.rb +++ b/app/models/holding/portfolio_cache.rb @@ -40,7 +40,7 @@ class Holding::PortfolioCache price_money = Money.new(price.price, price.currency) begin - converted_amount = price_money.exchange_to(account.currency).amount + converted_amount = price_money.exchange_to(account.currency, date: date).amount rescue Money::ConversionError converted_amount = price.price end diff --git a/app/models/ibkr_account.rb b/app/models/ibkr_account.rb new file mode 100644 index 000000000..eb491f96a --- /dev/null +++ b/app/models/ibkr_account.rb @@ -0,0 +1,78 @@ +class IbkrAccount < ApplicationRecord + include CurrencyNormalizable, Encryptable + include IbkrAccount::DataHelpers + + if encryption_ready? + encrypts :raw_holdings_payload + encrypts :raw_activities_payload + encrypts :raw_cash_report_payload + encrypts :raw_equity_summary_payload + end + + belongs_to :ibkr_item + + has_one :account_provider, as: :provider, dependent: :destroy + has_one :account, through: :account_provider, source: :account + has_one :linked_account, through: :account_provider, source: :account + + validates :name, :currency, presence: true + validates :ibkr_account_id, uniqueness: { scope: :ibkr_item_id, allow_nil: true } + + def current_account + account || linked_account + end + + def ensure_account_provider!(account = nil) + if account_provider.present? + account_provider.update!(account: account) if account && account_provider.account_id != account.id + return account_provider + end + + acct = account || current_account + return nil unless acct + + provider = AccountProvider + .find_or_initialize_by(provider_type: "IbkrAccount", provider_id: id) + .tap do |record| + record.account = acct + record.save! + end + + reload_account_provider + provider + rescue => e + Rails.logger.warn("IbkrAccount##{id}: failed to ensure AccountProvider link: #{e.class} - #{e.message}") + nil + end + + def upsert_from_ibkr_statement!(account_data) + data = account_data.with_indifferent_access + + update!( + ibkr_account_id: data[:ibkr_account_id], + name: data[:name], + currency: parse_currency(data[:currency]) || "USD", + current_balance: data[:current_balance], + cash_balance: data[:cash_balance], + institution_metadata: { + provider_name: "Interactive Brokers", + statement_from_date: data.dig(:statement, :from_date), + statement_to_date: data.dig(:statement, :to_date) + }.compact, + report_date: data[:report_date], + raw_holdings_payload: data[:open_positions] || [], + raw_activities_payload: { + trades: data[:trades] || [], + cash_transactions: data[:cash_transactions] || [] + }, + raw_cash_report_payload: data[:cash_report] || [], + raw_equity_summary_payload: data[:equity_summary_in_base] || [], + last_holdings_sync: Time.current, + last_activities_sync: Time.current + ) + end + + def ibkr_provider + ibkr_item.ibkr_provider + end +end diff --git a/app/models/ibkr_account/activities_processor.rb b/app/models/ibkr_account/activities_processor.rb new file mode 100644 index 000000000..3b33c224d --- /dev/null +++ b/app/models/ibkr_account/activities_processor.rb @@ -0,0 +1,221 @@ +class IbkrAccount::ActivitiesProcessor + include IbkrAccount::DataHelpers + + SUPPORTED_CASH_TRANSACTION_TYPES = [ "DEPOSITS/WITHDRAWALS", "DIVIDENDS" ].freeze + + def initialize(ibkr_account) + @ibkr_account = ibkr_account + end + + def process + return { trades: 0, transactions: 0 } unless account.present? + + activities = (@ibkr_account.raw_activities_payload || {}).with_indifferent_access + trades = Array(activities[:trades]) + cash_transactions = Array(activities[:cash_transactions]) + @fee_transactions_count = 0 + + trades_count = trades.sum { |trade| process_trade(trade.with_indifferent_access) ? 1 : 0 } + cash_transactions_count = cash_transactions.sum { |cash_transaction| process_cash_transaction(cash_transaction.with_indifferent_access) ? 1 : 0 } + + { + trades: trades_count, + transactions: cash_transactions_count + @fee_transactions_count + } + end + + private + + def account + @ibkr_account.current_account + end + + def import_adapter + @import_adapter ||= Account::ProviderImportAdapter.new(account) + end + + def process_trade(row) + return false unless supported_trade?(row) + + security = resolve_security(row) + return false unless security + + quantity = parse_decimal(row[:quantity]) + native_price = parse_decimal(row[:trade_price]) + return false if quantity.nil? || native_price.nil? + + buy_sell = row[:buy_sell].to_s.upcase + signed_quantity = buy_sell == "SELL" ? -quantity.abs : quantity.abs + native_amount = buy_sell == "SELL" ? -(native_price * quantity.abs) : (native_price * quantity.abs) + currency = extract_currency(row, fallback: @ibkr_account.currency) + date = trade_date_for(row) + external_id = "ibkr_trade_#{row[:trade_id]}" + + import_adapter.import_trade( + external_id: external_id, + security: security, + quantity: signed_quantity, + price: native_price, + amount: native_amount, + currency: currency, + date: date, + name: build_trade_name(security.ticker, signed_quantity), + source: "ibkr", + activity_label: buy_sell == "SELL" ? "Sell" : "Buy", + exchange_rate: parse_decimal(row[:fx_rate_to_base])&.to_f + ) + + import_commission_transaction(row, security, date) + true + rescue => e + Rails.logger.error("IbkrAccount::ActivitiesProcessor - Failed to process trade #{row[:trade_id]}: #{e.message}") + false + end + + def process_cash_transaction(row) + return false unless supported_cash_transaction?(row) + + amount = parse_decimal(row[:amount]) + return false if amount.nil? || amount.zero? + + label, signed_amount = classify_cash_transaction(row, amount) + return false unless label + currency = extract_currency(row, fallback: @ibkr_account.currency) + security = resolve_security_for_cash_transaction(row) + + import_adapter.import_transaction( + external_id: "ibkr_cash_#{row[:transaction_id]}", + amount: signed_amount, + currency: currency, + date: parse_date(row[:report_date]), + name: build_cash_transaction_name(row, label, security), + source: "ibkr", + investment_activity_label: label, + extra: { + exchange_rate: parse_decimal(row[:fx_rate_to_base])&.to_f, + security_id: security&.id, + ibkr: { + transaction_id: row[:transaction_id], + type: row[:type], + conid: row[:conid], + amount: row[:amount], + currency: row[:currency], + fx_rate_to_base: row[:fx_rate_to_base], + report_date: row[:report_date] + }.compact + } + ) + + true + rescue => e + Rails.logger.error("IbkrAccount::ActivitiesProcessor - Failed to process cash transaction #{row[:transaction_id]}: #{e.message}") + false + end + + def import_commission_transaction(row, security, date) + commission = parse_decimal(row[:ib_commission]) + return if commission.nil? || commission.zero? + currency = row.with_indifferent_access[:ib_commission_currency].to_s.upcase.presence || @ibkr_account.currency + ticker = security&.ticker || row.with_indifferent_access[:symbol] + + result = import_adapter.import_transaction( + external_id: "ibkr_trade_fee_#{row[:trade_id]}", + amount: commission.abs, + currency: currency, + date: date, + name: "Trade Commission for #{ticker}", + source: "ibkr", + investment_activity_label: "Fee", + extra: { + exchange_rate: parse_decimal(row[:fx_rate_to_base])&.to_f, + security_id: security&.id, + ibkr: { + trade_id: row[:trade_id], + transaction_id: row[:transaction_id], + ib_commission: row[:ib_commission], + ib_commission_currency: row[:ib_commission_currency], + fx_rate_to_base: row[:fx_rate_to_base] + }.compact + } + ) + + @fee_transactions_count += 1 if result + end + + def build_trade_name(ticker, signed_quantity) + action = signed_quantity.negative? ? "Sell" : "Buy" + "#{action} #{signed_quantity.abs} shares of #{ticker}" + end + + def supported_trade?(row) + row[:asset_category].to_s == "STK" && + row[:buy_sell].present? && + row[:conid].present? && + row[:currency].present? && + row[:quantity].present? && + row[:symbol].present? && + row[:trade_date].present? && + row[:trade_id].present? && + row[:trade_price].present? && + row[:transaction_id].present? && + fx_rate_available?(row) + end + + def supported_cash_transaction?(row) + type = row[:type].to_s.upcase.strip + return false unless SUPPORTED_CASH_TRANSACTION_TYPES.include?(type) + return false unless row[:transaction_id].present? && row[:amount].present? && row[:currency].present? && row[:report_date].present? + return false unless fx_rate_available?(row) + + type != "DIVIDENDS" || row[:conid].present? + end + + def classify_cash_transaction(row, amount) + type = row[:type].to_s.upcase.strip + + case type + when "DEPOSITS/WITHDRAWALS" + amount.positive? ? [ "Contribution", -amount.abs ] : [ "Withdrawal", amount.abs ] + when "DIVIDENDS" + [ "Dividend", -amount.abs ] + else + [ nil, nil ] + end + end + + def build_cash_transaction_name(row, label, security = nil) + return label unless label == "Dividend" + + ticker = security&.ticker || security_symbol_for_conid(row[:conid]) || row[:conid] + "Dividend from #{ticker}" + end + + def resolve_security_for_cash_transaction(row) + symbol = security_symbol_for_conid(row[:conid]) + return nil if symbol.blank? + + resolve_security({ symbol: symbol }) + end + + def security_symbol_for_conid(conid) + return nil if conid.blank? + + holding_symbol = Array(@ibkr_account.raw_holdings_payload).find do |holding| + holding.with_indifferent_access[:conid].to_s == conid.to_s + end&.with_indifferent_access&.dig(:symbol) + return holding_symbol if holding_symbol.present? + + Array(@ibkr_account.raw_activities_payload&.dig("trades") || @ibkr_account.raw_activities_payload&.dig(:trades)).find do |trade| + trade.with_indifferent_access[:conid].to_s == conid.to_s + end&.with_indifferent_access&.dig(:symbol) + end + + + def fx_rate_available?(row) + source_currency = extract_currency(row, fallback: nil) + return false if source_currency.blank? + return true if source_currency == @ibkr_account.currency + + row[:fx_rate_to_base].present? + end +end diff --git a/app/models/ibkr_account/data_helpers.rb b/app/models/ibkr_account/data_helpers.rb new file mode 100644 index 000000000..c3416d74b --- /dev/null +++ b/app/models/ibkr_account/data_helpers.rb @@ -0,0 +1,78 @@ +module IbkrAccount::DataHelpers + extend ActiveSupport::Concern + + private + + def parse_decimal(value) + return nil if value.nil? + + normalized = value.is_a?(String) ? value.delete(",").strip : value.to_s + return nil if normalized.blank? || normalized == "-" + + BigDecimal(normalized) + rescue ArgumentError + nil + end + + def parse_date(value) + return nil if value.blank? + + case value + when Date + value + when Time, DateTime, ActiveSupport::TimeWithZone + value.to_date + else + normalized = value.to_s.tr(";", " ") + Time.zone.parse(normalized)&.to_date || Date.parse(normalized) + end + rescue ArgumentError, TypeError + nil + end + + def parse_datetime(value) + return nil if value.blank? + + case value + when Time, DateTime, ActiveSupport::TimeWithZone + value.in_time_zone + when Date + value.in_time_zone + else + Time.zone.parse(value.to_s.tr(";", " ")) + end + rescue ArgumentError, TypeError + nil + end + + def resolve_security(row) + data = row.with_indifferent_access + ticker = data[:symbol].to_s.strip.upcase + return nil if ticker.blank? + + Security.find_by(ticker: ticker) || create_security_from_row(ticker) + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique + Security.find_by(ticker: ticker) + end + + def trade_date_for(row) + data = row.with_indifferent_access + parsed_trade_date = parse_date(data[:trade_date]) + return parsed_trade_date if parsed_trade_date + + Rails.logger.warn( + "IbkrAccount::DataHelpers - Missing or invalid trade_date, falling back to Date.current. " \ + "trade_id=#{data[:trade_id].inspect}" + ) + Date.current + end + + def extract_currency(row, fallback: nil) + value = row.with_indifferent_access[:currency] + value.present? ? value.to_s.upcase : fallback + end + + def create_security_from_row(ticker) + Security.create!(ticker: ticker, name: ticker) + end +end diff --git a/app/models/ibkr_account/historical_balances_sync.rb b/app/models/ibkr_account/historical_balances_sync.rb new file mode 100644 index 000000000..a7a0bc363 --- /dev/null +++ b/app/models/ibkr_account/historical_balances_sync.rb @@ -0,0 +1,79 @@ +class IbkrAccount::HistoricalBalancesSync + include IbkrAccount::DataHelpers + + attr_reader :ibkr_account + + def initialize(ibkr_account) + @ibkr_account = ibkr_account + end + + def sync! + return unless account.present? + return if normalized_rows.empty? + + account.balances.upsert_all( + balance_rows, + unique_by: %i[account_id date currency] + ) + end + + private + def account + ibkr_account.current_account + end + + def normalized_rows + @normalized_rows ||= Array(ibkr_account.raw_equity_summary_payload) + .filter_map do |row| + next unless row.is_a?(Hash) + + data = row.with_indifferent_access + currency = data[:currency].presence&.upcase + account_currency = ibkr_account.currency.to_s.upcase + next if currency.present? && currency != account_currency + + date = parse_date(data[:report_date]) + total = parse_decimal(data[:total]) + cash = parse_decimal(data[:cash]) || BigDecimal("0") + next unless date && total + + { + date: date, + total: total, + cash: cash, + non_cash: total - cash + } + end + .sort_by { |row| row[:date] } + end + + def balance_rows + current_time = Time.current + + normalized_rows.each_with_index.map do |row, index| + previous_row = index.zero? ? nil : normalized_rows[index - 1] + start_cash_balance = previous_row ? previous_row[:cash] : row[:cash] + start_non_cash_balance = previous_row ? previous_row[:non_cash] : row[:non_cash] + + { + account_id: account.id, + date: row[:date], + balance: row[:total], + cash_balance: row[:cash], + currency: account.currency, + start_cash_balance: start_cash_balance, + start_non_cash_balance: start_non_cash_balance, + cash_inflows: 0, + cash_outflows: 0, + non_cash_inflows: 0, + non_cash_outflows: 0, + net_market_flows: 0, + cash_adjustments: row[:cash] - start_cash_balance, + non_cash_adjustments: row[:non_cash] - start_non_cash_balance, + flows_factor: 1, + created_at: current_time, + updated_at: current_time + } + end + end +end diff --git a/app/models/ibkr_account/holdings_processor.rb b/app/models/ibkr_account/holdings_processor.rb new file mode 100644 index 000000000..9ff657f3f --- /dev/null +++ b/app/models/ibkr_account/holdings_processor.rb @@ -0,0 +1,105 @@ +class IbkrAccount::HoldingsProcessor + include IbkrAccount::DataHelpers + + def initialize(ibkr_account) + @ibkr_account = ibkr_account + end + + def process + return unless account.present? + + grouped_positions.each_value do |group| + process_group(group) + end + end + + private + + def account + @ibkr_account.current_account + end + + def import_adapter + @import_adapter ||= Account::ProviderImportAdapter.new(account) + end + + def grouped_positions + Array(@ibkr_account.raw_holdings_payload).each_with_object({}) do |position, groups| + data = position.with_indifferent_access + next unless supported_position?(data) + + symbol_key = data[:conid].presence || data[:symbol].presence || data[:security_id].presence + currency = extract_currency(data, fallback: @ibkr_account.currency) + report_date = parse_date(data[:report_date]) || @ibkr_account.report_date || Date.current + key = [ symbol_key, currency, report_date ] + groups[key] ||= [] + groups[key] << data + end + end + + def process_group(rows) + sample = rows.first + security = resolve_security(sample) + return unless security + + quantity = rows.sum { |row| parse_decimal(row[:position]) || BigDecimal("0") } + return if quantity.zero? + + price = parse_decimal(sample[:mark_price]) + cost_basis = weighted_cost_basis_for(rows) + return unless price && cost_basis + + amount = quantity.abs * price + + currency = extract_currency(sample, fallback: @ibkr_account.currency) + report_date = parse_date(sample[:report_date]) || @ibkr_account.report_date || Date.current + external_id = [ "ibkr", @ibkr_account.ibkr_account_id, sample[:conid].presence || security.ticker, report_date, currency ].join("_") + + import_adapter.import_holding( + security: security, + quantity: quantity, + amount: amount, + currency: currency, + date: report_date, + price: price || BigDecimal("0"), + cost_basis: cost_basis, + external_id: external_id, + source: "ibkr", + account_provider_id: @ibkr_account.account_provider&.id, + delete_future_holdings: false + ) + end + + def weighted_cost_basis_for(rows) + total_quantity = BigDecimal("0") + total_cost = BigDecimal("0") + + rows.each do |row| + row_quantity = parse_decimal(row[:position]) + row_cost_basis = parse_decimal(row[:cost_basis_price]) + return nil unless row_quantity && row_cost_basis + + total_quantity += row_quantity.abs + total_cost += row_quantity.abs * row_cost_basis + end + + return nil if total_quantity.zero? + + total_cost / total_quantity + end + + def supported_position?(row) + row[:asset_category].to_s == "STK" && + row[:side].to_s == "Long" && + row[:conid].present? && + row[:security_id].present? && + row[:security_id_type].present? && + row[:symbol].present? && + row[:currency].present? && + row[:fx_rate_to_base].present? && + row[:position].present? && + row[:mark_price].present? && + row[:cost_basis_price].present? && + row[:report_date].present? + end +end diff --git a/app/models/ibkr_account/processor.rb b/app/models/ibkr_account/processor.rb new file mode 100644 index 000000000..2fa30e4e2 --- /dev/null +++ b/app/models/ibkr_account/processor.rb @@ -0,0 +1,56 @@ +class IbkrAccount::Processor + attr_reader :ibkr_account + + def initialize(ibkr_account) + @ibkr_account = ibkr_account + end + + def process + return unless ibkr_account.current_account.present? + + update_account_balance! + IbkrAccount::HoldingsProcessor.new(ibkr_account).process + IbkrAccount::ActivitiesProcessor.new(ibkr_account).process + repair_default_opening_anchor! + + ibkr_account.current_account.broadcast_sync_complete + end + + private + + def update_account_balance! + account = ibkr_account.current_account + + total_balance = ibkr_account.current_balance || ibkr_account.cash_balance || 0 + cash_balance = ibkr_account.cash_balance || 0 + + account.assign_attributes( + balance: total_balance, + cash_balance: cash_balance, + currency: ibkr_account.currency + ) + account.save! + account.set_current_balance(total_balance) + end + + def repair_default_opening_anchor! + account = ibkr_account.current_account + return unless account&.linked_to?("IbkrAccount") + return unless account.has_opening_anchor? + + opening_anchor_entry = account.valuations.opening_anchor.includes(:entry).first&.entry + return unless opening_anchor_entry + return unless opening_anchor_entry.created_at.to_date == account.created_at.to_date + return unless account.entries.where.not(entryable_type: "Valuation").exists? + + imported_current_balance = (ibkr_account.current_balance || ibkr_account.cash_balance || 0).to_d + return unless opening_anchor_entry.amount.to_d == imported_current_balance + + result = Account::OpeningBalanceManager.new(account).set_opening_balance( + balance: 0, + date: opening_anchor_entry.date + ) + + raise result.error if result.error + end +end diff --git a/app/models/ibkr_item.rb b/app/models/ibkr_item.rb new file mode 100644 index 000000000..aca60d1b2 --- /dev/null +++ b/app/models/ibkr_item.rb @@ -0,0 +1,124 @@ +class IbkrItem < ApplicationRecord + include Syncable, Provided, Unlinking, Encryptable + + enum :status, { good: "good", requires_update: "requires_update" }, default: :good + + if encryption_ready? + encrypts :query_id, deterministic: true + encrypts :token + encrypts :raw_payload + end + + belongs_to :family + has_one_attached :logo, dependent: :purge_later + + has_many :ibkr_accounts, dependent: :destroy + + validates :name, presence: true + validates :query_id, presence: true, on: :create + validates :token, presence: true, on: :create + + scope :active, -> { where(scheduled_for_deletion: false) } + scope :syncable, -> { active.where.not(query_id: [ nil, "" ]).where.not(token: nil) } + scope :ordered, -> { order(created_at: :desc) } + scope :needs_update, -> { where(status: :requires_update) } + + def destroy_later + update!(scheduled_for_deletion: true) + DestroyJob.perform_later(self) + end + + def credentials_configured? + query_id.present? && token.present? + end + + def import_latest_ibkr_data + provider = ibkr_provider + raise StandardError, "IBKR provider is not configured" unless provider + + IbkrItem::Importer.new(self, ibkr_provider: provider).import + rescue => e + Rails.logger.error("IbkrItem #{id} - Failed to import data: #{e.message}") + raise + end + + def process_accounts + return [] if ibkr_accounts.empty? + + linked_ibkr_accounts.includes(account_provider: :account).each_with_object([]) do |ibkr_account, results| + account = ibkr_account.current_account + next unless account + next if account.pending_deletion? || account.disabled? + + begin + result = IbkrAccount::Processor.new(ibkr_account).process + results << { ibkr_account_id: ibkr_account.id, success: true, result: result } + rescue => e + Rails.logger.error("IbkrItem #{id} - Failed to process account #{ibkr_account.id}: #{e.message}") + results << { ibkr_account_id: ibkr_account.id, success: false, error: e.message } + end + end + end + + def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) + accounts.reject { |account| account.pending_deletion? || account.disabled? }.each_with_object([]) do |account, results| + begin + account.sync_later( + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: window_end_date + ) + results << { account_id: account.id, success: true } + rescue => e + Rails.logger.error("IbkrItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}") + results << { account_id: account.id, success: false, error: e.message } + end + end + end + + def upsert_ibkr_snapshot!(payload) + update!(raw_payload: payload, status: :good) + end + + def accounts + ibkr_accounts.includes(account_provider: :account).filter_map(&:current_account).uniq + end + + def linked_ibkr_accounts + ibkr_accounts.joins(:account_provider) + end + + def linked_accounts_count + ibkr_accounts.joins(:account_provider).count + end + + def unlinked_accounts_count + ibkr_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).count + end + + def total_accounts_count + ibkr_accounts.count + end + + def has_completed_initial_setup? + accounts.any? + end + + def sync_status_summary + total_accounts = total_accounts_count + linked_count = linked_accounts_count + unlinked_count = unlinked_accounts_count + + if total_accounts.zero? + I18n.t("ibkr_items.sync_status.no_accounts") + elsif unlinked_count.zero? + I18n.t("ibkr_items.sync_status.all_linked", count: linked_count) + else + I18n.t("ibkr_items.sync_status.partial", linked: linked_count, unlinked: unlinked_count) + end + end + + def institution_display_name + I18n.t("ibkr_items.defaults.name") + end +end diff --git a/app/models/ibkr_item/importer.rb b/app/models/ibkr_item/importer.rb new file mode 100644 index 000000000..30a5916da --- /dev/null +++ b/app/models/ibkr_item/importer.rb @@ -0,0 +1,33 @@ +class IbkrItem::Importer + attr_reader :ibkr_item, :ibkr_provider + + def initialize(ibkr_item, ibkr_provider:) + @ibkr_item = ibkr_item + @ibkr_provider = ibkr_provider + end + + def import + xml_body = ibkr_provider.download_statement + parsed_report = IbkrItem::ReportParser.new(xml_body).parse + + accounts_imported = 0 + ibkr_item.transaction do + ibkr_item.upsert_ibkr_snapshot!(parsed_report[:metadata].merge("fetched_at" => Time.current.iso8601)) + + parsed_report[:accounts].each do |account_data| + next if account_data[:ibkr_account_id].blank? + + ibkr_account = ibkr_item.ibkr_accounts.find_or_initialize_by(ibkr_account_id: account_data[:ibkr_account_id]) + ibkr_account.upsert_from_ibkr_statement!(account_data) + accounts_imported += 1 + end + + ibkr_item.update!(status: :good) + end + + { + success: true, + accounts_imported: accounts_imported + } + end +end diff --git a/app/models/ibkr_item/provided.rb b/app/models/ibkr_item/provided.rb new file mode 100644 index 000000000..55c6d43fb --- /dev/null +++ b/app/models/ibkr_item/provided.rb @@ -0,0 +1,9 @@ +module IbkrItem::Provided + extend ActiveSupport::Concern + + def ibkr_provider + return nil unless credentials_configured? + + Provider::IbkrFlex.new(query_id: query_id, token: token) + end +end diff --git a/app/models/ibkr_item/report_parser.rb b/app/models/ibkr_item/report_parser.rb new file mode 100644 index 000000000..edab81812 --- /dev/null +++ b/app/models/ibkr_item/report_parser.rb @@ -0,0 +1,143 @@ +class IbkrItem::ReportParser + include IbkrAccount::DataHelpers + + class ParseError < StandardError; end + + POSITION_VALUE_CONTAINER_NAMES = %w[ChangeInPositionValues].freeze + POSITION_VALUE_ROW_NAMES = %w[ChangeInPositionValue].freeze + CASH_REPORT_CONTAINER_NAMES = %w[CashReport CashReports].freeze + CASH_REPORT_ROW_NAMES = %w[CashReport CashReportCurrency CashReportRow].freeze + EQUITY_SUMMARY_CONTAINER_NAMES = %w[EquitySummaryInBase].freeze + EQUITY_SUMMARY_ROW_NAMES = %w[EquitySummaryByReportDateInBase].freeze + OPEN_POSITION_CONTAINER_NAMES = %w[OpenPositions].freeze + OPEN_POSITION_ROW_NAMES = %w[OpenPosition].freeze + TRADES_CONTAINER_NAMES = %w[Trades].freeze + TRADE_ROW_NAMES = %w[Trade].freeze + CASH_TRANSACTION_CONTAINER_NAMES = %w[CashTransactions].freeze + CASH_TRANSACTION_ROW_NAMES = %w[CashTransaction].freeze + + def initialize(xml_body) + @document = Nokogiri::XML(xml_body.to_s) { |config| config.strict.noblanks } + rescue Nokogiri::XML::SyntaxError => e + raise ParseError, "Invalid IBKR Flex XML: #{e.message}" + end + + def parse + validate_document! + + { + metadata: root_metadata, + accounts: flex_statements.map { |statement| parse_statement(statement) } + } + end + + private + + def validate_document! + raise ParseError, "Invalid IBKR Flex XML: missing FlexQueryResponse root." unless @document.at_xpath("//FlexQueryResponse") + raise ParseError, "Invalid IBKR Flex XML: no FlexStatement nodes found." if flex_statements.empty? + end + + def flex_statements + @document.xpath("//FlexStatement") + end + + def root_metadata + node_attributes(@document.at_xpath("//FlexQueryResponse")) + end + + def parse_statement(statement) + statement_data = node_attributes(statement) + account_information = node_attributes(statement.at_xpath("./AccountInformation")) + position_values = section_rows(statement, POSITION_VALUE_CONTAINER_NAMES, POSITION_VALUE_ROW_NAMES) + cash_report = section_rows(statement, CASH_REPORT_CONTAINER_NAMES, CASH_REPORT_ROW_NAMES) + equity_summary_in_base = section_rows(statement, EQUITY_SUMMARY_CONTAINER_NAMES, EQUITY_SUMMARY_ROW_NAMES) + open_positions = section_rows(statement, OPEN_POSITION_CONTAINER_NAMES, OPEN_POSITION_ROW_NAMES) + trades = section_rows(statement, TRADES_CONTAINER_NAMES, TRADE_ROW_NAMES) + cash_transactions = section_rows(statement, CASH_TRANSACTION_CONTAINER_NAMES, CASH_TRANSACTION_ROW_NAMES) + account_id = account_information["account_id"].presence || statement_data["account_id"] + + raise ParseError, "Invalid IBKR Flex XML: missing account identifier in FlexStatement." if account_id.blank? + + currency = account_information["currency"].presence&.upcase || "USD" + report_date = open_positions.filter_map { |row| parse_date(row["report_date"]) }.max || + equity_summary_in_base.filter_map { |row| parse_date(row["report_date"]) }.max || + parse_date(statement_data["to_date"]) || + Date.current + + { + ibkr_account_id: account_id, + name: account_id, + currency: currency, + cash_balance: extract_cash_balance(cash_report, currency), + current_balance: extract_total_balance(position_values, cash_report, currency), + report_date: report_date, + statement: statement_data, + cash_report: cash_report, + equity_summary_in_base: equity_summary_in_base, + open_positions: open_positions, + trades: trades, + cash_transactions: cash_transactions, + raw_payload: { + statement: statement_data, + cash_report: cash_report, + equity_summary_in_base: equity_summary_in_base, + open_positions: open_positions, + trades: trades, + cash_transactions: cash_transactions + } + } + end + + def section_rows(statement, container_names, row_names) + rows = [] + + container_names.each do |container_name| + statement.xpath("./#{container_name}").each do |container| + children = container.element_children + + if children.any? + rows.concat(children.select { |child| row_names.include?(child.name) }) + elsif row_names.include?(container.name) + rows << container + end + end + end + + if rows.empty? + row_names.each do |row_name| + rows.concat(statement.xpath("./#{row_name}")) + end + end + + rows.map { |row| node_attributes(row) }.reject(&:blank?) + end + + def node_attributes(node) + return {} unless node + + node.attribute_nodes.each_with_object({}) do |attribute, result| + result[attribute.name.underscore] = attribute.value + end + end + + def extract_cash_balance(cash_rows, account_currency) + base_summary = cash_rows.find { |row| row["currency"] == "BASE_SUMMARY" } + account_row = cash_rows.find { |row| row["currency"] == account_currency } + row = base_summary || account_row + + parse_decimal(row&.fetch("ending_cash", nil)) || BigDecimal("0") + end + + def extract_current_balance(position_values, account_currency) + base_summary = position_values.find { |row| row["currency"] == "BASE_SUMMARY" } + account_row = position_values.find { |row| row["currency"] == account_currency } + row = base_summary || account_row + + parse_decimal(row&.fetch("end_of_period_value", nil)) || BigDecimal("0") + end + + def extract_total_balance(position_values, cash_rows, account_currency) + extract_current_balance(position_values, account_currency) + extract_cash_balance(cash_rows, account_currency) + end +end diff --git a/app/models/ibkr_item/sync_complete_event.rb b/app/models/ibkr_item/sync_complete_event.rb new file mode 100644 index 000000000..46ebe39ac --- /dev/null +++ b/app/models/ibkr_item/sync_complete_event.rb @@ -0,0 +1,22 @@ +class IbkrItem::SyncCompleteEvent + attr_reader :ibkr_item + + def initialize(ibkr_item) + @ibkr_item = ibkr_item + end + + def broadcast + ibkr_item.accounts.each do |account| + account.broadcast_sync_complete + end + + ibkr_item.broadcast_replace_to( + ibkr_item.family, + target: "ibkr_item_#{ibkr_item.id}", + partial: "ibkr_items/ibkr_item", + locals: { ibkr_item: ibkr_item } + ) + + ibkr_item.family.broadcast_sync_complete + end +end diff --git a/app/models/ibkr_item/syncer.rb b/app/models/ibkr_item/syncer.rb new file mode 100644 index 000000000..003c26855 --- /dev/null +++ b/app/models/ibkr_item/syncer.rb @@ -0,0 +1,68 @@ +class IbkrItem::Syncer + include SyncStats::Collector + + attr_reader :ibkr_item + + def initialize(ibkr_item) + @ibkr_item = ibkr_item + end + + def perform_sync(sync) + sync.update!(status_text: "Checking IBKR credentials...") if sync.respond_to?(:status_text) + unless ibkr_item.credentials_configured? + ibkr_item.update!(status: :requires_update) + raise Provider::IbkrFlex::ConfigurationError, "IBKR credentials are missing." + end + + sync.update!(status_text: "Importing IBKR accounts...") if sync.respond_to?(:status_text) + ibkr_item.import_latest_ibkr_data + + sync.update!(status_text: "Checking account configuration...") if sync.respond_to?(:status_text) + collect_setup_stats(sync, provider_accounts: ibkr_item.ibkr_accounts.to_a) + + unlinked_accounts = ibkr_item.ibkr_accounts.left_joins(:account_provider).where(account_providers: { id: nil }) + linked_accounts = ibkr_item.ibkr_accounts.joins(:account).merge(Account.visible) + + if unlinked_accounts.any? + ibkr_item.update!(pending_account_setup: true) + sync.update!(status_text: "#{unlinked_accounts.count} IBKR account(s) need setup...") if sync.respond_to?(:status_text) + else + ibkr_item.update!(pending_account_setup: false) + end + + if linked_accounts.any? + sync.update!(status_text: "Processing holdings and activity...") if sync.respond_to?(:status_text) + ibkr_item.process_accounts + + sync.update!(status_text: "Calculating balances...") if sync.respond_to?(:status_text) + ibkr_item.schedule_account_syncs( + parent_sync: sync, + window_start_date: sync.window_start_date, + window_end_date: sync.window_end_date + ) + + account_ids = linked_accounts.includes(:account).filter_map { |provider_account| provider_account.account&.id } + collect_transaction_stats(sync, account_ids: account_ids, source: "ibkr") if account_ids.any? + collect_trades_stats(sync, account_ids: account_ids, source: "ibkr") if account_ids.any? + collect_holdings_stats(sync, holdings_count: count_holdings, label: "processed") + end + + collect_health_stats(sync, errors: nil) + rescue Provider::IbkrFlex::AuthenticationError, Provider::IbkrFlex::ConfigurationError => e + ibkr_item.update!(status: :requires_update) + collect_health_stats(sync, errors: [ { message: e.message, category: "auth_error" } ]) + raise + rescue => e + collect_health_stats(sync, errors: [ { message: e.message, category: "sync_error" } ]) + raise + end + + def perform_post_sync + end + + private + + def count_holdings + ibkr_item.ibkr_accounts.sum { |account| Array(account.raw_holdings_payload).size } + end +end diff --git a/app/models/ibkr_item/unlinking.rb b/app/models/ibkr_item/unlinking.rb new file mode 100644 index 000000000..ddc827a56 --- /dev/null +++ b/app/models/ibkr_item/unlinking.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module IbkrItem::Unlinking + extend ActiveSupport::Concern + + def unlink_all!(dry_run: false) + results = [] + + ibkr_accounts.find_each do |provider_account| + links = AccountProvider.where(provider_type: "IbkrAccount", provider_id: provider_account.id).to_a + link_ids = links.map(&:id) + result = { + provider_account_id: provider_account.id, + name: provider_account.name, + provider_link_ids: link_ids + } + results << result + + next if dry_run + + begin + ActiveRecord::Base.transaction do + Holding.where(account_provider_id: link_ids).update_all(account_provider_id: nil) if link_ids.any? + links.each(&:destroy!) + end + rescue => e + Rails.logger.warn( + "IbkrItem Unlinker: failed to fully unlink provider account ##{provider_account.id} " \ + "(links=#{link_ids.inspect}): #{e.class} - #{e.message}" + ) + result[:error] = e.message + end + end + + results + end +end diff --git a/app/models/provider/ibkr_adapter.rb b/app/models/provider/ibkr_adapter.rb new file mode 100644 index 000000000..639831937 --- /dev/null +++ b/app/models/provider/ibkr_adapter.rb @@ -0,0 +1,59 @@ +class Provider::IbkrAdapter < Provider::Base + include Provider::Syncable + include Provider::InstitutionMetadata + + Provider::Factory.register("IbkrAccount", self) + + def self.supported_account_types + %w[Investment] + end + + def self.connection_configs(family:) + return [] unless family.can_connect_ibkr? + + [ { + key: "ibkr", + name: I18n.t("providers.ibkr.name"), + description: I18n.t("providers.ibkr.connection_description"), + can_connect: true, + new_account_path: ->(_accountable_type, _return_to) { + Rails.application.routes.url_helpers.select_accounts_ibkr_items_path + }, + existing_account_path: ->(account_id) { + Rails.application.routes.url_helpers.select_existing_account_ibkr_items_path(account_id: account_id) + } + } ] + end + + def provider_name + "ibkr" + end + + def sync_path + Rails.application.routes.url_helpers.sync_ibkr_item_path(item) + end + + def item + provider_account.ibkr_item + end + + def can_delete_holdings? + false + end + + def institution_domain + "interactivebrokers.com" + end + + def institution_name + I18n.t("providers.ibkr.institution_name") + end + + def institution_url + "https://www.interactivebrokers.com" + end + + def institution_color + "#D32F2F" + end +end diff --git a/app/models/provider/ibkr_flex.rb b/app/models/provider/ibkr_flex.rb new file mode 100644 index 000000000..16da5b803 --- /dev/null +++ b/app/models/provider/ibkr_flex.rb @@ -0,0 +1,144 @@ +class Provider::IbkrFlex + include HTTParty + extend SslConfigurable + + class Error < StandardError; end + class AuthenticationError < Error; end + class ConfigurationError < Error; end + class ApiError < Error + attr_reader :status_code, :response_body, :error_code + + def initialize(message, status_code: nil, response_body: nil, error_code: nil) + super(message) + @status_code = status_code + @response_body = response_body + @error_code = error_code + end + end + + base_uri "https://ndcdyn.interactivebrokers.com/AccountManagement/FlexWebService" + headers "User-Agent" => "Sure Finance IBKR Flex Client" + default_options.merge!({ timeout: 120 }.merge(httparty_ssl_options)) + + MAX_RETRIES = 3 + INITIAL_RETRY_DELAY = 2 + MAX_RETRY_DELAY = 30 + POLL_INTERVAL = 3 + MAX_POLL_ATTEMPTS = 20 + PENDING_ERROR_CODES = %w[1004 1019].freeze + + RETRYABLE_ERRORS = [ + SocketError, + Net::OpenTimeout, + Net::ReadTimeout, + Errno::ECONNRESET, + Errno::ECONNREFUSED, + Errno::ETIMEDOUT, + EOFError + ].freeze + + attr_reader :query_id, :token + + def initialize(query_id:, token:) + raise ConfigurationError, "query_id is required" if query_id.blank? + raise ConfigurationError, "token is required" if token.blank? + + @query_id = query_id.to_s.strip + @token = token.to_s.strip + end + + def download_statement + reference_code = request_reference_code + poll_statement(reference_code) + end + + private + + def request_reference_code + response = with_retries("SendRequest") do + self.class.get("/SendRequest", query: { t: token, q: query_id, v: 3 }) + end + + xml = parse_xml(response.body) + error = response_error(xml, response) + raise error if error + + reference_code = xml.at_xpath("//ReferenceCode")&.text.to_s.strip + raise ApiError.new("IBKR Flex did not return a reference code.", status_code: response.code, response_body: response.body) if reference_code.blank? + + reference_code + end + + def poll_statement(reference_code) + attempts = 0 + + loop do + attempts += 1 + response = with_retries("GetStatement") do + self.class.get("/GetStatement", query: { t: token, q: reference_code, v: 3 }) + end + + xml = parse_xml(response.body) + return response.body if xml.at_xpath("//FlexQueryResponse") + + error = response_error(xml, response) + if error.is_a?(ApiError) && PENDING_ERROR_CODES.include?(error.error_code.to_s) + raise ApiError.new("IBKR Flex statement is still being generated.", error_code: error.error_code) if attempts >= MAX_POLL_ATTEMPTS + + sleep(POLL_INTERVAL) + next + end + + raise(error || ApiError.new("IBKR Flex returned an unexpected response.", status_code: response.code, response_body: response.body)) + end + end + + def response_error(xml, response) + error_code = xml.at_xpath("//ErrorCode")&.text.to_s.strip.presence + error_message = xml.at_xpath("//ErrorMessage")&.text.to_s.strip.presence + + return nil if error_code.blank? && response.success? + + message = error_message.presence || "IBKR Flex request failed" + + case error_code + when "1012", "1015" + AuthenticationError.new(message) + when "1014" + ConfigurationError.new(message) + else + ApiError.new(message, status_code: response.code, response_body: response.body, error_code: error_code) + end + end + + def parse_xml(body) + Nokogiri::XML(body.to_s) + end + + def with_retries(operation_name, max_retries: MAX_RETRIES) + retries = 0 + + begin + yield + rescue *RETRYABLE_ERRORS => e + retries += 1 + + if retries <= max_retries + delay = calculate_retry_delay(retries) + Rails.logger.warn( + "IBKR Flex: #{operation_name} failed (attempt #{retries}/#{max_retries}): #{e.class}: #{e.message}. Retrying in #{delay}s..." + ) + sleep(delay) + retry + end + + raise ApiError.new("Network error after #{max_retries} retries: #{e.message}") + end + end + + def calculate_retry_delay(retry_count) + base_delay = INITIAL_RETRY_DELAY * (2**(retry_count - 1)) + jitter = base_delay * rand * 0.25 + [ base_delay + jitter, MAX_RETRY_DELAY ].min + end +end diff --git a/app/models/provider/metadata.rb b/app/models/provider/metadata.rb index 0850c524c..3d8472d70 100644 --- a/app/models/provider/metadata.rb +++ b/app/models/provider/metadata.rb @@ -10,6 +10,7 @@ class Provider binance: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "BI", logo_bg: "bg-yellow-600" }, kraken: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "KR", logo_bg: "bg-violet-600" }, snaptrade: { region: "US / CA", kind: "Investment", maturity: :beta, logo_text: "ST", logo_bg: "bg-green-600" }, + ibkr: { region: "Global", kind: "Investment", maturity: :beta, logo_text: "IB", logo_bg: "bg-red-600" }, indexa_capital: { region: "ES", kind: "Investment", maturity: :alpha, logo_text: "IC", logo_bg: "bg-red-600" }, sophtron: { region: "US", kind: "Bank", maturity: :alpha, logo_text: "SO", logo_bg: "bg-teal-600" }, plaid: { region: "US", kind: "Bank", tier: "Paid", maturity: :stable, logo_text: "PL", logo_bg: "bg-indigo-600" }, diff --git a/app/models/provider_connection_status.rb b/app/models/provider_connection_status.rb index 6ed6f4a34..47ce145ea 100644 --- a/app/models/provider_connection_status.rb +++ b/app/models/provider_connection_status.rb @@ -11,6 +11,7 @@ class ProviderConnectionStatus { key: "kraken", type: "KrakenItem", association: :kraken_items, accounts: :kraken_accounts }, { key: "coinstats", type: "CoinstatsItem", association: :coinstats_items, accounts: :coinstats_accounts }, { key: "snaptrade", type: "SnaptradeItem", association: :snaptrade_items, accounts: :snaptrade_accounts, linked_accounts: :linked_accounts }, + { key: "ibkr", type: "IbkrItem", association: :ibkr_items, accounts: :ibkr_accounts }, { key: "mercury", type: "MercuryItem", association: :mercury_items, accounts: :mercury_accounts }, { key: "sophtron", type: "SophtronItem", association: :sophtron_items, accounts: :sophtron_accounts }, { key: "indexa_capital", type: "IndexaCapitalItem", association: :indexa_capital_items, accounts: :indexa_capital_accounts } diff --git a/app/models/snaptrade_account/processor.rb b/app/models/snaptrade_account/processor.rb index b5edcb5e8..a20893e63 100644 --- a/app/models/snaptrade_account/processor.rb +++ b/app/models/snaptrade_account/processor.rb @@ -71,6 +71,11 @@ class SnaptradeAccount::Processor end def calculate_total_balance + if use_api_total_balance? + Rails.logger.debug "SnaptradeAccount::Processor - Using API total for multi-currency holdings for snaptrade_account=#{snaptrade_account.id}" + return snaptrade_account.current_balance || 0 + end + # Calculate total from holdings + cash for accuracy # SnapTrade's current_balance can sometimes be stale or just the cash value holdings_value = calculate_holdings_value @@ -109,4 +114,24 @@ class SnaptradeAccount::Processor units * price end end + + def use_api_total_balance? + return false unless snaptrade_account.current_balance.present? + + holdings_currencies.any? { |currency| currency.present? && currency != snaptrade_account.currency } + end + + def holdings_currencies + Array(snaptrade_account.raw_holdings_payload).filter_map do |holding| + data = holding.respond_to?(:with_indifferent_access) ? holding.with_indifferent_access : {} + extract_currency(data, extract_symbol_data(data), snaptrade_account.currency) + end.uniq + end + + def extract_symbol_data(data) + symbol_wrapper = data[:symbol].is_a?(Hash) ? data[:symbol].with_indifferent_access : {} + raw_symbol_data = symbol_wrapper[:symbol] + + raw_symbol_data.is_a?(Hash) ? raw_symbol_data.with_indifferent_access : {} + end end diff --git a/app/models/trade.rb b/app/models/trade.rb index dbdb90f31..a0c6df58a 100644 --- a/app/models/trade.rb +++ b/app/models/trade.rb @@ -14,6 +14,27 @@ class Trade < ApplicationRecord validates :price, :currency, presence: true validates :investment_activity_label, inclusion: { in: ACTIVITY_LABELS }, allow_nil: true + def exchange_rate + extra&.dig("exchange_rate") + end + + def exchange_rate=(value) + if value.blank? + self.extra = (extra || {}).merge("exchange_rate" => nil, "exchange_rate_invalid" => false) + else + begin + normalized_value = Float(value) + raise ArgumentError unless normalized_value.finite? + + self.extra = (extra || {}).merge("exchange_rate" => normalized_value, "exchange_rate_invalid" => false) + rescue ArgumentError, TypeError + self.extra = (extra || {}).merge("exchange_rate" => value, "exchange_rate_invalid" => true) + end + end + end + + validate :exchange_rate_must_be_valid + # Trade types for categorization def buy? qty.positive? @@ -57,6 +78,17 @@ class Trade < ApplicationRecord private + def exchange_rate_must_be_valid + if extra&.dig("exchange_rate_invalid") + errors.add(:exchange_rate, "must be a number") + elsif exchange_rate.present? + numeric_rate = Float(exchange_rate) rescue nil + if numeric_rate.nil? || !numeric_rate.finite? || numeric_rate <= 0 + errors.add(:exchange_rate, "must be greater than 0") + end + end + end + def calculate_realized_gain_loss return nil unless sell? diff --git a/app/models/transaction.rb b/app/models/transaction.rb index 6ebc807be..0334298d1 100644 --- a/app/models/transaction.rb +++ b/app/models/transaction.rb @@ -35,11 +35,13 @@ class Transaction < ApplicationRecord def exchange_rate=(value) if value.blank? - self.extra = (extra || {}).merge("exchange_rate" => nil) + self.extra = (extra || {}).merge("exchange_rate" => nil, "exchange_rate_invalid" => false) else begin normalized_value = Float(value) - self.extra = (extra || {}).merge("exchange_rate" => normalized_value) + raise ArgumentError unless normalized_value.finite? + + self.extra = (extra || {}).merge("exchange_rate" => normalized_value, "exchange_rate_invalid" => false) rescue ArgumentError, TypeError # Store the raw value for validation error reporting self.extra = (extra || {}).merge("exchange_rate" => value, "exchange_rate_invalid" => true) @@ -55,9 +57,8 @@ class Transaction < ApplicationRecord if extra&.dig("exchange_rate_invalid") errors.add(:exchange_rate, "must be a number") elsif exchange_rate.present? - # Convert to float for comparison - numeric_rate = exchange_rate.to_d rescue nil - if numeric_rate.nil? || numeric_rate <= 0 + numeric_rate = Float(exchange_rate) rescue nil + if numeric_rate.nil? || !numeric_rate.finite? || numeric_rate <= 0 errors.add(:exchange_rate, "must be greater than 0") end end @@ -151,6 +152,24 @@ class Transaction < ApplicationRecord false end + def activity_security_id + extra&.dig("security_id").presence || extra&.dig("security", "id").presence + end + + def activity_security + security_id = activity_security_id.to_s + return @activity_security = nil if security_id.blank? + return @activity_security if defined?(@activity_security_id) && @activity_security_id == security_id + + @activity_security_id = security_id + @activity_security = Security.find_by(id: security_id) + end + + def set_preloaded_activity_security(security) + @activity_security_id = security&.id&.to_s + @activity_security = security + end + # Potential duplicate matching methods # These help users review and resolve fuzzy-matched pending/posted pairs diff --git a/app/models/transaction/activity_security_preloader.rb b/app/models/transaction/activity_security_preloader.rb new file mode 100644 index 000000000..2939cf29d --- /dev/null +++ b/app/models/transaction/activity_security_preloader.rb @@ -0,0 +1,36 @@ +class Transaction::ActivitySecurityPreloader + def initialize(records) + @records = Array(records) + end + + def preload + transactions.each do |transaction| + transaction.set_preloaded_activity_security(securities_by_id[transaction.activity_security_id.to_s]) + end + + records + end + + private + attr_reader :records + + def transactions + @transactions ||= records.filter_map do |record| + case record + when Transaction + record + when Entry + record.transaction? ? record.entryable : nil + end + end + end + + def securities_by_id + @securities_by_id ||= begin + security_ids = transactions.filter_map(&:activity_security_id).uniq + return {} if security_ids.empty? + + Security.where(id: security_ids).index_by { |security| security.id.to_s } + end + end +end diff --git a/app/views/accounts/index.html.erb b/app/views/accounts/index.html.erb index 2dcd38118..bd9d4b3af 100644 --- a/app/views/accounts/index.html.erb +++ b/app/views/accounts/index.html.erb @@ -17,7 +17,7 @@ ) %> <% end %> -<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @snaptrade_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? %> +<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @snaptrade_items.empty? && @ibkr_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? %> <%= render "empty" %> <% else %>
@@ -57,6 +57,10 @@ <%= render @snaptrade_items.sort_by(&:created_at) %> <% end %> + <% if @ibkr_items.any? %> + <%= render @ibkr_items.sort_by(&:created_at) %> + <% end %> + <% if @indexa_capital_items.any? %> <%= render @indexa_capital_items.sort_by(&:created_at) %> <% end %> diff --git a/app/views/holdings/_cash.html.erb b/app/views/holdings/_cash.html.erb index 6fde392d7..72e8ceaf4 100644 --- a/app/views/holdings/_cash.html.erb +++ b/app/views/holdings/_cash.html.erb @@ -29,7 +29,7 @@
- <%= tag.p format_money account.cash_balance_money %> + <%= tag.p format_money(account.cash_balance_money), class: "privacy-sensitive" %>
diff --git a/app/views/holdings/_cost_basis_cell.html.erb b/app/views/holdings/_cost_basis_cell.html.erb index 7bc19c237..8d595baee 100644 --- a/app/views/holdings/_cost_basis_cell.html.erb +++ b/app/views/holdings/_cost_basis_cell.html.erb @@ -12,7 +12,7 @@ <% if holding.cost_basis_locked? && !editable %> <%# Locked and not editable (from holdings list) - just show value, right-aligned %>
- <%= tag.span format_money(holding.avg_cost) %> + <%= tag.span format_money(holding.avg_cost), class: "privacy-sensitive" %> <%= icon "lock", size: "xs", class: "text-secondary" %>
<% else %> @@ -21,7 +21,7 @@ <% menu.with_button(class: "hover:text-primary cursor-pointer group") do %> <% if holding.avg_cost %>
- <%= tag.span format_money(holding.avg_cost) %> + <%= tag.span format_money(holding.avg_cost), class: "privacy-sensitive" %> <% if holding.cost_basis_locked? %> <%= icon "lock", size: "xs", class: "text-secondary" %> <% end %> diff --git a/app/views/holdings/_holding.html.erb b/app/views/holdings/_holding.html.erb index 8d6adbfdb..533b2a7a6 100644 --- a/app/views/holdings/_holding.html.erb +++ b/app/views/holdings/_holding.html.erb @@ -39,7 +39,7 @@ <% else %> <%= tag.p "--", class: "text-secondary" %> <% end %> - <%= tag.p t(".shares", qty: format_quantity(holding.qty)), class: "font-normal text-secondary" %> + <%= tag.p t(".shares", qty: format_quantity(holding.qty)), class: "font-normal text-secondary privacy-sensitive" %>
diff --git a/app/views/ibkr_items/_ibkr_item.html.erb b/app/views/ibkr_items/_ibkr_item.html.erb new file mode 100644 index 000000000..511550b5e --- /dev/null +++ b/app/views/ibkr_items/_ibkr_item.html.erb @@ -0,0 +1,114 @@ +<%# locals: (ibkr_item:) %> + +<%= tag.div id: dom_id(ibkr_item) do %> + <% unlinked_count = ibkr_item.unlinked_accounts_count %> + +
+ +
+ <%= icon "chevron-right", class: "group-open:transform group-open:rotate-90" %> + +
+ IB +
+ +
+
+ <%= tag.p ibkr_item.institution_display_name, class: "font-medium text-primary" %> + <% if ibkr_item.scheduled_for_deletion? %> +

<%= t(".deletion_in_progress") %>

+ <% end %> +
+

<%= t(".flex_web_service") %>

+ <% if ibkr_item.syncing? %> +
+ <%= icon "loader", size: "sm", class: "animate-spin" %> + <%= tag.span t(".syncing") %> +
+ <% elsif ibkr_item.requires_update? %> +
+ <%= icon "alert-triangle", size: "sm", color: "warning" %> + <%= tag.span t(".requires_update") %> +
+ <% elsif ibkr_item.sync_error.present? %> +
+ <%= render DS::Tooltip.new(text: ibkr_item.sync_error, icon: "alert-circle", size: "sm", color: "destructive") %> + <%= tag.span t(".error"), class: "text-destructive" %> +
+ <% else %> +

+ <% if ibkr_item.last_synced_at %> + <%= t(".synced", time: time_ago_in_words(ibkr_item.last_synced_at), summary: ibkr_item.sync_status_summary) %> + <% else %> + <%= t(".never_synced") %> + <% end %> +

+ <% end %> +
+
+ + <% if Current.user&.admin? %> +
+ <%= icon( + "refresh-cw", + as_button: true, + href: sync_ibkr_item_path(ibkr_item), + disabled: ibkr_item.syncing? + ) %> + + <%= render DS::Menu.new do |menu| %> + <% if unlinked_count > 0 %> + <% menu.with_item( + variant: "link", + text: t(".setup_accounts"), + icon: "settings", + href: setup_accounts_ibkr_item_path(ibkr_item), + frame: :modal + ) %> + <% end %> + <% menu.with_item( + variant: "button", + text: t(".delete"), + icon: "trash-2", + href: ibkr_item_path(ibkr_item), + method: :delete, + confirm: CustomConfirm.for_resource_deletion(ibkr_item.institution_display_name, high_severity: true) + ) %> + <% end %> +
+ <% end %> + + + <% unless ibkr_item.scheduled_for_deletion? %> +
+ <% if ibkr_item.accounts.any? %> + <%= render "accounts/index/account_groups", accounts: ibkr_item.accounts %> + <% end %> + + <% stats = ibkr_item.syncs.ordered.first&.sync_stats || {} %> + <%= render ProviderSyncSummary.new(stats: stats, provider_item: ibkr_item) %> + + <% if Current.user&.admin? %> + <% if unlinked_count > 0 && ibkr_item.accounts.empty? %> +
+

<%= t(".accounts_need_setup") %>

+

<%= t(".accounts_need_setup_description") %>

+ <%= render DS::Link.new( + text: t(".setup_accounts"), + icon: "settings", + variant: "primary", + href: setup_accounts_ibkr_item_path(ibkr_item), + frame: :modal + ) %> +
+ <% elsif ibkr_item.ibkr_accounts.none? %> +
+

<%= t(".no_accounts_discovered") %>

+

<%= t(".no_accounts_discovered_description") %>

+
+ <% end %> + <% end %> +
+ <% end %> +
+<% end %> diff --git a/app/views/ibkr_items/select_existing_account.html.erb b/app/views/ibkr_items/select_existing_account.html.erb new file mode 100644 index 000000000..3598da843 --- /dev/null +++ b/app/views/ibkr_items/select_existing_account.html.erb @@ -0,0 +1,39 @@ +<%= turbo_frame_tag "modal" do %> + <%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: "Link Interactive Brokers account") %> + + <% dialog.with_body do %> + <% if @available_ibkr_accounts.blank? %> +
+

No unlinked Interactive Brokers accounts are available yet.

+
    +
  • Run a sync from Settings > Providers after updating your Flex query.
    • +
  • Wait for the account discovery sync to finish.
    • +
+
+ <% else %> + <%= form_with url: link_existing_account_ibkr_items_path, method: :post, data: { turbo_frame: "_top" }, class: "space-y-4" do %> + <%= hidden_field_tag :account_id, @account.id %> +
+ <% @available_ibkr_accounts.each do |ibkr_account| %> + + <% end %> +
+ +
+ <%= render DS::Button.new(text: "Link", variant: :primary, icon: "link-2", type: :submit) %> + <%= render DS::Link.new(text: "Cancel", variant: :secondary, href: accounts_path, data: { turbo_frame: "_top" }) %> +
+ <% end %> + <% end %> + <% end %> + <% end %> +<% end %> diff --git a/app/views/ibkr_items/setup_accounts.html.erb b/app/views/ibkr_items/setup_accounts.html.erb new file mode 100644 index 000000000..80fb4d1e3 --- /dev/null +++ b/app/views/ibkr_items/setup_accounts.html.erb @@ -0,0 +1,174 @@ +<% content_for :title, t(".page_title") %> + +<%= render DS::Dialog.new(disable_click_outside: true) do |dialog| %> + <% dialog.with_header(title: t(".dialog_title")) do %> +
+ <%= icon "chart-line", class: "text-primary" %> + <%= t(".subtitle") %> +
+ <% end %> + + <% dialog.with_body do %> +
+
+
+ <%= icon "info", size: "sm", class: "text-primary mt-0.5 flex-shrink-0" %> +
+

<%= t(".info_box.title") %>

+
    +
  • <%= t(".info_box.items.item_1") %>
    • +
  • <%= t(".info_box.items.item_2") %>
    • +
  • <%= t(".info_box.items.item_3") %>
    • +
+

+ <%= icon "alert-triangle", size: "xs", class: "inline-block mr-1" %> + <%= t(".info_box.warning") %> +

+
+
+
+ + <% if @waiting_for_sync %> +
+
+

<%= t(".status.fetching_accounts") %>

+
+
+ <%= render DS::Link.new( + text: t(".buttons.refresh"), + variant: "secondary", + icon: "refresh-cw", + href: setup_accounts_ibkr_item_path(@ibkr_item), + frame: "_top" + ) %> + <%= render DS::Link.new( + text: t(".buttons.cancel"), + variant: "ghost", + href: accounts_path, + frame: "_top" + ) %> +
+ <% elsif @no_accounts_found %> +
+ <%= icon "alert-circle", size: "lg", class: "text-warning" %> +

<%= t(".status.no_accounts_found_title") %>

+

<%= t(".status.no_accounts_found_description") %>

+
+
+ <%= render DS::Link.new( + text: t(".buttons.back_to_settings"), + variant: "secondary", + href: settings_providers_path, + frame: "_top" + ) %> +
+ <% else %> + <%= form_with url: complete_account_setup_ibkr_item_path(@ibkr_item), method: :post, data: { turbo_frame: "_top" } do %> + <% if @unlinked_accounts.any? %> +
+

<%= t(".available_accounts.title") %>

+ + <% @unlinked_accounts.each do |ibkr_account| %> +
+
+ + +
+
+ <% end %> +
+ +
+ <%= render DS::Button.new( + text: t(".buttons.create_selected_accounts"), + variant: "primary", + icon: "plus", + type: "submit", + class: "flex-1" + ) %> + <%= render DS::Link.new( + text: t(".buttons.cancel"), + variant: "secondary", + href: accounts_path, + frame: "_top" + ) %> +
+ <% end %> + <% end %> + + <% if @unlinked_accounts.any? && @linkable_accounts.any? %> +
+

<%= t(".link_existing.description") %>

+
+ <% @unlinked_accounts.each do |ibkr_account| %> + <%= form_with url: link_existing_account_ibkr_items_path, method: :post, data: { turbo_frame: "_top" } do |link_form| %> + <%= link_form.hidden_field :ibkr_account_id, value: ibkr_account.id %> +

<%= ibkr_account.name %>

+
+ <%= link_form.select :account_id, + options_for_select(@linkable_accounts.map { |account| [t(".link_existing.manual_account_option", name: account.name, balance: number_to_currency(account.balance, unit: Money::Currency.new(account.currency || 'USD').symbol)), account.id] }), + { prompt: t(".link_existing.select_prompt") }, + class: "bg-container border border-primary rounded px-2 py-1 text-sm text-primary flex-1 min-w-0" %> + <%= render DS::Button.new( + text: t(".buttons.link"), + variant: "secondary", + size: "sm", + type: "submit" + ) %> +
+ <% end %> + <% end %> +
+
+ <% end %> + + <% if @linked_accounts.any? %> +
+

<%= t(".linked_accounts.title") %>

+ <% @linked_accounts.each do |ibkr_account| %> +
+
+
+ <%= icon "check-circle", class: "text-success" %> +
+

<%= ibkr_account.name %>

+

<%= t(".linked_accounts.linked_to_html", account: link_to(ibkr_account.current_account.name, account_path(ibkr_account.current_account), class: "link")) %>

+
+
+
+
+ <% end %> +
+ + <% if @unlinked_accounts.blank? %> +
+ <%= render DS::Link.new( + text: t(".buttons.done"), + variant: "primary", + href: accounts_path, + frame: "_top" + ) %> +
+ <% end %> + <% end %> + <% end %> +
+ <% end %> +<% end %> diff --git a/app/views/settings/providers/_ibkr_panel.html.erb b/app/views/settings/providers/_ibkr_panel.html.erb new file mode 100644 index 000000000..d20d03f31 --- /dev/null +++ b/app/views/settings/providers/_ibkr_panel.html.erb @@ -0,0 +1,152 @@ +
+ <% error_msg = local_assigns[:error_message] || @error_message %> + <% if error_msg.present? %> + <%= render DS::Alert.new(message: error_msg, variant: :error) %> + <% end %> + + <%= render "settings/providers/setup_steps", + steps: [ + t(".steps.step_1"), + t(".steps.step_2"), + t(".steps.step_3"), + t(".steps.step_4"), + t(".steps.step_5") + ] %> + +
+ +
+
+

<%= t(".flex_query_details.eyebrow") %>

+

<%= t(".flex_query_details.title") %>

+

<%= t(".flex_query_details.summary") %>

+
+ <%= icon "chevron-down", class: "mt-0.5 text-secondary transition-transform group-open:rotate-180" %> +
+ + +
+
+

<%= t(".flex_query_details.sections_heading") %>

+
    +
  • + <%= t(".sections.account_information") %> +
    • +
  • + <%= t(".sections.cash_report") %> +
      +
    • <%= t(".sections.cash_report_options") %>
      • +
    • <%= t(".sections.cash_report_fields") %>
      • +
    +
    • +
  • + <%= t(".sections.cash_transactions") %> +
      +
    • <%= t(".sections.cash_transactions_options") %>
      • +
    • <%= t(".sections.cash_transactions_fields") %>
      • +
    +
    • +
  • + <%= t(".sections.change_in_position_value_summary") %> +
    • +
  • + <%= t(".sections.net_asset_value") %> +
      +
    • <%= t(".sections.net_asset_value_options") %>
      • +
    • <%= t(".sections.net_asset_value_fields") %>
      • +
    +
    • +
  • + <%= t(".sections.open_positions") %> +
      +
    • <%= t(".sections.open_positions_options") %>
      • +
    • <%= t(".sections.open_positions_fields") %>
      • +
    +
    • +
  • + <%= t(".sections.trades") %> +
      +
    • <%= t(".sections.trades_options") %>
      • +
    • <%= t(".sections.trades_fields") %>
      • +
    +
    • +
+
+ +
+

<%= t(".flex_query_details.configuration_heading") %>

+
    +
  • <%= t(".configuration.models") %>
    • +
  • <%= t(".configuration.format") %>
    • +
  • <%= t(".configuration.period") %>
    • +
  • <%= t(".configuration.date_format") %>
    • +
  • <%= t(".configuration.time_format") %>
    • +
  • <%= t(".configuration.date_time_separator") %>
    • +
  • <%= t(".configuration.profit_and_loss") %>
    • +
  • <%= t(".configuration.all_other_options") %>
    • +
+
+ +

<%= t(".report_window_note") %>

+
+
+ + <% + ibkr_item = Current.family.ibkr_items.first_or_initialize(name: "Interactive Brokers") + is_new_record = ibkr_item.new_record? + %> + + <% if ibkr_item.persisted? %> +
+ <%= button_to sync_ibkr_item_path(ibkr_item), + method: :post, + class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-secondary hover:text-primary border border-secondary rounded-lg hover:border-primary", + disabled: ibkr_item.syncing? do %> + <%= icon "refresh-cw", size: "sm" %> + <%= t(".sync") %> + <% end %> + + <%= button_to ibkr_item_path(ibkr_item), + method: :delete, + class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-destructive hover:bg-destructive/10 rounded-lg", + data: { turbo_confirm: t(".disconnect_confirm") } do %> + <%= icon "trash-2", size: "sm" %> + <% end %> +
+ <% end %> + + <%= styled_form_with model: ibkr_item, + url: is_new_record ? ibkr_items_path : ibkr_item_path(ibkr_item), + scope: :ibkr_item, + method: is_new_record ? :post : :patch, + data: { turbo: true }, + class: "space-y-3" do |form| %> + <%= form.text_field :query_id, + label: t(".query_id_label"), + placeholder: is_new_record ? t(".query_id_placeholder_new") : t(".query_id_placeholder_existing"), + type: :password %> + + <%= form.text_field :token, + label: t(".token_label"), + placeholder: is_new_record ? t(".token_placeholder_new") : t(".token_placeholder_existing"), + type: :password %> + +
+ <%= form.submit(is_new_record ? t(".save_configuration") : t(".update_configuration"), class: "btn btn--primary") %> +
+ <% end %> + +
+ <% if ibkr_item.persisted? && ibkr_item.credentials_configured? %> +
+

+ <%= t(".status_configured_prefix", summary: ibkr_item.sync_status_summary) %> + <%= link_to t(".accounts_tab"), accounts_path, class: "link" %> + <%= t(".status_configured_suffix") %> +

+ <% else %> +
+

<%= t(".not_configured") %>

+ <% end %> +
+
diff --git a/app/views/trades/_trade.html.erb b/app/views/trades/_trade.html.erb index 04b67e645..162a7da09 100644 --- a/app/views/trades/_trade.html.erb +++ b/app/views/trades/_trade.html.erb @@ -1,6 +1,7 @@ <%# locals: (entry:, balance_trend: nil, **) %> <% trade = entry.entryable %> +<% trade_logo_url = trade.security&.display_logo_url %> <%= turbo_frame_tag dom_id(entry) do %> <%= turbo_frame_tag dom_id(trade) do %> @@ -17,14 +18,20 @@ } %>
- <%= tag.div class: ["flex items-center gap-2 min-w-0"] do %> + <%= tag.div class: ["flex items-center gap-3 lg:gap-4 min-w-0"] do %>
diff --git a/app/views/transactions/_transaction.html.erb b/app/views/transactions/_transaction.html.erb index 8ee5761c1..659591bff 100644 --- a/app/views/transactions/_transaction.html.erb +++ b/app/views/transactions/_transaction.html.erb @@ -1,6 +1,7 @@ <%# locals: (entry:, balance_trend: nil, view_ctx: "global", in_split_group: false) %> <% transaction = entry.entryable %> +<% transaction_security_logo_url = transaction.activity_security&.display_logo_url %> <%= turbo_frame_tag dom_id(entry) do %> <%= turbo_frame_tag dom_id(transaction) do %> @@ -20,7 +21,11 @@
<%= render "transactions/transaction_category", transaction: transaction, variant: "mobile", in_split_group: in_split_group %> - <% if transaction.merchant&.logo_url.present? %> + <% if transaction_security_logo_url.present? %> + <%= image_tag Setting.transform_brand_fetch_url(transaction_security_logo_url), + class: "w-5 h-5 rounded-full absolute -bottom-1 -right-1 border border-secondary pointer-events-none", + loading: "lazy" %> + <% elsif transaction.merchant&.logo_url.present? %> <%= image_tag Setting.transform_brand_fetch_url(transaction.merchant.logo_url), class: "w-5 h-5 rounded-full absolute -bottom-1 -right-1 border border-secondary pointer-events-none", loading: "lazy" %> @@ -37,6 +42,10 @@ size: "lg", rounded: true ) %> + <% elsif transaction_security_logo_url.present? %> + <%= image_tag Setting.transform_brand_fetch_url(transaction_security_logo_url), + class: "w-9 h-9 rounded-full border border-secondary", + loading: "lazy" %> <% elsif transaction.merchant&.logo_url.present? %> <%= image_tag Setting.transform_brand_fetch_url(transaction.merchant.logo_url), class: "w-9 h-9 rounded-full border border-secondary", diff --git a/config/locales/views/ibkr_items/en.yml b/config/locales/views/ibkr_items/en.yml new file mode 100644 index 000000000..a5d3bae44 --- /dev/null +++ b/config/locales/views/ibkr_items/en.yml @@ -0,0 +1,84 @@ +--- +en: + providers: + ibkr: + name: Interactive Brokers + connection_description: Connect an Interactive Brokers Flex Web Service report + institution_name: Interactive Brokers + ibkr_items: + defaults: + name: Interactive Brokers + ibkr_item: + deletion_in_progress: Deletion in progress + flex_web_service: Flex Web Service + syncing: Syncing + requires_update: Credentials need attention + error: Error + synced: Synced %{time} ago. %{summary}. + never_synced: Never synced. + setup_accounts: Set up accounts + delete: Delete + accounts_need_setup: Accounts need setup + accounts_need_setup_description: Some accounts from IBKR need to be linked to Sure accounts. + no_accounts_discovered: No IBKR accounts discovered yet. + no_accounts_discovered_description: Run a sync after configuring your Flex query to discover accounts. + setup_accounts: + page_title: Set Up Interactive Brokers Accounts + dialog_title: Set Up Your Interactive Brokers Accounts + subtitle: Select which IBKR brokerage accounts to link. + info_box: + title: IBKR Flex Query Import + items: + item_1: Holdings with current prices and quantities + item_2: Cost basis per position + item_3: Trades, dividends, commissions, and cash deposits or withdrawals + warning: Historical activity is limited to the report window of the Flex Query + status: + fetching_accounts: Fetching accounts from Interactive Brokers... + no_accounts_found_title: No accounts found. + no_accounts_found_description: Sure could not find any IBKR accounts in the latest Flex report. + available_accounts: + title: Available accounts + account_type_investment: Investment + account_summary: "%{account_type} • Balance: %{balance}" + account_id: "Account ID: %{account_id}" + link_existing: + description: Or link a discovered IBKR account to an existing manual investment account. + manual_account_option: "%{name} (%{balance})" + select_prompt: Select an account... + linked_accounts: + title: Already linked + linked_to_html: "Linked to: %{account}" + buttons: + refresh: Refresh + cancel: Cancel + back_to_settings: Back to Settings + create_selected_accounts: Create selected accounts + link: Link + done: Done + sync_status: + no_accounts: No IBKR accounts discovered yet + all_linked: + one: 1 account linked + other: "%{count} accounts linked" + partial: "%{linked} linked, %{unlinked} need setup" + create: + success: Successfully configured Interactive Brokers. + update: + success: Successfully updated Interactive Brokers configuration. + destroy: + success: Scheduled Interactive Brokers connection for deletion. + select_accounts: + not_configured: Interactive Brokers is not configured. + link_existing_account: + not_found: Account or Interactive Brokers configuration not found. + only_manual_investment: Only manual investment accounts can be linked to Interactive Brokers. + already_linked: This Interactive Brokers account is already linked. + success: Successfully linked to Interactive Brokers account. + failed: Failed to link Interactive Brokers account. + complete_account_setup: + success: + one: Successfully created %{count} Interactive Brokers account. + other: Successfully created %{count} Interactive Brokers accounts. + none_selected: No accounts were selected. + none_created: No accounts were created. diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index 87114313b..13887befc 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -237,6 +237,7 @@ en: binance: Sync your Binance spot balances using a read-only API key. kraken: Sync Kraken balances and spot trade fills using a read-only API key. snaptrade: Connect brokerage accounts via the SnapTrade aggregation network. + ibkr: Sync Interactive Brokers investment accounts via Flex Query imports. indexa_capital: Track your Indexa Capital automated investment portfolio. sophtron: Connect US & Canadian banks and utilities. plaid: Connect thousands of US financial institutions via Plaid. @@ -329,3 +330,58 @@ en: step_1_html: "Open the %{link} and copy your EU Client ID and Secret Key." not_found: Provider not found. sync_provider_no_items: No connections available to sync. + ibkr_panel: + steps: + step_1: 'In your IBKR Client Portal, navigate to "Performance & Reports" > "Flex Queries".' + step_2: 'Click the "+" icon in the "Activity Flex Query" section to create a new query.' + step_3: 'Name your query (e.g., "Sure Sync"), then review the Flex Query details below and enable the listed sections, fields, and configuration options.' + step_4: 'Save the query, note your "Query ID", then use the gear icon in the "Flex Web Service Configuration" section to generate an access Token.' + step_5: "Paste your Query ID and Token below, save the configuration, then head to Accounts to link discovered IBKR accounts." + flex_query_details: + eyebrow: Flex Query + title: Sections, fields, and configuration + summary: Expand to see the exact sections, fields, and settings your IBKR Activity Flex Query must include. + sections_heading: Enable these sections and fields + configuration_heading: Set these query options + sections: + account_information: "Account Information: Account ID, Currency" + cash_report: "Cash Report:" + cash_report_options: "Options: None" + cash_report_fields: "Fields: Currency, Ending Cash" + cash_transactions: "Cash Transactions:" + cash_transactions_options: "Options: Dividends, Deposits & Withdrawals, Detail" + cash_transactions_fields: "Fields: Amount, Conid, Currency, FX Rate To Base, Report Date, Transaction ID, Type" + change_in_position_value_summary: "Change In Position Value Summary: Currency, End Of Period Value" + net_asset_value: "Net Asset Value (NAV) in Base:" + net_asset_value_options: "Options: None" + net_asset_value_fields: "Fields: Currency, Report Date, Total" + open_positions: "Open Positions:" + open_positions_options: "Options: Summary" + open_positions_fields: "Fields: Asset Class, Conid, Cost Basis Price, Currency, FX Rate To Base, Mark Price, Quantity, Report Date, Security ID, Security ID Type, Side, Symbol" + trades: "Trades:" + trades_options: "Options: Execution" + trades_fields: "Fields: Asset Class, Buy/Sell, Conid, Currency, FX Rate To Base, IB Commission, IB Commission Currency, Quantity, Symbol, Trade Date, Trade ID, TradePrice, Transaction ID" + configuration: + models: "Models: Optional" + format: "Format: XML" + period: "Period: Last 365 Calendar Days" + date_format: "Date Format: yyyy-MM-dd" + time_format: "Time Format: HH:mm:ss" + date_time_separator: "Date/Time Separator: ; (semi-colon)" + profit_and_loss: "Profit and Loss: Default" + all_other_options: 'All other configuration options: "No"' + report_window_note: "IBKR Flex reports are limited to the query window you configured in IBKR. Sure will import full current holdings plus up to the last 365 days of activity from this report." + sync: Sync + disconnect_confirm: Disconnect Interactive Brokers? + query_id_label: Query ID + query_id_placeholder_new: Enter your IBKR Flex Query ID + query_id_placeholder_existing: Leave blank to keep the current Query ID + token_label: Token + token_placeholder_new: Enter your IBKR Flex Web Service Token + token_placeholder_existing: Leave blank to keep the current Token + save_configuration: Save Configuration + update_configuration: Update Configuration + status_configured_prefix: "%{summary}. Visit the" + accounts_tab: Accounts + status_configured_suffix: tab to manage discovered accounts. + not_configured: Not configured. diff --git a/config/routes.rb b/config/routes.rb index a9fee8dd1..ba00f86f4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -100,6 +100,20 @@ Rails.application.routes.draw do end end + resources :ibkr_items, only: [ :create, :update, :destroy ] do + collection do + get :select_accounts + get :select_existing_account + post :link_existing_account + end + + member do + post :sync + get :setup_accounts + post :complete_account_setup + end + end + # CoinStats routes resources :coinstats_items, only: [ :index, :new, :create, :update, :destroy ] do collection do diff --git a/db/migrate/20260512210600_create_ibkr_items_and_accounts.rb b/db/migrate/20260512210600_create_ibkr_items_and_accounts.rb new file mode 100644 index 000000000..b1daaeb2b --- /dev/null +++ b/db/migrate/20260512210600_create_ibkr_items_and_accounts.rb @@ -0,0 +1,41 @@ +class CreateIbkrItemsAndAccounts < ActiveRecord::Migration[7.2] + def change + create_table :ibkr_items, id: :uuid do |t| + t.references :family, null: false, foreign_key: true, type: :uuid + t.string :name + t.string :status, default: "good", null: false + t.boolean :scheduled_for_deletion, default: false, null: false + t.boolean :pending_account_setup, default: false, null: false + t.jsonb :raw_payload + t.string :query_id + t.string :token + + t.timestamps + end + + add_index :ibkr_items, :status + + create_table :ibkr_accounts, id: :uuid do |t| + t.references :ibkr_item, null: false, foreign_key: true, type: :uuid + t.string :name + t.string :ibkr_account_id + t.string :currency + t.decimal :current_balance, precision: 19, scale: 4 + t.decimal :cash_balance, precision: 19, scale: 4 + t.jsonb :institution_metadata + t.jsonb :raw_holdings_payload, default: [], null: false + t.jsonb :raw_activities_payload, default: {}, null: false + t.jsonb :raw_cash_report_payload, default: [], null: false + t.date :report_date + t.datetime :last_holdings_sync + t.datetime :last_activities_sync + + t.timestamps + end + + add_index :ibkr_accounts, [ :ibkr_item_id, :ibkr_account_id ], + unique: true, + where: "(ibkr_account_id IS NOT NULL)", + name: "index_ibkr_accounts_on_item_and_ibkr_account_id" + end +end diff --git a/db/migrate/20260512211000_add_extra_to_trades.rb b/db/migrate/20260512211000_add_extra_to_trades.rb new file mode 100644 index 000000000..2e27be775 --- /dev/null +++ b/db/migrate/20260512211000_add_extra_to_trades.rb @@ -0,0 +1,6 @@ +class AddExtraToTrades < ActiveRecord::Migration[7.2] + def change + add_column :trades, :extra, :jsonb, default: {}, null: false + add_index :trades, :extra, using: :gin + end +end diff --git a/db/migrate/20260512211200_add_raw_equity_summary_payload_to_ibkr_accounts.rb b/db/migrate/20260512211200_add_raw_equity_summary_payload_to_ibkr_accounts.rb new file mode 100644 index 000000000..d3f9e7969 --- /dev/null +++ b/db/migrate/20260512211200_add_raw_equity_summary_payload_to_ibkr_accounts.rb @@ -0,0 +1,5 @@ +class AddRawEquitySummaryPayloadToIbkrAccounts < ActiveRecord::Migration[7.2] + def change + add_column :ibkr_accounts, :raw_equity_summary_payload, :jsonb, default: [], null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index ecd756f91..f013a0ed2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do +ActiveRecord::Schema[7.2].define(version: 2026_05_12_211200) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -659,6 +659,42 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do t.index ["security_id"], name: "index_holdings_on_security_id" end + create_table "ibkr_accounts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "ibkr_item_id", null: false + t.string "name" + t.string "ibkr_account_id" + t.string "currency" + t.decimal "current_balance", precision: 19, scale: 4 + t.decimal "cash_balance", precision: 19, scale: 4 + t.jsonb "institution_metadata" + t.jsonb "raw_holdings_payload", default: [] + t.jsonb "raw_activities_payload", default: {} + t.jsonb "raw_cash_report_payload", default: [] + t.date "report_date" + t.datetime "last_holdings_sync" + t.datetime "last_activities_sync" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.jsonb "raw_equity_summary_payload", default: [], null: false + t.index ["ibkr_item_id", "ibkr_account_id"], name: "index_ibkr_accounts_on_item_and_ibkr_account_id", unique: true, where: "(ibkr_account_id IS NOT NULL)" + t.index ["ibkr_item_id"], name: "index_ibkr_accounts_on_ibkr_item_id" + end + + create_table "ibkr_items", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "family_id", null: false + t.string "name" + t.string "status", default: "good" + t.boolean "scheduled_for_deletion", default: false + t.boolean "pending_account_setup", default: false, null: false + t.jsonb "raw_payload" + t.string "query_id" + t.string "token" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["family_id"], name: "index_ibkr_items_on_family_id" + t.index ["status"], name: "index_ibkr_items_on_status" + end + create_table "impersonation_session_logs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "impersonation_session_id", null: false t.string "controller" @@ -1603,6 +1639,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do t.jsonb "locked_attributes", default: {} t.string "investment_activity_label" t.decimal "fee", precision: 19, scale: 4, default: "0.0", null: false + t.jsonb "extra", default: {}, null: false + t.index ["extra"], name: "index_trades_on_extra", using: :gin t.index ["investment_activity_label"], name: "index_trades_on_investment_activity_label" t.index ["security_id"], name: "index_trades_on_security_id" end @@ -1709,9 +1747,9 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do t.datetime "last_used_at" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.check_constraint "sign_count >= 0", name: "chk_webauthn_credentials_sign_count_non_negative" t.index ["credential_id"], name: "index_webauthn_credentials_on_credential_id", unique: true t.index ["user_id"], name: "index_webauthn_credentials_on_user_id" + t.check_constraint "sign_count >= 0", name: "chk_webauthn_credentials_sign_count_non_negative" end add_foreign_key "account_providers", "accounts", on_delete: :cascade @@ -1754,6 +1792,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_11_090000) do add_foreign_key "holdings", "accounts", on_delete: :cascade add_foreign_key "holdings", "securities" add_foreign_key "holdings", "securities", column: "provider_security_id" + add_foreign_key "ibkr_accounts", "ibkr_items" + add_foreign_key "ibkr_items", "families" add_foreign_key "impersonation_session_logs", "impersonation_sessions" add_foreign_key "impersonation_sessions", "users", column: "impersonated_id" add_foreign_key "impersonation_sessions", "users", column: "impersonator_id" diff --git a/test/controllers/ibkr_items_controller_test.rb b/test/controllers/ibkr_items_controller_test.rb new file mode 100644 index 000000000..fbb0bccb0 --- /dev/null +++ b/test/controllers/ibkr_items_controller_test.rb @@ -0,0 +1,100 @@ +require "test_helper" + +class IbkrItemsControllerTest < ActionDispatch::IntegrationTest + setup do + sign_in @user = users(:family_admin) + @ibkr_item = ibkr_items(:configured_item) + end + + test "select_existing_account renders available ibkr accounts" do + get select_existing_account_ibkr_items_url, params: { account_id: accounts(:investment).id } + + assert_response :success + assert_includes response.body, ibkr_accounts(:main_account).name + end + + test "create redirects to accounts on success" do + assert_difference "IbkrItem.count", 1 do + post ibkr_items_url, params: { + ibkr_item: { + query_id: "QUERYNEW", + token: "TOKENNEW" + } + } + end + + assert_redirected_to accounts_path + end + + test "update redirects to accounts on success" do + patch ibkr_item_url(@ibkr_item), params: { + ibkr_item: { + query_id: "", + token: "" + } + } + + assert_redirected_to accounts_path + end + + test "complete_account_setup creates investment account and provider link" do + assert_difference "Account.count", 1 do + assert_difference "AccountProvider.count", 1 do + post complete_account_setup_ibkr_item_url(@ibkr_item), params: { + account_ids: [ ibkr_accounts(:main_account).id ] + } + end + end + + created_account = Account.order(created_at: :desc).first + assert_equal "Investment", created_account.accountable_type + assert_equal "brokerage", created_account.accountable.subtype + assert_redirected_to accounts_path + + ibkr_accounts(:main_account).reload + assert_equal created_account, ibkr_accounts(:main_account).current_account + end + + test "link_existing_account links manual investment account" do + account = accounts(:investment) + + assert_difference "AccountProvider.count", 1 do + post link_existing_account_ibkr_items_url, params: { + account_id: account.id, + ibkr_account_id: ibkr_accounts(:main_account).id + } + end + + assert_redirected_to account_path(account) + ibkr_accounts(:main_account).reload + assert_equal account, ibkr_accounts(:main_account).current_account + end + + test "link_existing_account rejects already linked ibkr account" do + original_account = accounts(:investment) + ibkr_account = ibkr_accounts(:main_account) + AccountProvider.create!(account: original_account, provider: ibkr_account) + + replacement_account = Account.create!( + family: @ibkr_item.family, + owner: @user, + name: "Replacement Brokerage Account", + balance: 2500, + cash_balance: 2500, + currency: "USD", + accountable: Investment.create!(subtype: "brokerage") + ) + + assert_no_difference "AccountProvider.count" do + post link_existing_account_ibkr_items_url, params: { + account_id: replacement_account.id, + ibkr_account_id: ibkr_account.id + } + end + + assert_redirected_to account_path(replacement_account) + assert_equal "This Interactive Brokers account is already linked.", flash[:alert] + ibkr_account.reload + assert_equal original_account, ibkr_account.current_account + end +end diff --git a/test/controllers/settings/providers_controller_test.rb b/test/controllers/settings/providers_controller_test.rb index 00b98e893..f000bed7b 100644 --- a/test/controllers/settings/providers_controller_test.rb +++ b/test/controllers/settings/providers_controller_test.rb @@ -355,6 +355,37 @@ class Settings::ProvidersControllerTest < ActionDispatch::IntegrationTest assert_match(/Sync started/i, response.body) end + test "GET show includes Interactive Brokers in bank sync providers" do + get settings_providers_url + + assert_response :success + assert_match(/Interactive Brokers/i, response.body) + assert_match(/Flex Query/i, response.body) + end + + test "GET connect_form renders Interactive Brokers panel" do + get connect_form_settings_providers_path(provider_key: "ibkr") + + assert_response :success + assert_match(/Interactive Brokers/i, response.body) + assert_match(/Query ID/i, response.body) + end + + test "POST sync for ibkr without an active Ibkr sync enqueues SyncJob" do + item = ibkr_items(:configured_item) + Sync.where(syncable_type: "IbkrItem", syncable_id: item.id).delete_all + + assert_enqueued_jobs 1, only: SyncJob do + post sync_provider_settings_providers_path(provider_key: "ibkr") + end + + assert_redirected_to settings_providers_path + + follow_redirect! + assert_response :success + assert_match(/Sync started/i, response.body) + end + test "non-admin users cannot update providers" do with_self_hosting do sign_in users(:family_member) diff --git a/test/fixtures/files/ibkr/flex_statement.xml b/test/fixtures/files/ibkr/flex_statement.xml new file mode 100644 index 000000000..ffc2c5f7f --- /dev/null +++ b/test/fixtures/files/ibkr/flex_statement.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/fixtures/ibkr_accounts.yml b/test/fixtures/ibkr_accounts.yml new file mode 100644 index 000000000..8ef3649d3 --- /dev/null +++ b/test/fixtures/ibkr_accounts.yml @@ -0,0 +1,27 @@ +main_account: + ibkr_item: configured_item + name: Main IBKR + ibkr_account_id: U1234567 + currency: CHF + current_balance: 3351.0 + cash_balance: 1000.5 + institution_metadata: + provider_name: Interactive Brokers + raw_holdings_payload: [] + raw_activities_payload: {} + raw_cash_report_payload: [] + report_date: 2026-05-08 + +secondary_account: + ibkr_item: configured_item + name: Retirement IBKR + ibkr_account_id: U7654321 + currency: USD + current_balance: 250 + cash_balance: 250 + institution_metadata: + provider_name: Interactive Brokers + raw_holdings_payload: [] + raw_activities_payload: {} + raw_cash_report_payload: [] + report_date: 2026-05-08 diff --git a/test/fixtures/ibkr_items.yml b/test/fixtures/ibkr_items.yml new file mode 100644 index 000000000..f03973935 --- /dev/null +++ b/test/fixtures/ibkr_items.yml @@ -0,0 +1,15 @@ +configured_item: + family: dylan_family + name: Interactive Brokers + status: good + query_id: QUERY123 + token: TOKEN123 + pending_account_setup: true + +empty_item: + family: empty + name: Interactive Brokers + status: good + query_id: QUERYEMPTY + token: TOKENEMPTY + pending_account_setup: false diff --git a/test/models/account/opening_balance_manager_test.rb b/test/models/account/opening_balance_manager_test.rb index 67becb60a..78411bd00 100644 --- a/test/models/account/opening_balance_manager_test.rb +++ b/test/models/account/opening_balance_manager_test.rb @@ -192,6 +192,75 @@ class Account::OpeningBalanceManagerTest < ActiveSupport::TestCase assert_equal original_date, opening_anchor.entry.date # Should remain unchanged end + test "when existing anchor date is before later activity, update can preserve anchor date" do + manager = Account::OpeningBalanceManager.new(@depository_account) + original_date = 4.months.ago.to_date + + result = manager.set_opening_balance( + balance: 1000, + date: original_date + ) + assert result.success? + + @depository_account.entries.create!( + date: 2.months.ago.to_date, + name: "Later transaction", + amount: 100, + currency: "USD", + entryable: Transaction.new + ) + + result = manager.set_opening_balance( + balance: 0, + date: original_date + ) + + assert result.success? + assert result.changes_made? + + opening_anchor = @depository_account.valuations.opening_anchor.first + opening_anchor.reload + assert_equal 0, opening_anchor.entry.amount + assert_equal original_date, opening_anchor.entry.date + end + + test "recomputes oldest entry date when older activity is added after manager initialization" do + oldest_date = 60.days.ago.to_date + @depository_account.entries.create!( + date: oldest_date, + name: "Existing transaction", + amount: 100, + currency: "USD", + entryable: Transaction.new + ) + + manager = Account::OpeningBalanceManager.new(@depository_account) + + result = manager.set_opening_balance( + balance: 1000, + date: oldest_date - 1.day + ) + assert result.success? + + newly_oldest_date = oldest_date - 2.days + @depository_account.entries.create!( + date: newly_oldest_date, + name: "New older transaction", + amount: 50, + currency: "USD", + entryable: Transaction.new + ) + + result = manager.set_opening_balance( + balance: 1200, + date: newly_oldest_date + ) + + assert_not result.success? + assert_not result.changes_made? + assert_equal "Opening balance date must be before the oldest entry date", result.error + end + test "when date is equal to or greater than account's oldest entry, returns error result" do # Create an entry with a specific date oldest_date = 60.days.ago.to_date diff --git a/test/models/account/provider_import_adapter_test.rb b/test/models/account/provider_import_adapter_test.rb index 2f004632f..bbfa31b08 100644 --- a/test/models/account/provider_import_adapter_test.rb +++ b/test/models/account/provider_import_adapter_test.rb @@ -280,6 +280,25 @@ class Account::ProviderImportAdapterTest < ActiveSupport::TestCase end end + test "imports trade with provider exchange rate" do + investment_account = accounts(:investment) + adapter = Account::ProviderImportAdapter.new(investment_account) + security = securities(:aapl) + + entry = adapter.import_trade( + security: security, + quantity: 5, + price: 150.00, + amount: 750.00, + currency: "USD", + date: Date.today, + source: "plaid", + exchange_rate: 0.91 + ) + + assert_equal 0.91, entry.entryable.exchange_rate + end + test "raises error when security is missing for trade import" do exception = assert_raises(ArgumentError) do @adapter.import_trade( @@ -489,7 +508,8 @@ class Account::ProviderImportAdapterTest < ActiveSupport::TestCase amount: 2000.00, currency: "USD", date: Date.today, - source: "plaid" + source: "plaid", + exchange_rate: 0.95 ) assert_equal entry.id, updated_entry.id @@ -498,11 +518,46 @@ class Account::ProviderImportAdapterTest < ActiveSupport::TestCase assert_equal 10, updated_entry.entryable.qty assert_equal 200.00, updated_entry.entryable.price assert_equal "USD", updated_entry.entryable.currency + assert_equal 0.95, updated_entry.entryable.exchange_rate # Entry attributes should also be updated assert_equal 2000.00, updated_entry.amount end end + test "preserves existing exchange rate when reimport omits it" do + investment_account = accounts(:investment) + adapter = Account::ProviderImportAdapter.new(investment_account) + aapl = securities(:aapl) + + entry = adapter.import_trade( + external_id: "plaid_trade_exchange_rate_preserved", + security: aapl, + quantity: 5, + price: 150.00, + amount: 750.00, + currency: "USD", + date: Date.today, + source: "plaid", + exchange_rate: 0.95 + ) + + assert_no_difference "investment_account.entries.count" do + updated_entry = adapter.import_trade( + external_id: "plaid_trade_exchange_rate_preserved", + security: aapl, + quantity: 10, + price: 200.00, + amount: 2000.00, + currency: "USD", + date: Date.today, + source: "plaid" + ) + + assert_equal entry.id, updated_entry.id + assert_equal 0.95, updated_entry.entryable.exchange_rate + end + end + test "raises error when external_id collision occurs across different entryable types for transaction" do investment_account = accounts(:investment) adapter = Account::ProviderImportAdapter.new(investment_account) diff --git a/test/models/account/syncer_test.rb b/test/models/account/syncer_test.rb new file mode 100644 index 000000000..6eeaec117 --- /dev/null +++ b/test/models/account/syncer_test.rb @@ -0,0 +1,31 @@ +require "test_helper" +require "ostruct" + +class Account::SyncerTest < ActiveSupport::TestCase + test "applies IBKR historical balance overrides after materialization" do + family = families(:empty) + account = family.accounts.create!( + name: "IBKR Brokerage", + balance: 0, + cash_balance: 0, + currency: "CHF", + accountable: Investment.new(subtype: "brokerage") + ) + ibkr_account = family.ibkr_items.create!( + name: "IBKR", + query_id: "QUERY123", + token: "TOKEN123" + ).ibkr_accounts.create!( + name: "Main", + ibkr_account_id: "U1234567", + currency: "CHF" + ) + ibkr_account.ensure_account_provider!(account) + + Account::MarketDataImporter.any_instance.expects(:import_all).once + Balance::Materializer.any_instance.expects(:materialize_balances).once + IbkrAccount::HistoricalBalancesSync.any_instance.expects(:sync!).once + + Account::Syncer.new(account).perform_sync(OpenStruct.new(window_start_date: nil)) + end +end diff --git a/test/models/account_ibkr_creation_test.rb b/test/models/account_ibkr_creation_test.rb new file mode 100644 index 000000000..2ab8f6fcf --- /dev/null +++ b/test/models/account_ibkr_creation_test.rb @@ -0,0 +1,30 @@ +require "test_helper" + +class AccountIbkrCreationTest < ActiveSupport::TestCase + fixtures :families, :ibkr_items, :ibkr_accounts + + test "uses interactive brokers account id as part of the default name" do + ibkr_account = ibkr_accounts(:main_account) + + account = Account.create_from_ibkr_account(ibkr_account) + + assert_equal "Interactive Brokers (U1234567)", account.name + assert_equal "Investment", account.accountable_type + assert_equal "CHF", account.currency + end + + test "falls back to provider name when ibkr account id is missing" do + family = families(:empty) + ibkr_item = ibkr_items(:empty_item) + ibkr_account = ibkr_item.ibkr_accounts.create!( + name: "Imported IBKR Account", + ibkr_account_id: nil, + currency: "USD" + ) + + account = Account.create_from_ibkr_account(ibkr_account) + + assert_equal "Interactive Brokers", account.name + assert_equal family, account.family + end +end diff --git a/test/models/account_test.rb b/test/models/account_test.rb index 48bfbb6c3..766b5ab2f 100644 --- a/test/models/account_test.rb +++ b/test/models/account_test.rb @@ -288,4 +288,55 @@ class AccountTest < ActiveSupport::TestCase assert_equal "read_write", share.permission assert share.include_in_finances? end + + test "current_holdings prefers latest provider snapshot holdings across currencies" do + account = @family.accounts.create!( + owner: @admin, + name: "Linked Brokerage", + balance: 1000, + currency: "USD", + accountable: Investment.new + ) + + coinstats_item = @family.coinstats_items.create!(name: "CoinStats", api_key: "test-key") + coinstats_account = coinstats_item.coinstats_accounts.create!(name: "Brokerage", currency: "USD") + account_provider = AccountProvider.create!(account: account, provider: coinstats_account) + + eur_security = Security.create!(ticker: "ASML", name: "ASML") + chf_security = Security.create!(ticker: "NOVN", name: "Novartis") + + provider_holding = account.holdings.create!( + security: eur_security, + date: Date.current, + qty: 2, + price: 500, + amount: 1000, + currency: "EUR", + account_provider: account_provider, + cost_basis: 450 + ) + + account.holdings.create!( + security: eur_security, + date: Date.current, + qty: 2, + price: 540, + amount: 1080, + currency: "USD" + ) + + second_provider_holding = account.holdings.create!( + security: chf_security, + date: Date.current, + qty: 3, + price: 90, + amount: 270, + currency: "CHF", + account_provider: account_provider, + cost_basis: 80 + ) + + assert_equal [ provider_holding.id, second_provider_holding.id ].sort, account.current_holdings.pluck(:id).sort + assert_equal %w[CHF EUR], account.current_holdings.pluck(:currency).sort + end end diff --git a/test/models/balance/sync_cache_test.rb b/test/models/balance/sync_cache_test.rb index 775a730d3..cbc14df7b 100644 --- a/test/models/balance/sync_cache_test.rb +++ b/test/models/balance/sync_cache_test.rb @@ -60,6 +60,31 @@ class Balance::SyncCacheTest < ActiveSupport::TestCase assert_equal 120.0, converted_entry.amount # 100 * 1.2 = 120 end + test "uses custom exchange rate from trade when present" do + security = Security.create!(ticker: "TST", name: "Test") + + _entry = @account.entries.create!( + date: Date.current, + name: "Test Trade", + amount: 100, + currency: "EUR", + entryable: Trade.new( + security: security, + qty: 1, + price: 100, + currency: "EUR", + exchange_rate: 1.5 + ) + ) + + sync_cache = Balance::SyncCache.new(@account) + converted_entries = sync_cache.send(:converted_entries) + + converted_entry = converted_entries.first + assert_equal "USD", converted_entry.currency + assert_equal 150.0, converted_entry.amount + end + test "converts multiple entries with correct rates" do # Create exchange rates ExchangeRate.create!( @@ -197,4 +222,35 @@ class Balance::SyncCacheTest < ActiveSupport::TestCase # Should use custom rate (1.5), not fetched rate (1.2) assert_equal 150.0, converted_entry.amount # 100 * 1.5, not 100 * 1.2 end + + test "prioritizes trade custom rate over fetched rate" do + ExchangeRate.create!( + from_currency: "EUR", + to_currency: "USD", + date: Date.current, + rate: 1.2 + ) + + security = Security.create!(ticker: "TST2", name: "Test 2") + + _entry = @account.entries.create!( + date: Date.current, + name: "EUR Trade with custom rate", + amount: 100, + currency: "EUR", + entryable: Trade.new( + security: security, + qty: 1, + price: 100, + currency: "EUR", + exchange_rate: 1.5 + ) + ) + + sync_cache = Balance::SyncCache.new(@account) + converted_entries = sync_cache.send(:converted_entries) + + converted_entry = converted_entries.first + assert_equal 150.0, converted_entry.amount + end end diff --git a/test/models/holding/materializer_test.rb b/test/models/holding/materializer_test.rb index 4d14a26c8..605f00478 100644 --- a/test/models/holding/materializer_test.rb +++ b/test/models/holding/materializer_test.rb @@ -7,6 +7,7 @@ class Holding::MaterializerTest < ActiveSupport::TestCase @family = families(:empty) @account = @family.accounts.create!(name: "Test", balance: 20000, cash_balance: 20000, currency: "USD", accountable: Investment.new) @aapl = securities(:aapl) + @msft = securities(:msft) end test "syncs holdings" do @@ -123,4 +124,83 @@ class Holding::MaterializerTest < ActiveSupport::TestCase assert_equal BigDecimal("10"), yesterday_holding.qty assert_equal yesterday_holding.qty * yesterday_holding.price, yesterday_holding.amount end + + test "cleans up calculated current-day holdings when a provider snapshot exists in another currency" do + ExchangeRate.create!(from_currency: "EUR", to_currency: "USD", date: Date.current, rate: 1.2) + + coinstats_item = @family.coinstats_items.create!(name: "CoinStats", api_key: "test-key") + coinstats_account = coinstats_item.coinstats_accounts.create!( + name: "Brokerage", + currency: "USD" + ) + account_provider = AccountProvider.create!(account: @account, provider: coinstats_account) + + Holding.create!( + account: @account, + security: @aapl, + qty: 10, + price: 200, + amount: 2000, + currency: "EUR", + date: Date.current, + account_provider: account_provider, + cost_basis: 150 + ) + + Holding::Materializer.new(@account, strategy: :reverse).materialize_holdings + + today_holdings = @account.holdings.where(security: @aapl, date: Date.current).order(:currency) + + assert_equal [ "EUR" ], today_holdings.pluck(:currency) + assert_equal [ account_provider.id ], today_holdings.pluck(:account_provider_id) + end + + test "preserves same-day non-provider holdings for securities absent from the provider snapshot" do + ExchangeRate.create!(from_currency: "EUR", to_currency: "USD", date: Date.current, rate: 1.2) + + coinstats_item = @family.coinstats_items.create!(name: "CoinStats", api_key: "test-key") + coinstats_account = coinstats_item.coinstats_accounts.create!( + name: "Brokerage", + currency: "USD" + ) + account_provider = AccountProvider.create!(account: @account, provider: coinstats_account) + + Holding.create!( + account: @account, + security: @aapl, + qty: 10, + price: 200, + amount: 2000, + currency: "EUR", + date: Date.current, + account_provider: account_provider, + cost_basis: 150 + ) + + manual_holding = Holding.create!( + account: @account, + security: @msft, + qty: 3, + price: 250, + amount: 750, + currency: "USD", + date: Date.current, + cost_basis: 225, + cost_basis_source: "manual", + cost_basis_locked: true + ) + + Holding::Materializer.new(@account, strategy: :reverse).materialize_holdings + + assert_equal manual_holding.id, manual_holding.reload.id + assert_equal @msft.id, manual_holding.security_id + assert_nil manual_holding.account_provider_id + + today_holdings = @account.holdings.where(date: Date.current) + + assert_equal( + [ [ @aapl.id, "EUR" ], [ @msft.id, "USD" ] ].sort, + today_holdings.pluck(:security_id, :currency).sort + ) + end end diff --git a/test/models/holding/portfolio_cache_test.rb b/test/models/holding/portfolio_cache_test.rb index 4677fc411..fed652165 100644 --- a/test/models/holding/portfolio_cache_test.rb +++ b/test/models/holding/portfolio_cache_test.rb @@ -56,4 +56,40 @@ class Holding::PortfolioCacheTest < ActiveSupport::TestCase cache = Holding::PortfolioCache.new(@account, use_holdings: true) assert_equal holding.price, cache.get_price(@security.id, holding.date).price end + + test "converts historical prices using the requested date exchange rate" do + account = families(:empty).accounts.create!( + name: "CHF Brokerage", + balance: 10000, + currency: "CHF", + accountable: Investment.new + ) + holding_date = 2.days.ago.to_date + + ExchangeRate.create!(from_currency: "USD", to_currency: "CHF", date: holding_date, rate: 0.80) + ExchangeRate.create!(from_currency: "USD", to_currency: "CHF", date: Date.current, rate: 0.95) + + Holding.create!( + security: @security, + account: account, + date: holding_date, + qty: 1, + price: 100, + amount: 100, + currency: "USD" + ) + + Security::Price.create!( + security: @security, + date: holding_date, + price: 100, + currency: "USD" + ) + + cache = Holding::PortfolioCache.new(account, use_holdings: true) + converted_price = cache.get_price(@security.id, holding_date) + + assert_equal BigDecimal("80.0"), converted_price.price + assert_equal "CHF", converted_price.currency + end end diff --git a/test/models/holding_test.rb b/test/models/holding_test.rb index 39fdf6ac1..32eb1b072 100644 --- a/test/models/holding_test.rb +++ b/test/models/holding_test.rb @@ -20,6 +20,22 @@ class HoldingTest < ActiveSupport::TestCase assert_in_delta expected_nvda_weight, @nvda.weight, 0.001 end + test "calculates portfolio weight after converting foreign-currency holdings" do + ExchangeRate.create!(from_currency: "EUR", to_currency: "USD", date: Date.current, rate: 1.5) + + foreign_security = Security.create!(ticker: "ASML", name: "ASML") + foreign_holding = @account.holdings.create!( + security: foreign_security, + date: Date.current, + qty: 1, + price: 100, + amount: 100, + currency: "EUR" + ) + + assert_in_delta 0.75, foreign_holding.weight, 0.001 + end + test "calculates average cost basis" do create_trade(@amzn.security, account: @account, qty: 10, price: 212.00, date: 1.day.ago.to_date) create_trade(@amzn.security, account: @account, qty: 15, price: 216.00, date: Date.current) diff --git a/test/models/ibkr_account/historical_balances_sync_test.rb b/test/models/ibkr_account/historical_balances_sync_test.rb new file mode 100644 index 000000000..22d082284 --- /dev/null +++ b/test/models/ibkr_account/historical_balances_sync_test.rb @@ -0,0 +1,116 @@ +require "test_helper" + +class IbkrAccount::HistoricalBalancesSyncTest < ActiveSupport::TestCase + setup do + @family = families(:empty) + @account = @family.accounts.create!( + name: "IBKR Brokerage", + balance: 0, + cash_balance: 0, + currency: "CHF", + accountable: Investment.new(subtype: "brokerage") + ) + @ibkr_account = @family.ibkr_items.create!( + name: "IBKR", + query_id: "QUERY123", + token: "TOKEN123" + ).ibkr_accounts.create!( + name: "Main", + ibkr_account_id: "U1234567", + currency: "CHF", + current_balance: 3351, + cash_balance: 1000.5, + raw_equity_summary_payload: [ + { + currency: "CHF", + report_date: "2026-05-07", + cash: "900.50", + stock: "2300.50", + total: "3201.00" + }, + { + currency: "CHF", + report_date: "2026-05-08", + cash: "1000.50", + stock: "2350.50", + total: "3351.00" + } + ] + ) + @ibkr_account.ensure_account_provider!(@account) + end + + test "upserts historical balances without creating activity entries" do + @account.balances.create!( + date: Date.new(2026, 5, 7), + balance: 0, + cash_balance: 0, + currency: "CHF", + start_cash_balance: 0, + start_non_cash_balance: 0, + cash_inflows: 0, + cash_outflows: 0, + non_cash_inflows: 0, + non_cash_outflows: 0, + net_market_flows: 0, + cash_adjustments: 0, + non_cash_adjustments: 0, + flows_factor: 1 + ) + + assert_no_difference "@account.entries.count" do + IbkrAccount::HistoricalBalancesSync.new(@ibkr_account).sync! + end + + first_balance = @account.balances.find_by!(date: Date.new(2026, 5, 7), currency: "CHF") + second_balance = @account.balances.find_by!(date: Date.new(2026, 5, 8), currency: "CHF") + + assert_equal BigDecimal("3201.0"), first_balance.end_balance + assert_equal BigDecimal("900.5"), first_balance.end_cash_balance + assert_equal BigDecimal("2300.5"), first_balance.end_non_cash_balance + + assert_equal BigDecimal("3351.0"), second_balance.end_balance + assert_equal BigDecimal("1000.5"), second_balance.end_cash_balance + assert_equal BigDecimal("2350.5"), second_balance.end_non_cash_balance + assert_equal BigDecimal("900.5"), second_balance.start_cash_balance + assert_equal BigDecimal("2300.5"), second_balance.start_non_cash_balance + end + + test "accepts equity summary rows when stored account currency casing differs" do + @ibkr_account.update!(currency: "chf") + + IbkrAccount::HistoricalBalancesSync.new(@ibkr_account).sync! + + first_balance = @account.balances.find_by!(date: Date.new(2026, 5, 7), currency: "CHF") + second_balance = @account.balances.find_by!(date: Date.new(2026, 5, 8), currency: "CHF") + + assert_equal BigDecimal("3201.0"), first_balance.end_balance + assert_equal BigDecimal("3351.0"), second_balance.end_balance + end + + test "skips malformed equity summary rows and still imports valid rows" do + @ibkr_account.update!( + raw_equity_summary_payload: [ + nil, + "bad-row", + [], + { + currency: "CHF", + report_date: "2026-05-09", + cash: "1100.50", + total: "3400.00" + } + ] + ) + + assert_nothing_raised do + IbkrAccount::HistoricalBalancesSync.new(@ibkr_account).sync! + end + + balance = @account.balances.find_by!(date: Date.new(2026, 5, 9), currency: "CHF") + + assert_equal BigDecimal("3400.0"), balance.end_balance + assert_equal BigDecimal("1100.5"), balance.end_cash_balance + assert_equal BigDecimal("2299.5"), balance.end_non_cash_balance + end +end diff --git a/test/models/ibkr_account_processor_test.rb b/test/models/ibkr_account_processor_test.rb new file mode 100644 index 000000000..94555bb61 --- /dev/null +++ b/test/models/ibkr_account_processor_test.rb @@ -0,0 +1,281 @@ +require "test_helper" + +class IbkrAccountProcessorTest < ActiveSupport::TestCase + fixtures :families, :ibkr_items, :ibkr_accounts, :accounts, :securities + + setup do + @family = families(:dylan_family) + @ibkr_account = ibkr_accounts(:main_account) + + @account = @family.accounts.create!( + name: "IBKR Investment", + balance: 0, + cash_balance: 0, + currency: "CHF", + accountable: Investment.new(subtype: "brokerage") + ) + @ibkr_account.ensure_account_provider!(@account) + @ibkr_account.update!( + raw_holdings_payload: [ + { + "asset_category" => "STK", + "conid" => "265598", + "security_id" => "US0378331005", + "security_id_type" => "ISIN", + "symbol" => securities(:aapl).ticker, + "position" => "10", + "mark_price" => "150.00", + "currency" => "USD", + "fx_rate_to_base" => "0.90", + "cost_basis_price" => "125.50", + "report_date" => Date.current.to_s, + "side" => "Long" + } + ], + raw_activities_payload: { + trades: [ + { + "asset_category" => "STK", + "trade_id" => "1001", + "transaction_id" => "1001a", + "conid" => "265598", + "symbol" => securities(:aapl).ticker, + "quantity" => "2", + "trade_price" => "140.00", + "currency" => "USD", + "fx_rate_to_base" => "0.90", + "buy_sell" => "BUY", + "trade_date" => Date.current.to_s, + "ib_commission" => "-1.25", + "ib_commission_currency" => "USD" + }, + { + "asset_category" => "STK", + "trade_id" => "1002", + "transaction_id" => "1002a", + "conid" => "265598", + "symbol" => securities(:aapl).ticker, + "quantity" => "-1", + "trade_price" => "155.00", + "currency" => "USD", + "fx_rate_to_base" => "0.92", + "buy_sell" => "SELL", + "trade_date" => Date.current.to_s, + "ib_commission" => "-1.10", + "ib_commission_currency" => "USD" + } + ], + cash_transactions: [ + { + "transaction_id" => "4001", + "type" => "Deposits/Withdrawals", + "amount" => "500.00", + "currency" => "CHF", + "fx_rate_to_base" => "1", + "report_date" => Date.current.to_s + }, + { + "transaction_id" => "4002", + "type" => "Dividends", + "amount" => "2.50", + "currency" => "USD", + "fx_rate_to_base" => "0.91", + "report_date" => Date.current.to_s, + "conid" => "265598" + } + ] + }, + report_date: Date.current, + current_balance: BigDecimal("3351.00"), + cash_balance: BigDecimal("1000.50"), + currency: "CHF" + ) + end + + test "processor imports holdings, trades, cash transactions, and commissions" do + IbkrAccount::Processor.new(@ibkr_account).process + + @account.reload + assert_equal BigDecimal("3351.00"), @account.balance + assert_equal BigDecimal("1000.50"), @account.cash_balance + assert_equal "CHF", @account.currency + + holding = @account.holdings.find_by(security: securities(:aapl), date: Date.current) + assert_not_nil holding + assert_equal BigDecimal("10"), holding.qty + assert_equal BigDecimal("150.00"), holding.price + assert_equal BigDecimal("125.50"), holding.cost_basis + assert_equal "USD", holding.currency + + buy_trade = @account.entries.find_by(external_id: "ibkr_trade_1001") + sell_trade = @account.entries.find_by(external_id: "ibkr_trade_1002") + assert_not_nil buy_trade + assert_not_nil sell_trade + assert_equal "Buy", buy_trade.entryable.investment_activity_label + assert_equal "Sell", sell_trade.entryable.investment_activity_label + assert_equal BigDecimal("2"), buy_trade.entryable.qty + assert_equal BigDecimal("-1"), sell_trade.entryable.qty + assert_equal BigDecimal("280.0"), buy_trade.amount + assert_equal BigDecimal("-155.0"), sell_trade.amount + assert_equal "USD", buy_trade.currency + assert_equal "USD", sell_trade.currency + assert_equal 0.9, buy_trade.entryable.exchange_rate + assert_equal 0.92, sell_trade.entryable.exchange_rate + + dividend = @account.entries.find_by(external_id: "ibkr_cash_4002") + assert_not_nil dividend + assert_equal "Dividend", dividend.entryable.investment_activity_label + assert_equal BigDecimal("-2.5"), dividend.amount + assert_equal securities(:aapl).id, dividend.entryable.extra["security_id"] + + commission_one = @account.entries.find_by(external_id: "ibkr_trade_fee_1001") + commission_two = @account.entries.find_by(external_id: "ibkr_trade_fee_1002") + assert_not_nil commission_one + assert_not_nil commission_two + assert_equal BigDecimal("1.25"), commission_one.amount + assert_equal BigDecimal("1.1"), commission_two.amount + assert_equal "USD", commission_one.currency + assert_equal "USD", commission_two.currency + assert_equal securities(:aapl).id, commission_one.entryable.extra["security_id"] + assert_equal securities(:aapl).id, commission_two.entryable.extra["security_id"] + + deposit = @account.entries.find_by(external_id: "ibkr_cash_4001") + + assert_not_nil deposit + assert_equal "Contribution", deposit.entryable.investment_activity_label + assert_equal BigDecimal("-500"), deposit.amount + assert_equal "CHF", deposit.currency + + assert_equal "USD", dividend.currency + end + + test "processor computes weighted provider cost basis for grouped lots" do + @ibkr_account.update!( + raw_holdings_payload: [ + { + "asset_category" => "STK", + "conid" => "265598", + "security_id" => "US0378331005", + "security_id_type" => "ISIN", + "symbol" => securities(:aapl).ticker, + "position" => "10", + "mark_price" => "150.00", + "currency" => "USD", + "fx_rate_to_base" => "0.90", + "cost_basis_price" => "125.50", + "report_date" => Date.current.to_s, + "side" => "Long" + }, + { + "asset_category" => "STK", + "conid" => "265598", + "security_id" => "US0378331005", + "security_id_type" => "ISIN", + "symbol" => securities(:aapl).ticker, + "position" => "20", + "mark_price" => "150.00", + "currency" => "USD", + "fx_rate_to_base" => "0.90", + "cost_basis_price" => "122.00", + "report_date" => Date.current.to_s, + "side" => "Long" + } + ] + ) + + IbkrAccount::Processor.new(@ibkr_account).process + + holding = @account.holdings.find_by(security: securities(:aapl), date: Date.current) + + assert_not_nil holding + assert_equal BigDecimal("30"), holding.qty + assert_equal BigDecimal("123.1667"), holding.cost_basis + end + + test "processor repairs default opening anchor after importing activity entries" do + result = Account::OpeningBalanceManager.new(@account).set_opening_balance( + balance: @ibkr_account.current_balance, + date: 2.years.ago.to_date + ) + + assert result.success? + + opening_anchor = @account.valuations.opening_anchor.includes(:entry).first + assert_not_nil opening_anchor + assert_equal @ibkr_account.current_balance.to_d, opening_anchor.entry.amount.to_d + + IbkrAccount::Processor.new(@ibkr_account).process + + opening_anchor.reload + assert_equal BigDecimal("0"), opening_anchor.entry.amount.to_d + end + + test "processor imports commission-free trades without creating fee entries" do + @ibkr_account.update!( + raw_activities_payload: { + trades: [ + { + "asset_category" => "STK", + "trade_id" => "1003", + "transaction_id" => "1003a", + "conid" => "265598", + "symbol" => securities(:aapl).ticker, + "quantity" => "3", + "trade_price" => "145.00", + "currency" => "USD", + "fx_rate_to_base" => "0.91", + "buy_sell" => "BUY", + "trade_date" => Date.current.to_s + } + ], + cash_transactions: [] + } + ) + + IbkrAccount::Processor.new(@ibkr_account).process + + trade = @account.entries.find_by(external_id: "ibkr_trade_1003") + fee = @account.entries.find_by(external_id: "ibkr_trade_fee_1003") + + assert_not_nil trade + assert_equal BigDecimal("3"), trade.entryable.qty + assert_equal BigDecimal("435.0"), trade.amount + assert_equal "USD", trade.currency + assert_nil fee + end + + test "processor logs and falls back to current date for invalid trade_date" do + @ibkr_account.update!( + raw_activities_payload: { + trades: [ + { + "asset_category" => "STK", + "trade_id" => "1004", + "transaction_id" => "1004a", + "conid" => "265598", + "symbol" => securities(:aapl).ticker, + "quantity" => "1", + "trade_price" => "146.00", + "currency" => "USD", + "fx_rate_to_base" => "0.91", + "buy_sell" => "BUY", + "trade_date" => "not-a-date" + } + ], + cash_transactions: [] + } + ) + + Rails.logger.expects(:warn).with do |message| + message.include?("IbkrAccount::DataHelpers - Missing or invalid trade_date") && + message.include?("1004") + end + + IbkrAccount::Processor.new(@ibkr_account).process + + trade = @account.entries.find_by(external_id: "ibkr_trade_1004") + + assert_not_nil trade + assert_equal Date.current, trade.date + end +end diff --git a/test/models/ibkr_item/sync_complete_event_test.rb b/test/models/ibkr_item/sync_complete_event_test.rb new file mode 100644 index 000000000..5fe628434 --- /dev/null +++ b/test/models/ibkr_item/sync_complete_event_test.rb @@ -0,0 +1,23 @@ +require "test_helper" + +class IbkrItem::SyncCompleteEventTest < ActiveSupport::TestCase + fixtures :families, :ibkr_items + + test "broadcast refreshes linked accounts, provider item, and family stream" do + ibkr_item = ibkr_items(:configured_item) + family = ibkr_item.family + account = mock("account") + + ibkr_item.stubs(:accounts).returns([ account ]) + account.expects(:broadcast_sync_complete).once + ibkr_item.expects(:broadcast_replace_to).with( + family, + target: "ibkr_item_#{ibkr_item.id}", + partial: "ibkr_items/ibkr_item", + locals: { ibkr_item: ibkr_item } + ).once + family.expects(:broadcast_sync_complete).once + + IbkrItem::SyncCompleteEvent.new(ibkr_item).broadcast + end +end diff --git a/test/models/ibkr_item/syncer_test.rb b/test/models/ibkr_item/syncer_test.rb new file mode 100644 index 000000000..ecb6786bb --- /dev/null +++ b/test/models/ibkr_item/syncer_test.rb @@ -0,0 +1,26 @@ +require "test_helper" + +class IbkrItem::SyncerTest < ActiveSupport::TestCase + fixtures :families, :ibkr_items + + setup do + @ibkr_item = ibkr_items(:configured_item) + end + + test "perform_sync records a single auth error when credentials are missing" do + @ibkr_item.update!(token: nil) + syncer = IbkrItem::Syncer.new(@ibkr_item) + sync = @ibkr_item.syncs.create! + + error = assert_raises(Provider::IbkrFlex::ConfigurationError) do + syncer.perform_sync(sync) + end + + assert_equal "IBKR credentials are missing.", error.message + assert_equal "requires_update", @ibkr_item.reload.status + + stats = sync.reload.sync_stats + assert_equal 1, stats["total_errors"] + assert_equal [ { "message" => "IBKR credentials are missing.", "category" => "auth_error" } ], stats["errors"] + end +end diff --git a/test/models/ibkr_item_importer_test.rb b/test/models/ibkr_item_importer_test.rb new file mode 100644 index 000000000..c91d967e5 --- /dev/null +++ b/test/models/ibkr_item_importer_test.rb @@ -0,0 +1,42 @@ +require "test_helper" + +class IbkrItemImporterTest < ActiveSupport::TestCase + setup do + @family = families(:empty) + @ibkr_item = @family.ibkr_items.create!( + name: "Interactive Brokers", + query_id: "QUERY123", + token: "TOKEN123" + ) + end + + test "imports accounts from parsed flex statement" do + provider = mock("ibkr_provider") + provider.expects(:download_statement).returns(file_fixture("ibkr/flex_statement.xml").read) + + assert_difference "IbkrAccount.count", 2 do + result = IbkrItem::Importer.new(@ibkr_item, ibkr_provider: provider).import + assert_equal true, result[:success] + assert_equal 2, result[:accounts_imported] + end + + primary_account = @ibkr_item.ibkr_accounts.find_by!(ibkr_account_id: "U1234567") + assert_equal "CHF", primary_account.currency + assert_equal BigDecimal("3351.0"), primary_account.current_balance + assert_equal 2, primary_account.raw_equity_summary_payload.size + assert_equal 1, primary_account.raw_holdings_payload.size + assert_equal 2, primary_account.raw_activities_payload["trades"].size + assert_equal 2, primary_account.raw_activities_payload["cash_transactions"].size + end + + test "raises parse error for malformed flex statement xml" do + provider = mock("ibkr_provider") + provider.expects(:download_statement).returns("") + + error = assert_raises(IbkrItem::ReportParser::ParseError) do + IbkrItem::Importer.new(@ibkr_item, ibkr_provider: provider).import + end + + assert_match "Invalid IBKR Flex XML", error.message + end +end diff --git a/test/models/ibkr_item_report_parser_test.rb b/test/models/ibkr_item_report_parser_test.rb new file mode 100644 index 000000000..b2c252221 --- /dev/null +++ b/test/models/ibkr_item_report_parser_test.rb @@ -0,0 +1,58 @@ +require "test_helper" + +class IbkrItemReportParserTest < ActiveSupport::TestCase + test "parses accounts, balances, and positions from flex xml" do + parsed = IbkrItem::ReportParser.new(file_fixture("ibkr/flex_statement.xml").read).parse + + assert_equal "Sure Test", parsed[:metadata]["query_name"] + assert_equal 2, parsed[:accounts].size + + first_account = parsed[:accounts].first + assert_equal "U1234567", first_account[:ibkr_account_id] + assert_equal "CHF", first_account[:currency] + assert_equal BigDecimal("1000.50"), first_account[:cash_balance] + assert_equal BigDecimal("3351.00"), first_account[:current_balance] + assert_equal 2, first_account[:equity_summary_in_base].size + assert_equal 1, first_account[:open_positions].size + assert_equal 2, first_account[:trades].size + assert_equal 2, first_account[:cash_transactions].size + + second_account = parsed[:accounts].second + assert_equal "U7654321", second_account[:ibkr_account_id] + assert_equal BigDecimal("250"), second_account[:cash_balance] + assert_equal BigDecimal("250"), second_account[:current_balance] + assert_equal 1, second_account[:equity_summary_in_base].size + end + + test "raises parse error for malformed xml" do + error = assert_raises(IbkrItem::ReportParser::ParseError) do + IbkrItem::ReportParser.new("").parse + end + + assert_match "Invalid IBKR Flex XML", error.message + end + + test "raises parse error when flex statements are missing" do + error = assert_raises(IbkrItem::ReportParser::ParseError) do + IbkrItem::ReportParser.new('').parse + end + + assert_equal "Invalid IBKR Flex XML: no FlexStatement nodes found.", error.message + end + + test "raises parse error when flex statement account id is missing" do + xml = <<~XML + + + + + + XML + + error = assert_raises(IbkrItem::ReportParser::ParseError) do + IbkrItem::ReportParser.new(xml).parse + end + + assert_equal "Invalid IBKR Flex XML: missing account identifier in FlexStatement.", error.message + end +end diff --git a/test/models/ibkr_item_test.rb b/test/models/ibkr_item_test.rb new file mode 100644 index 000000000..53737a120 --- /dev/null +++ b/test/models/ibkr_item_test.rb @@ -0,0 +1,20 @@ +require "test_helper" + +class IbkrItemTest < ActiveSupport::TestCase + fixtures :families, :ibkr_items + + test "syncable excludes items without token" do + item = IbkrItem.create!( + family: families(:empty), + name: "Interactive Brokers", + query_id: "QUERYNEW", + token: "TOKENNEW" + ) + + item.token = nil + item.save!(validate: false) + + assert_includes IbkrItem.syncable, ibkr_items(:configured_item) + refute_includes IbkrItem.syncable, item + end +end diff --git a/test/models/snaptrade_account_processor_test.rb b/test/models/snaptrade_account_processor_test.rb index 154b0690c..afc90c787 100644 --- a/test/models/snaptrade_account_processor_test.rb +++ b/test/models/snaptrade_account_processor_test.rb @@ -129,6 +129,36 @@ class SnaptradeAccountProcessorTest < ActiveSupport::TestCase assert_equal 0, @account.holdings.count end + test "processor trusts API total for multi-currency holdings" do + security = securities(:aapl) + Account.any_instance.stubs(:set_current_balance) + + @snaptrade_account.update!( + currency: "CHF", + current_balance: BigDecimal("15000.00"), + cash_balance: BigDecimal("1000.00"), + raw_holdings_payload: [ + { + "symbol" => { + "symbol" => { "symbol" => security.ticker, "description" => security.name } + }, + "units" => "10", + "price" => "150.00", + "currency" => "USD", + "average_purchase_price" => "125.50" + } + ], + raw_activities_payload: [] + ) + + SnaptradeAccount::Processor.new(@snaptrade_account).process + + @account.reload + assert_equal BigDecimal("15000.00"), @account.balance + assert_equal BigDecimal("1000.00"), @account.cash_balance + assert_equal "CHF", @account.currency + end + # === ActivitiesProcessor Tests === test "activities processor maps BUY type to Buy label" do diff --git a/test/models/trade_test.rb b/test/models/trade_test.rb index bcf09b192..6223a5fc7 100644 --- a/test/models/trade_test.rb +++ b/test/models/trade_test.rb @@ -52,6 +52,30 @@ class TradeTest < ActiveSupport::TestCase assert_equal 0, trade.fee end + test "exchange_rate setter stores normalized numeric value in extra" do + trade = Trade.new + trade.exchange_rate = "0.91" + + assert_equal 0.91, trade.exchange_rate + assert_equal 0.91, trade.extra["exchange_rate"] + end + + test "exchange_rate validation rejects invalid values" do + trade = Trade.new + trade.exchange_rate = "invalid" + + assert_not trade.valid? + assert_includes trade.errors[:exchange_rate], "must be a number" + end + + test "exchange_rate validation rejects non-finite values" do + trade = Trade.new + trade.exchange_rate = "NaN" + + assert_not trade.valid? + assert_includes trade.errors[:exchange_rate], "must be a number" + end + test "price is rounded to 10 decimal places" do security = Security.create!(ticker: "TEST", exchange_operating_mic: "XNAS") diff --git a/test/models/transaction/activity_security_preloader_test.rb b/test/models/transaction/activity_security_preloader_test.rb new file mode 100644 index 000000000..017dfb865 --- /dev/null +++ b/test/models/transaction/activity_security_preloader_test.rb @@ -0,0 +1,28 @@ +require "test_helper" + +class Transaction::ActivitySecurityPreloaderTest < ActiveSupport::TestCase + test "preloads activity securities for transactions" do + transaction = Transaction.new(extra: { "security_id" => securities(:aapl).id }) + + Transaction::ActivitySecurityPreloader.new([ transaction ]).preload + + assert_equal securities(:aapl), transaction.activity_security + end + + test "preloads activity securities for entry collections" do + transaction = Transaction.new(extra: { "security_id" => securities(:aapl).id }) + entry = Entry.new(account: accounts(:depository), entryable: transaction, date: Date.current, name: "Dividend", amount: 10, currency: "USD") + + Transaction::ActivitySecurityPreloader.new([ entry ]).preload + + assert_equal securities(:aapl), transaction.activity_security + end + + test "sets nil when the referenced security cannot be found" do + transaction = Transaction.new(extra: { "security_id" => SecureRandom.uuid }) + + Transaction::ActivitySecurityPreloader.new([ transaction ]).preload + + assert_nil transaction.activity_security + end +end diff --git a/test/models/transaction_test.rb b/test/models/transaction_test.rb index b6086d5c7..dfbf8e96b 100644 --- a/test/models/transaction_test.rb +++ b/test/models/transaction_test.rb @@ -100,6 +100,23 @@ class TransactionTest < ActiveSupport::TestCase assert transaction.extra["exchange_rate_invalid"] end + test "exchange_rate setter rejects non-finite input" do + transaction = Transaction.new + transaction.exchange_rate = "Infinity" + + assert_equal "Infinity", transaction.extra["exchange_rate"] + assert transaction.extra["exchange_rate_invalid"] + end + + test "exchange_rate setter clears invalid flag for valid input" do + transaction = Transaction.new + transaction.exchange_rate = "not a number" + transaction.exchange_rate = "1.5" + + assert_equal 1.5, transaction.exchange_rate + assert_equal false, transaction.extra["exchange_rate_invalid"] + end + test "exchange_rate validation rejects non-numeric input" do transaction = Transaction.new( category: categories(:income), @@ -139,4 +156,27 @@ class TransactionTest < ActiveSupport::TestCase assert transaction.valid? end + + test "activity_security returns the referenced security from extra metadata" do + security = securities(:aapl) + transaction = Transaction.new(extra: { "security_id" => security.id }) + + assert_equal security, transaction.activity_security + end + + test "activity_security returns nil when no security metadata is present" do + transaction = Transaction.new(extra: {}) + + assert_nil transaction.activity_security + end + + test "activity_security refreshes when security metadata changes on the same instance" do + transaction = Transaction.new(extra: { "security_id" => securities(:aapl).id }) + + assert_equal securities(:aapl), transaction.activity_security + + transaction.extra["security_id"] = securities(:msft).id + + assert_equal securities(:msft), transaction.activity_security + end end From 6106341e49775e4fc8cce2d7ace74495a16847c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Mata?= Date: Wed, 13 May 2026 13:03:59 +0200 Subject: [PATCH 26/31] Update token usage in publish.yml workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan José Mata --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index afdfd35a7..b7b74e37f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -429,7 +429,7 @@ jobs: uses: actions/checkout@v4.2.0 with: ref: ${{ steps.source_branch.outputs.branch }} - token: ${{ secrets.GH_PAT }} + token: ${{ secrets.GH_PAT || github.token }} - name: Bump pre-release version run: | From 7f9b1439e7345810caecc693a18bf9b054a22047 Mon Sep 17 00:00:00 2001 From: "Sure Admin (bot)" Date: Wed, 13 May 2026 13:59:14 +0200 Subject: [PATCH 27/31] ci: split unit and system test jobs (#1787) Co-authored-by: KiloClaw --- .github/workflows/ci.yml | 48 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eae6bb5e3..a040c7a31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: - name: Lint/Format js code run: npm run lint - test: + test_unit: runs-on: ubuntu-latest timeout-minutes: 10 @@ -121,6 +121,52 @@ jobs: - name: Unit and integration tests run: bin/rails test + test_system: + runs-on: ubuntu-latest + timeout-minutes: 10 + + env: + PLAID_CLIENT_ID: foo + PLAID_SECRET: bar + DATABASE_URL: postgres://postgres:postgres@localhost:5432 + REDIS_URL: redis://localhost:6379 + RAILS_ENV: test + + services: + postgres: + image: postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: --health-cmd="pg_isready" --health-interval=10s --health-timeout=5s --health-retries=3 + + redis: + image: redis + ports: + - 6379:6379 + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + steps: + - name: Install packages + run: sudo apt-get update && sudo apt-get install --no-install-recommends -y google-chrome-stable curl libvips postgresql-client libpq-dev + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + bundler-cache: true + + - name: DB setup and smoke test + run: | + bin/rails db:create + bin/rails db:schema:load + bin/rails db:seed + - name: System tests run: DISABLE_PARALLELIZATION=true bin/rails test:system From 406e7217a12e3819b6c8919a5cc377a12dd2322b Mon Sep 17 00:00:00 2001 From: CrossDrain <32982516+CrossDrain@users.noreply.github.com> Date: Wed, 13 May 2026 12:03:37 +0000 Subject: [PATCH 28/31] =?UTF-8?q?fix(enable-banking):=20fix=20pending?= =?UTF-8?q?=E2=86=92posted=20auto-claim=20producing=20badge,=20duplicate,?= =?UTF-8?q?=20and=20wrong=20date=20(#1783)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(enable-banking): clear pending flag and prevent stale re-import after auto-claim When a booked transaction claims a pending entry via the amount/date heuristic (find_pending_transaction), two bugs caused the entry to remain incorrectly pending and the old pending transaction to reappear on subsequent syncs. Bug 1: The extra["enable_banking"]["pending"] flag was never cleared on the claimed entry. For simple booked transactions with nil extra the deep-merge path is skipped entirely, so the pending badge persisted forever. Bug 2: After the claim the old pending external_id (e.g. PDNG_123) stayed in the stored raw_transactions_payload. The importer's C4 filter only removes pending entries whose transaction_id matches a BOOK id — Enable Banking issues completely different ids for pending vs booked transactions — so PDNG_123 was never pruned. On the next sync find_or_initialize_by(PDNG_123) couldn't find the claimed entry (now keyed as BOOK_456) and created a fresh pending duplicate with no category. Fix: on claim, explicitly clear all providers' pending keys from extra in-memory, and store the displaced pending external_id in extra["auto_claimed_pending_ids"]. The Processor now queries this field alongside manual_merge to build the excluded_ids set, so the stale pending data is skipped on every future sync. Co-Authored-By: Claude Sonnet 4.6 * fix(enable-banking): preserve pending date when claiming transactions When a pending transaction is claimed by a booked transaction, the original pending date is now preserved instead of being overwritten by the booked transaction's date. This ensures historical accuracy for transactions that were originally recorded on a different date. --------- Co-authored-by: Claude Sonnet 4.6 --- app/models/account/provider_import_adapter.rb | 22 ++++- .../transactions/processor.rb | 66 ++++++++++----- .../account/provider_import_adapter_test.rb | 45 ++++++++++ .../transactions/processor_test.rb | 83 +++++++++++++++++++ 4 files changed, 193 insertions(+), 23 deletions(-) diff --git a/app/models/account/provider_import_adapter.rb b/app/models/account/provider_import_adapter.rb index a207468aa..6b2dbf932 100644 --- a/app/models/account/provider_import_adapter.rb +++ b/app/models/account/provider_import_adapter.rb @@ -109,8 +109,28 @@ class Account::ProviderImportAdapter end if pending_match + old_pending_external_id = pending_match.external_id + pending_entry_date = pending_match.date entry = pending_match entry.assign_attributes(external_id: external_id) + + # Clear the pending flag so this entry no longer shows as pending after being claimed + # by a booked transaction. Also record the old external_id so the sync engine can + # exclude it from re-import (preventing the old pending from being recreated on the + # next sync when the stored raw payload still contains the pending transaction data). + if entry.entryable.is_a?(Transaction) + ex = (entry.transaction.extra || {}).deep_dup + Transaction::PENDING_PROVIDERS.each do |provider| + next unless ex.key?(provider) + ex[provider].delete("pending") + ex.delete(provider) if ex[provider].empty? + end + if old_pending_external_id.present? + existing_claims = Array.wrap(ex["auto_claimed_pending_ids"]) + ex["auto_claimed_pending_ids"] = (existing_claims + [ old_pending_external_id ]).uniq + end + entry.transaction.extra = ex + end end end @@ -120,7 +140,7 @@ class Account::ProviderImportAdapter entry.assign_attributes( amount: amount, currency: currency, - date: date + date: pending_entry_date || date ) # Use enrichment pattern to respect user overrides diff --git a/app/models/enable_banking_account/transactions/processor.rb b/app/models/enable_banking_account/transactions/processor.rb index fb50333e0..2809c6f9e 100644 --- a/app/models/enable_banking_account/transactions/processor.rb +++ b/app/models/enable_banking_account/transactions/processor.rb @@ -23,29 +23,51 @@ class EnableBankingAccount::Transactions::Processor Account::ProviderImportAdapter.new(enable_banking_account.current_account) end - # Pre-fetch external_ids that were manually merged and must not be re-imported. - # One query per sync; O(1) Set lookup per transaction — avoids N+1. - # Uses a lateral jsonb_array_elements join to extract only the ID strings in SQL, - # avoiding loading full extra blobs into Ruby. Handles both Array (current) and - # Hash (legacy) formats via jsonb_typeof. + # Pre-fetch external_ids that must not be re-imported. + # One query per category per sync; O(1) Set lookup per transaction — avoids N+1. excluded_ids = if enable_banking_account.current_account - Transaction.joins(:entry) - .where(entries: { account_id: enable_banking_account.current_account.id }) - .where("transactions.extra ? 'manual_merge'") - .joins( - Arel.sql(<<~SQL.squish) - CROSS JOIN LATERAL jsonb_array_elements( - CASE jsonb_typeof(transactions.extra->'manual_merge') - WHEN 'array' THEN transactions.extra->'manual_merge' - WHEN 'object' THEN jsonb_build_array(transactions.extra->'manual_merge') - ELSE '[]'::jsonb - END - ) AS merge_elem - SQL - ) - .pluck(Arel.sql("merge_elem->>'merged_from_external_id'")) - .compact - .to_set + account_id = enable_banking_account.current_account.id + + # 1. Manually merged: pending entries the user explicitly merged into a posted transaction. + # Uses a lateral join to extract merged_from_external_id from the manual_merge JSON + # (handles both Array current format and legacy Hash format via jsonb_typeof). + manually_merged_ids = Transaction.joins(:entry) + .where(entries: { account_id: account_id }) + .where("transactions.extra ? 'manual_merge'") + .joins( + Arel.sql(<<~SQL.squish) + CROSS JOIN LATERAL jsonb_array_elements( + CASE jsonb_typeof(transactions.extra->'manual_merge') + WHEN 'array' THEN transactions.extra->'manual_merge' + WHEN 'object' THEN jsonb_build_array(transactions.extra->'manual_merge') + ELSE '[]'::jsonb + END + ) AS merge_elem + SQL + ) + .pluck(Arel.sql("merge_elem->>'merged_from_external_id'")) + .compact + .to_set + + # 2. Auto-claimed: pending entries that were automatically matched to a booked transaction + # by the amount/date heuristic. Their old external_ids are stored in + # extra["auto_claimed_pending_ids"] so they are not re-imported as new pending entries + # on subsequent syncs (the stored raw payload still contains the old pending data). + auto_claimed_ids = Transaction.joins(:entry) + .where(entries: { account_id: account_id }) + .where("transactions.extra ? 'auto_claimed_pending_ids'") + .joins( + Arel.sql(<<~SQL.squish) + CROSS JOIN LATERAL jsonb_array_elements_text( + transactions.extra->'auto_claimed_pending_ids' + ) AS claimed_id + SQL + ) + .pluck(Arel.sql("claimed_id")) + .compact + .to_set + + manually_merged_ids | auto_claimed_ids else Set.new end diff --git a/test/models/account/provider_import_adapter_test.rb b/test/models/account/provider_import_adapter_test.rb index bbfa31b08..cff61d866 100644 --- a/test/models/account/provider_import_adapter_test.rb +++ b/test/models/account/provider_import_adapter_test.rb @@ -881,6 +881,51 @@ class Account::ProviderImportAdapterTest < ActiveSupport::TestCase end end + test "clears pending flag, preserves pending date, and records old external_id when claiming pending entry with nil extra" do + # Enable Banking booked transactions often have nil extra (no FX, no MCC). + # The deep_merge path is skipped, so we must clear the pending flag explicitly. + pending_date = Date.today - 2.days + pending_entry = @adapter.import_transaction( + external_id: "eb_pending_nil_extra", + amount: 42.00, + currency: "EUR", + date: pending_date, + name: "Supermarket", + source: "enable_banking", + extra: { "enable_banking" => { "pending" => true } } + ) + + assert pending_entry.transaction.pending?, "should be pending before claim" + + assert_no_difference "@account.entries.count" do + posted_entry = @adapter.import_transaction( + external_id: "eb_booked_nil_extra", + amount: 42.00, + currency: "EUR", + date: Date.today, # booked date is later than pending date + name: "Supermarket Posted", + source: "enable_banking", + extra: nil # typical for simple Enable Banking booked transactions + ) + + assert_equal pending_entry.id, posted_entry.id, "should claim the pending entry" + assert_equal "eb_booked_nil_extra", posted_entry.external_id + + posted_entry.reload + + # Pending flag must be cleared so the entry no longer shows a pending badge + assert_not posted_entry.transaction.pending?, "pending flag should be cleared after claim" + + # Date must be the original pending date, not the later booked date + assert_equal pending_date, posted_entry.date, "pending date should be preserved, not overwritten with booked date" + + # Old pending external_id must be stored so the sync engine can skip re-importing it + claimed_ids = posted_entry.transaction.extra&.dig("auto_claimed_pending_ids") || [] + assert_includes claimed_ids, "eb_pending_nil_extra", + "auto_claimed_pending_ids should record the old pending external_id" + end + end + test "does not reconcile when posted transaction has same external_id as pending" do # When external_id matches, normal dedup should handle it pending_entry = @adapter.import_transaction( diff --git a/test/models/enable_banking_account/transactions/processor_test.rb b/test/models/enable_banking_account/transactions/processor_test.rb index 00982bdef..81426dce3 100644 --- a/test/models/enable_banking_account/transactions/processor_test.rb +++ b/test/models/enable_banking_account/transactions/processor_test.rb @@ -186,6 +186,89 @@ class EnableBankingAccount::Transactions::ProcessorTest < ActiveSupport::TestCas result = EnableBankingAccount::Transactions::Processor.new(@enable_banking_account).process assert_equal 0, result[:failed] + end + + test "does not re-import a pending transaction whose external_id was auto-claimed" do + # When a pending entry is automatically matched to a booked transaction by the + # amount/date heuristic (find_pending_transaction), the old pending external_id + # is stored in auto_claimed_pending_ids so subsequent syncs don't recreate it. + pending_ext_id = "enable_banking_PDNG_AUTO_CLAIMED" + + booked_entry = create_transaction( + account: @account, + name: "Grocery Store", + date: 1.day.ago.to_date, + amount: 55, + currency: "EUR", + external_id: "enable_banking_BOOK_SETTLED", + source: "enable_banking" + ) + booked_entry.transaction.update!( + extra: { + "auto_claimed_pending_ids" => [ pending_ext_id ] + } + ) + + @enable_banking_account.update!( + raw_transactions_payload: [ + raw_pending_transaction(transaction_id: "PDNG_AUTO_CLAIMED") + ] + ) + + assert_no_difference "@account.entries.count" do + EnableBankingAccount::Transactions::Processor.new(@enable_banking_account).process + end + end + + test "does not re-import when both manual_merge and auto_claimed_pending_ids exclusions are present" do + manually_merged_ext_id = "enable_banking_PDNG_MANUAL" + auto_claimed_ext_id = "enable_banking_PDNG_AUTO" + + manual_entry = create_transaction( + account: @account, + name: "Manual Merge Entry", + date: 2.days.ago.to_date, + amount: 20, + currency: "EUR", + external_id: "enable_banking_BOOK_MANUAL", + source: "enable_banking" + ) + manual_entry.transaction.update!( + extra: { + "manual_merge" => { + "merged_from_external_id" => manually_merged_ext_id, + "merged_at" => Time.current.iso8601, + "source" => "enable_banking" + } + } + ) + + auto_entry = create_transaction( + account: @account, + name: "Auto Claimed Entry", + date: 1.day.ago.to_date, + amount: 30, + currency: "EUR", + external_id: "enable_banking_BOOK_AUTO", + source: "enable_banking" + ) + auto_entry.transaction.update!( + extra: { "auto_claimed_pending_ids" => [ auto_claimed_ext_id ] } + ) + + @enable_banking_account.update!( + raw_transactions_payload: [ + raw_pending_transaction(transaction_id: "PDNG_MANUAL"), # excluded via manual_merge + raw_pending_transaction(transaction_id: "PDNG_AUTO"), # excluded via auto_claimed_pending_ids + raw_pending_transaction(transaction_id: "PDNG_BRAND_NEW_XXXX") # new — should import + ] + ) + + result = nil + assert_difference "@account.entries.count", 1 do + result = EnableBankingAccount::Transactions::Processor.new(@enable_banking_account).process + end + assert_equal 2, result[:skipped] assert_equal 1, result[:imported] end From 7b21a619ec9d53991b824c2c78eaf7ad6e3763ca Mon Sep 17 00:00:00 2001 From: CrossDrain <32982516+CrossDrain@users.noreply.github.com> Date: Wed, 13 May 2026 15:54:09 +0000 Subject: [PATCH 29/31] fix(enable-banking): gracefully skip PDNG fetch for ASPSPs that don't support it (#1789) * fix(enable-banking): gracefully skip PDNG fetch for ASPSPs that don't support it Some banks reject the PDNG transaction status filter with a 422 validation error, causing the entire account sync to fail including booked transactions. Wrap the pending transaction fetch in a rescue block to catch validation errors from the provider. If the ASPSP does not support the "PDNG" status, the error is logged and the process continues without pending transactions instead of failing the entire import. * fix(enable-banking): gate PDNG fallback on transactionStatus error detail Tighten the rescue added in the previous commit so it only silences 422s that explicitly mention transactionStatus in the API error body. Any other validation error (bad date_from, malformed headers, etc.) re-raises and fails the sync as before, preventing silent data loss. Tests added for both branches: ASPSP-rejects-PDNG (success) and unrelated-validation-error (failure). --- app/models/enable_banking_item/importer.rb | 17 +++++---- .../importer_error_handling_test.rb | 36 +++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/app/models/enable_banking_item/importer.rb b/app/models/enable_banking_item/importer.rb index 708664212..696b4f673 100644 --- a/app/models/enable_banking_item/importer.rb +++ b/app/models/enable_banking_item/importer.rb @@ -243,12 +243,17 @@ class EnableBankingItem::Importer pending_transactions = [] if include_pending # Also fetch pending transactions (visible for 1-3 days before they become BOOK) if setting is enabled - pending_transactions = fetch_paginated_transactions( - enable_banking_account, - start_date: start_date, - transaction_status: "PDNG", - psu_headers: enable_banking_item.build_psu_headers - ) + begin + pending_transactions = fetch_paginated_transactions( + enable_banking_account, + start_date: start_date, + transaction_status: "PDNG", + psu_headers: enable_banking_item.build_psu_headers + ) + rescue Provider::EnableBanking::EnableBankingError => e + raise unless e.error_type == :validation_error && e.message.include?("transactionStatus") + Rails.logger.warn "EnableBankingItem::Importer - ASPSP does not support PDNG transaction status for account #{enable_banking_account.uid}, skipping pending transactions. API error: #{e.message}" + end end book_fingerprints = all_transactions diff --git a/test/models/enable_banking_item/importer_error_handling_test.rb b/test/models/enable_banking_item/importer_error_handling_test.rb index 305f07973..e5b9192cb 100644 --- a/test/models/enable_banking_item/importer_error_handling_test.rb +++ b/test/models/enable_banking_item/importer_error_handling_test.rb @@ -63,6 +63,42 @@ class EnableBankingItem::ImporterErrorHandlingTest < ActiveSupport::TestCase assert @enable_banking_item.reload.requires_update? end + test "fetch_and_store_transactions succeeds and skips pending when ASPSP rejects PDNG transaction_status" do + enable_banking_account = EnableBankingAccount.new(uid: "test_uid") + @importer.stubs(:determine_sync_start_date).returns(Date.today) + @importer.stubs(:include_pending?).returns(true) + + pdng_error = Provider::EnableBanking::EnableBankingError.new( + "Validation error from Enable Banking API: {\"message\":\"Wrong transactionStatus provided in getAccountTransactions call: PDNG\"}", + :validation_error + ) + + @importer.stubs(:fetch_paginated_transactions).with(enable_banking_account, has_entries(transaction_status: "BOOK")).returns([]) + @importer.stubs(:fetch_paginated_transactions).with(enable_banking_account, has_entries(transaction_status: "PDNG")).raises(pdng_error) + + result = @importer.send(:fetch_and_store_transactions, enable_banking_account) + + assert result[:success] + end + + test "fetch_and_store_transactions fails when validation error is unrelated to transactionStatus" do + enable_banking_account = EnableBankingAccount.new(uid: "test_uid") + @importer.stubs(:determine_sync_start_date).returns(Date.today) + @importer.stubs(:include_pending?).returns(true) + + date_error = Provider::EnableBanking::EnableBankingError.new( + "Validation error from Enable Banking API: {\"message\":\"Invalid date_from format\"}", + :validation_error + ) + + @importer.stubs(:fetch_paginated_transactions).with(enable_banking_account, has_entries(transaction_status: "BOOK")).returns([]) + @importer.stubs(:fetch_paginated_transactions).with(enable_banking_account, has_entries(transaction_status: "PDNG")).raises(date_error) + + result = @importer.send(:fetch_and_store_transactions, enable_banking_account) + + assert_not result[:success] + end + test "fetch_and_update_balance updates status to requires_update on unauthorized error" do enable_banking_account = EnableBankingAccount.new(uid: "test_uid") def @mock_provider.get_account_balances(**args) From 95f6451b3983df64f02e0ff25b999a7f31a92878 Mon Sep 17 00:00:00 2001 From: ghost <49853598+JSONbored@users.noreply.github.com> Date: Wed, 13 May 2026 09:13:48 -0700 Subject: [PATCH 30/31] feat(sync): add Brex provider connections (#1752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sync): add Brex provider schema Adds Brex item and account tables with per-family credentials, scoped upstream account uniqueness, encrypted token storage, and sanitized provider payload columns. * feat(sync): add Brex provider core Adds Brex item/account models, provider client and adapter support, family connection helpers, and provider enum registration for read-only Brex cash and card data. * feat(sync): add Brex import pipeline Adds Brex account discovery, linked-account sync, cash/card balance processors, transaction import, sanitized metadata handling, and idempotent provider entry processing. * feat(sync): add Brex connection flows Adds Mercury-style Brex connection management, explicit item-scoped account selection and linking, settings provider UI, account index visibility, localized copy, and per-item cache handling. * test(sync): cover Brex provider workflows Adds targeted coverage for Brex provider requests, adapter config, item/account guards, importer behavior, entry processing, and Mercury-style controller flows. * fix(sync): align Brex API edge cases Tightens Brex account fetching against the official card-account response shape, sends transaction start filters as RFC3339 date-times, and keeps provider error bodies out of user-facing messages while expanding provider client guard coverage. * fix(sync): harden Brex provider integration Restrict Brex API base URLs to official hosts, tighten account-selection UI behavior, and add tests for invalid credentials, cache scoping, and provider setup edge cases. * test(sync): avoid Brex secret-shaped fixtures * refactor(sync): extract Brex account flows * fix(sync): address Brex provider review feedback * fix(sync): address Brex review follow-ups Move remaining Brex review cleanup into focused model behavior, tighten link/setup edge cases, localize summaries, and add regression coverage from CodeRabbit feedback. Also records the security-review pass as no-findings after diff-scoped inspection and Brakeman validation. * refactor(sync): split Brex account flow controllers Route Brex account selection and setup actions through small namespaced controllers while keeping existing URLs and helpers stable. Business flow remains in BrexItem::AccountFlow; the main Brex item controller now only handles connection CRUD, provider-panel rendering, destroy, and sync. * fix(sync): address Brex CodeRabbit review * fix(sync): address Brex follow-up review * fix(sync): address Brex review follow-ups * fix(sync): address Brex sync review findings * fix(sync): polish Brex review copy and errors * fix(sync): register Brex provider health * fix(sync): polish Brex bank sync presentation * fix(sync): address Brex review follow-ups * fix(sync): tighten Brex setup params * test(api): stabilize usage rate-limit window * fix(sync): polish Brex setup flow nits * fix(sync): harden Brex setup params * fix(sync): finalize Brex review cleanup --------- Signed-off-by: Juan José Mata Co-authored-by: Juan José Mata --- .devcontainer/Dockerfile | 1 + .gitignore | 3 - app/controllers/accounts_controller.rb | 22 + .../brex_items/account_flows_controller.rb | 132 +++++ .../brex_items/account_setups_controller.rb | 109 ++++ app/controllers/brex_items_controller.rb | 98 ++++ .../settings/providers_controller.rb | 6 + app/helpers/brex_items_helper.rb | 76 +++ app/helpers/settings_helper.rb | 3 + .../account_type_selector_controller.js | 8 +- app/models/brex_account.rb | 204 ++++++++ app/models/brex_account/processor.rb | 78 +++ .../brex_account/transactions/processor.rb | 83 +++ app/models/brex_entry/processor.rb | 180 +++++++ app/models/brex_item.rb | 197 +++++++ app/models/brex_item/account_flow.rb | 425 +++++++++++++++ app/models/brex_item/account_flow/setup.rb | 242 +++++++++ app/models/brex_item/importer.rb | 245 +++++++++ app/models/brex_item/provided.rb | 16 + app/models/brex_item/syncer.rb | 148 ++++++ app/models/brex_item/unlinking.rb | 56 ++ app/models/concerns/encryptable.rb | 6 +- app/models/credit_card.rb | 2 + app/models/data_enrichment.rb | 1 + app/models/depository.rb | 2 + app/models/family.rb | 2 +- app/models/family/brex_connectable.rb | 29 ++ app/models/family/syncer.rb | 1 + app/models/provider/brex.rb | 271 ++++++++++ app/models/provider/brex_adapter.rb | 119 +++++ app/models/provider/metadata.rb | 1 + app/models/provider_connection_status.rb | 1 + app/models/provider_merchant.rb | 2 +- app/views/accounts/index.html.erb | 10 +- app/views/brex_items/_api_error.html.erb | 36 ++ app/views/brex_items/_brex_item.html.erb | 132 +++++ app/views/brex_items/_setup_required.html.erb | 34 ++ app/views/brex_items/_subtype_select.html.erb | 16 + app/views/brex_items/select_accounts.html.erb | 59 +++ .../select_existing_account.html.erb | 59 +++ app/views/brex_items/setup_accounts.html.erb | 106 ++++ .../settings/providers/_brex_panel.html.erb | 154 ++++++ .../providers/_drawer_header.html.erb | 4 +- .../initializers/active_record_encryption.rb | 8 +- config/locales/defaults/en.yml | 2 + config/locales/models/brex_item/en.yml | 14 + config/locales/views/brex_items/en.yml | 277 ++++++++++ config/locales/views/settings/en.yml | 1 + config/locales/views/valuations/nb.yml | 4 +- config/routes.rb | 16 + ...05010000_create_brex_items_and_accounts.rb | 59 +++ db/schema.rb | 45 ++ lib/active_record_encryption_config.rb | 58 +++ .../provider_connections_controller_test.rb | 18 + .../api/v1/usage_controller_test.rb | 38 +- .../controllers/brex_items_controller_test.rb | 488 ++++++++++++++++++ .../settings/providers_controller_test.rb | 36 ++ test/fixtures/brex_accounts.yml | 7 + test/fixtures/brex_items.yml | 7 + test/helpers/brex_items_helper_test.rb | 29 ++ .../active_record_encryption_config_test.rb | 56 ++ .../transactions/processor_test.rb | 92 ++++ test/models/brex_account_test.rb | 210 ++++++++ test/models/brex_entry/processor_test.rb | 131 +++++ test/models/brex_item/account_flow_test.rb | 394 ++++++++++++++ test/models/brex_item/importer_test.rb | 331 ++++++++++++ test/models/brex_item/syncer_test.rb | 136 +++++ test/models/brex_item_test.rb | 198 +++++++ test/models/family/syncer_test.rb | 7 + test/models/provider/brex_adapter_test.rb | 224 ++++++++ test/models/provider/brex_test.rb | 289 +++++++++++ 71 files changed, 6515 insertions(+), 39 deletions(-) create mode 100644 app/controllers/brex_items/account_flows_controller.rb create mode 100644 app/controllers/brex_items/account_setups_controller.rb create mode 100644 app/controllers/brex_items_controller.rb create mode 100644 app/helpers/brex_items_helper.rb create mode 100644 app/models/brex_account.rb create mode 100644 app/models/brex_account/processor.rb create mode 100644 app/models/brex_account/transactions/processor.rb create mode 100644 app/models/brex_entry/processor.rb create mode 100644 app/models/brex_item.rb create mode 100644 app/models/brex_item/account_flow.rb create mode 100644 app/models/brex_item/account_flow/setup.rb create mode 100644 app/models/brex_item/importer.rb create mode 100644 app/models/brex_item/provided.rb create mode 100644 app/models/brex_item/syncer.rb create mode 100644 app/models/brex_item/unlinking.rb create mode 100644 app/models/family/brex_connectable.rb create mode 100644 app/models/provider/brex.rb create mode 100644 app/models/provider/brex_adapter.rb create mode 100644 app/views/brex_items/_api_error.html.erb create mode 100644 app/views/brex_items/_brex_item.html.erb create mode 100644 app/views/brex_items/_setup_required.html.erb create mode 100644 app/views/brex_items/_subtype_select.html.erb create mode 100644 app/views/brex_items/select_accounts.html.erb create mode 100644 app/views/brex_items/select_existing_account.html.erb create mode 100644 app/views/brex_items/setup_accounts.html.erb create mode 100644 app/views/settings/providers/_brex_panel.html.erb create mode 100644 config/locales/models/brex_item/en.yml create mode 100644 config/locales/views/brex_items/en.yml create mode 100644 db/migrate/20260505010000_create_brex_items_and_accounts.rb create mode 100644 lib/active_record_encryption_config.rb create mode 100644 test/controllers/brex_items_controller_test.rb create mode 100644 test/fixtures/brex_accounts.yml create mode 100644 test/fixtures/brex_items.yml create mode 100644 test/helpers/brex_items_helper_test.rb create mode 100644 test/lib/active_record_encryption_config_test.rb create mode 100644 test/models/brex_account/transactions/processor_test.rb create mode 100644 test/models/brex_account_test.rb create mode 100644 test/models/brex_entry/processor_test.rb create mode 100644 test/models/brex_item/account_flow_test.rb create mode 100644 test/models/brex_item/importer_test.rb create mode 100644 test/models/brex_item/syncer_test.rb create mode 100644 test/models/brex_item_test.rb create mode 100644 test/models/provider/brex_adapter_test.rb create mode 100644 test/models/provider/brex_test.rb diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b871287d4..e024ef870 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update -qq \ git \ imagemagick \ iproute2 \ + libvips42 \ libpq-dev \ libyaml-dev \ libyaml-0-2 \ diff --git a/.gitignore b/.gitignore index e6b243328..6a4d77fe0 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,3 @@ scripts/ .claude_settings.json .security-key logs/security/ - -# Added by codex -.codex diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 4b4f34a31..084a37217 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -18,6 +18,7 @@ class AccountsController < ApplicationController @enable_banking_items = visible_provider_items(family.enable_banking_items.ordered.includes(:syncs)) @coinstats_items = visible_provider_items(family.coinstats_items.ordered.includes(:coinstats_accounts, :accounts, :syncs)) @mercury_items = visible_provider_items(family.mercury_items.ordered.includes(:syncs, :mercury_accounts)) + @brex_items = visible_provider_items(family.brex_items.ordered.includes(:accounts, :syncs, brex_accounts: :account_provider)) @coinbase_items = visible_provider_items(family.coinbase_items.ordered.includes(:coinbase_accounts, :accounts, :syncs)) @snaptrade_items = visible_provider_items(family.snaptrade_items.ordered.includes(:syncs, :snaptrade_accounts)) @ibkr_items = visible_provider_items(family.ibkr_items.ordered.includes(:syncs, :ibkr_accounts)) @@ -317,6 +318,27 @@ class AccountsController < ApplicationController @mercury_sync_stats_map[item.id] = latest_sync&.sync_stats || {} end + # Brex sync stats + @brex_sync_stats_map = {} + @brex_account_counts_map = {} + @brex_institutions_count_map = {} + @brex_items.each do |item| + latest_sync = item.syncs.ordered.first + @brex_sync_stats_map[item.id] = latest_sync&.sync_stats || {} + brex_accounts = item.brex_accounts.to_a + linked_count = brex_accounts.count { |brex_account| brex_account.account_provider.present? } + total_count = brex_accounts.count + @brex_account_counts_map[item.id] = { + linked: linked_count, + unlinked: total_count - linked_count, + total: total_count + } + @brex_institutions_count_map[item.id] = brex_accounts + .filter_map(&:institution_metadata) + .uniq { |institution| institution["name"] || institution["institution_name"] } + .count + end + # Coinbase sync stats @coinbase_sync_stats_map = {} @coinbase_unlinked_count_map = {} diff --git a/app/controllers/brex_items/account_flows_controller.rb b/app/controllers/brex_items/account_flows_controller.rb new file mode 100644 index 000000000..1a240708b --- /dev/null +++ b/app/controllers/brex_items/account_flows_controller.rb @@ -0,0 +1,132 @@ +class BrexItems::AccountFlowsController < ApplicationController + before_action :require_admin! + + def preload_accounts + render json: brex_account_flow.preload_payload + end + + def select_accounts + @accountable_type = params[:accountable_type] || "Depository" + @return_to = safe_return_to_path + result = brex_account_flow.select_accounts_result(accountable_type: @accountable_type) + + return handle_brex_selection_result(result, empty_path: new_account_path, api_return_path: @return_to) unless result.success? + + @brex_item = result.brex_item + @available_accounts = result.available_accounts + + render "brex_items/select_accounts", layout: false + end + + def link_accounts + result = brex_account_flow.link_new_accounts_result( + account_ids: params[:account_ids] || [], + accountable_type: params[:accountable_type] || "Depository" + ) + + redirect_with_navigation(result, return_to: safe_return_to_path) + end + + def select_existing_account + return redirect_to accounts_path, alert: t("brex_items.select_existing_account.no_account_specified") if params[:account_id].blank? + + @account = Current.family.accounts.find_by(id: params[:account_id]) + return redirect_to accounts_path, alert: t("brex_items.select_existing_account.no_account_specified") unless @account + + result = brex_account_flow.select_existing_account_result(account: @account) + + return handle_brex_selection_result(result, empty_path: accounts_path, api_return_path: accounts_path) unless result.success? + + @brex_item = result.brex_item + @available_accounts = result.available_accounts + @return_to = safe_return_to_path + + render "brex_items/select_existing_account", layout: false + end + + def link_existing_account + return redirect_to accounts_path, alert: t("brex_items.link_existing_account.no_account_specified") if params[:account_id].blank? + + account = Current.family.accounts.find_by(id: params[:account_id]) + return redirect_to accounts_path, alert: t("brex_items.link_existing_account.no_account_specified") unless account + + result = brex_account_flow.link_existing_account_result( + account: account, + brex_account_id: params[:brex_account_id] + ) + + redirect_with_navigation(result, return_to: safe_return_to_path) + end + + private + + def brex_account_flow + @brex_account_flow ||= BrexItem::AccountFlow.new(family: Current.family, brex_item_id: params[:brex_item_id]) + end + + def handle_brex_selection_result(result, empty_path:, api_return_path:) + case result.status + when :empty, :account_already_linked + redirect_to empty_path, alert: result.message + when :no_api_token, :select_connection + redirect_to settings_providers_path, alert: result.message + when :setup_required + if turbo_frame_request? + render partial: "brex_items/setup_required", layout: false + else + redirect_to settings_providers_path, alert: result.message + end + when :api_error, :unexpected_error + render_api_error_partial(result.message, api_return_path) + else + redirect_to settings_providers_path, alert: result.message + end + end + + def redirect_with_navigation(result, return_to:) + redirect_to navigation_path_for(result.target, return_to: return_to), result.flash_type => result.message + end + + def navigation_path_for(target, return_to:) + { + new_account: new_account_path, + settings_providers: settings_providers_path, + return_to_or_accounts: return_to || accounts_path + }.fetch(target, accounts_path) + end + + def render_api_error_partial(error_message, return_path) + render partial: "brex_items/api_error", locals: { error_message: error_message, return_path: return_path }, layout: false + end + + def safe_return_to_path + return nil if params[:return_to].blank? + + return_to = params[:return_to].to_s.strip + return nil unless return_to.start_with?("/") + + second_character = return_to[1] + return nil if second_character.blank? + return nil if second_character == "/" || second_character == "\\" + return nil if second_character.match?(/[[:space:][:cntrl:]]/) + return nil if encoded_path_separator?(return_to) + + uri = URI.parse(return_to) + + return nil if uri.scheme.present? || uri.host.present? + + return_to + rescue URI::InvalidURIError + nil + end + + def encoded_path_separator?(return_to) + encoded_second_character = return_to[1, 3] + return false unless encoded_second_character&.start_with?("%") + + decoded = URI.decode_www_form_component(encoded_second_character) + decoded == "/" || decoded == "\\" + rescue ArgumentError + false + end +end diff --git a/app/controllers/brex_items/account_setups_controller.rb b/app/controllers/brex_items/account_setups_controller.rb new file mode 100644 index 000000000..45678aef0 --- /dev/null +++ b/app/controllers/brex_items/account_setups_controller.rb @@ -0,0 +1,109 @@ +class BrexItems::AccountSetupsController < ApplicationController + before_action :require_admin! + before_action :set_brex_item + + def setup_accounts + flow = brex_account_flow + @api_error = flow.import_accounts_with_user_facing_error + @brex_accounts = flow.unlinked_brex_accounts + @account_type_options = flow.account_type_options + @displayable_account_type_options = flow.displayable_account_type_options + @subtype_options = flow.subtype_options + + render "brex_items/setup_accounts" + end + + def complete_account_setup + result = brex_account_flow.complete_setup_result( + account_types: sanitized_account_types, + account_subtypes: sanitized_account_subtypes + ) + + unless result.success? + redirect_to accounts_path, alert: result.message, status: :see_other + return + end + + flash[:notice] = result.message + + if turbo_frame_request? + render_accounts_update_after_setup + else + redirect_to accounts_path, status: :see_other + end + end + + private + + def set_brex_item + @brex_item = Current.family.brex_items.find(params[:id]) + end + + def brex_account_flow + @brex_account_flow ||= BrexItem::AccountFlow.new(family: Current.family, brex_item: @brex_item) + end + + def render_accounts_update_after_setup + @manual_accounts = Account.uncached { Current.family.accounts.visible_manual.order(:name).to_a } + @brex_items = Current.family.brex_items.ordered + + manual_accounts_stream = if @manual_accounts.any? + turbo_stream.update( + "manual-accounts", + partial: "accounts/index/manual_accounts", + locals: { accounts: @manual_accounts } + ) + else + turbo_stream.replace("manual-accounts", view_context.tag.div(id: "manual-accounts")) + end + + render turbo_stream: [ + manual_accounts_stream, + turbo_stream.replace( + ActionView::RecordIdentifier.dom_id(@brex_item), + partial: "brex_items/brex_item", + locals: { brex_item: @brex_item } + ) + ] + Array(flash_notification_stream_items) + end + + def sanitized_account_types + supported_types = Provider::BrexAdapter.supported_account_types + + setup_param_hash(:account_types, allowed_account_ids).each_with_object({}) do |(account_id, selected_type), sanitized| + next unless allowed_account_ids.include?(account_id.to_s) + + normalized_type = selected_type.to_s + sanitized[account_id.to_s] = supported_types.include?(normalized_type) ? normalized_type : "skip" + end + end + + def sanitized_account_subtypes + allowed_subtypes = (Depository::SUBTYPES.keys + CreditCard::SUBTYPES.keys).map(&:to_s) + + setup_param_hash(:account_subtypes, allowed_account_ids).each_with_object({}) do |(account_id, selected_subtype), sanitized| + next unless allowed_account_ids.include?(account_id.to_s) + next if selected_subtype.blank? + next unless allowed_subtypes.include?(selected_subtype.to_s) + + sanitized[account_id.to_s] = selected_subtype.to_s + end + end + + def setup_param_hash(key, allowed_keys) + raw_params = params.fetch(key, {}) + return {} if raw_params.blank? + + if raw_params.is_a?(ActionController::Parameters) + raw_params.permit(*allowed_keys).to_h + elsif raw_params.is_a?(Hash) + raw_params.slice(*allowed_keys) + else + {} + end + end + + def allowed_account_ids + @allowed_account_ids ||= @brex_item.brex_accounts.pluck(:id).map(&:to_s) + end +end diff --git a/app/controllers/brex_items_controller.rb b/app/controllers/brex_items_controller.rb new file mode 100644 index 000000000..551a36c47 --- /dev/null +++ b/app/controllers/brex_items_controller.rb @@ -0,0 +1,98 @@ +class BrexItemsController < ApplicationController + before_action :set_brex_item, only: [ :show, :edit, :update, :destroy, :sync ] + before_action :require_admin!, only: [ :new, :create, :edit, :update, :destroy, :sync ] + + def index + @brex_items = Current.family.brex_items.active.ordered + render layout: "settings" + end + + def show + end + + def new + @brex_item = Current.family.brex_items.build + end + + def create + @brex_item = Current.family.brex_items.build(brex_item_params) + @brex_item.name = t("brex_items.default_connection_name") if @brex_item.name.blank? + + if @brex_item.save + @brex_item.sync_later + render_provider_panel_success(t(".success")) + else + render_provider_panel_error + end + end + + def edit + end + + def update + if BrexItem::AccountFlow.update_item_with_cache_expiration(@brex_item, family: Current.family, attributes: brex_item_params) + render_provider_panel_success(t(".success")) + else + render_provider_panel_error + end + end + + def destroy + @brex_item.unlink_all!(dry_run: false) + @brex_item.destroy_later + redirect_to accounts_path, notice: t(".success") + end + + def sync + @brex_item.sync_later unless @brex_item.syncing? + + respond_to do |format| + format.html { redirect_back_or_to accounts_path } + format.json { head :ok } + end + end + + private + + def render_provider_panel_success(message) + return redirect_to accounts_path, notice: message, status: :see_other unless turbo_frame_request? + + flash.now[:notice] = message + @brex_items = Current.family.brex_items.active.ordered.includes(:syncs, :brex_accounts) + render_brex_provider_panel(locals: { brex_items: @brex_items }, include_flash: true) + end + + def render_provider_panel_error + @error_message = @brex_item.errors.full_messages.join(", ") + return redirect_to settings_providers_path, alert: @error_message, status: :see_other unless turbo_frame_request? + + render_brex_provider_panel(locals: { error_message: @error_message }, status: :unprocessable_entity) + end + + def render_brex_provider_panel(locals:, status: :ok, include_flash: false) + streams = [ + turbo_stream.replace( + "brex-providers-panel", + partial: "settings/providers/brex_panel", + locals: locals + ) + ] + streams += flash_notification_stream_items if include_flash + render turbo_stream: streams, status: status + end + + def set_brex_item + @brex_item = Current.family.brex_items.find(params[:id]) + end + + def brex_item_params + permitted = params.require(:brex_item).permit(:name, :sync_start_date, :token, :base_url) + permitted.delete(:token) if @brex_item&.persisted? && permitted[:token].blank? + permitted[:token] = permitted[:token].to_s.strip if permitted[:token].present? + if permitted.key?(:base_url) + permitted[:base_url] = permitted[:base_url].to_s.strip + permitted[:base_url] = nil if permitted[:base_url].blank? + end + permitted + end +end diff --git a/app/controllers/settings/providers_controller.rb b/app/controllers/settings/providers_controller.rb index b8852065f..bdd9b485e 100644 --- a/app/controllers/settings/providers_controller.rb +++ b/app/controllers/settings/providers_controller.rb @@ -187,6 +187,7 @@ class Settings::ProvidersController < ApplicationController { key: "enable_banking", title: "Enable Banking", turbo_id: "enable_banking", partial: "enable_banking_panel" }, { key: "coinstats", title: "CoinStats", turbo_id: "coinstats", partial: "coinstats_panel" }, { key: "mercury", title: "Mercury", turbo_id: "mercury", partial: "mercury_panel" }, + { key: "brex", title: "Brex", turbo_id: "brex", partial: "brex_panel" }, { key: "coinbase", title: "Coinbase", turbo_id: "coinbase", partial: "coinbase_panel" }, { key: "binance", title: "Binance", turbo_id: "binance", partial: "binance_panel" }, { key: "kraken", title: "Kraken", turbo_id: "kraken", partial: "kraken_panel" }, @@ -205,6 +206,7 @@ class Settings::ProvidersController < ApplicationController "enable_banking" => "EnableBankingItem", "coinstats" => "CoinstatsItem", "mercury" => "MercuryItem", + "brex" => "BrexItem", "coinbase" => "CoinbaseItem", "binance" => "BinanceItem", "kraken" => "KrakenItem", @@ -226,6 +228,8 @@ class Settings::ProvidersController < ApplicationController @coinstats_items = Current.family.coinstats_items.ordered when "mercury" @mercury_items = Current.family.mercury_items.active.ordered.includes(:syncs, :mercury_accounts) + when "brex" + @brex_items = Current.family.brex_items.active.ordered.includes(:syncs, :brex_accounts) when "coinbase" @coinbase_items = Current.family.coinbase_items.ordered when "binance" @@ -259,6 +263,7 @@ class Settings::ProvidersController < ApplicationController @sophtron_items = Current.family.sophtron_items.where.not(user_id: [ nil, "" ], access_key: [ nil, "" ]).ordered.select(:id) @coinstats_items = Current.family.coinstats_items.ordered # CoinStats panel needs account info for status display @mercury_items = Current.family.mercury_items.active.ordered + @brex_items = Current.family.brex_items.active.ordered @coinbase_items = Current.family.coinbase_items.ordered # Coinbase panel needs name and sync info for status display @snaptrade_items = Current.family.snaptrade_items.ordered @ibkr_items = Current.family.ibkr_items.ordered.select(:id) @@ -287,6 +292,7 @@ class Settings::ProvidersController < ApplicationController "enable_banking" => @enable_banking_items, "coinstats" => @coinstats_items, "mercury" => @mercury_items, + "brex" => @brex_items, "coinbase" => @coinbase_items, "binance" => @binance_items, "kraken" => @kraken_items, diff --git a/app/helpers/brex_items_helper.rb b/app/helpers/brex_items_helper.rb new file mode 100644 index 000000000..be30ecb40 --- /dev/null +++ b/app/helpers/brex_items_helper.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module BrexItemsHelper + BrexAccountDisplay = Struct.new( + :id, + :name, + :kind, + :currency, + :status, + :blank_name, + keyword_init: true + ) do + alias_method :blank_name?, :blank_name + end + + def brex_account_display(account) + data = account.with_indifferent_access + kind = BrexAccount.kind_for(data) + name = BrexAccount.name_for(data) + + BrexAccountDisplay.new( + id: data[:id], + name: name, + kind: kind, + currency: BrexAccount.currency_code_from_money(data[:current_balance] || data[:available_balance] || data[:account_limit]), + status: data[:status], + blank_name: name.blank? + ) + end + + def brex_account_metadata(display) + parts = [ + t("brex_items.account_metadata.provider"), + display.currency, + translated_brex_metadata_value("kinds", display.kind), + translated_brex_metadata_value("statuses", display.status) + ].compact + + parts.join(t("brex_items.account_metadata.separator")) + end + + def brex_item_render_locals(brex_item, sync_stats_map: nil, account_counts_map: nil, institutions_count_map: nil) + counts = (account_counts_map || {})[brex_item.id] || {} + + { + brex_item: brex_item, + stats: (sync_stats_map || {})[brex_item.id] || brex_item.syncs.ordered.first&.sync_stats || {}, + unlinked_count: counts[:unlinked] || brex_item.unlinked_accounts_count, + linked_count: counts[:linked] || brex_item.linked_accounts_count, + total_count: counts[:total] || brex_item.total_accounts_count, + institutions_count: (institutions_count_map || {})[brex_item.id] || brex_item.connected_institutions.size + } + end + + def default_brex_depository_subtype(account_name) + normalized_name = account_name.to_s.downcase + + if normalized_name.match?(/\bchecking\b|\bchequing\b|\bck\b|demand\s+deposit/) + "checking" + elsif normalized_name.match?(/\bsavings\b|\bsv\b/) + "savings" + elsif normalized_name.match?(/money\s+market|\bmm\b/) + "money_market" + else + "checking" + end + end + + private + def translated_brex_metadata_value(scope, value) + key = value.to_s + return nil if key.blank? + + t("brex_items.#{scope}.#{key}", default: key.titleize) + end +end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index b4e927292..1cc3f32e2 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -86,6 +86,9 @@ module SettingsHelper when "mercury" return { status: :off } unless @mercury_items&.any? sync_based_summary(key) + when "brex" + return { status: :off } unless @brex_items&.any? + sync_based_summary(key) when "coinbase" return { status: :off } unless @coinbase_items&.any? sync_based_summary(key) diff --git a/app/javascript/controllers/account_type_selector_controller.js b/app/javascript/controllers/account_type_selector_controller.js index 4504c41b7..ef9ced189 100644 --- a/app/javascript/controllers/account_type_selector_controller.js +++ b/app/javascript/controllers/account_type_selector_controller.js @@ -18,7 +18,8 @@ export default class extends Controller { // Hide all subtype selects const subtypeSelects = container.querySelectorAll('.subtype-select') subtypeSelects.forEach(select => { - select.style.display = 'none' + select.classList.add('hidden') + select.style.removeProperty('display') // Clear the name attribute so it doesn't get submitted const selectElement = select.querySelector('select') if (selectElement) { @@ -34,7 +35,8 @@ export default class extends Controller { // Show the relevant subtype select const relevantSubtype = container.querySelector(`[data-type="${selectedType}"]`) if (relevantSubtype) { - relevantSubtype.style.display = 'block' + relevantSubtype.classList.remove('hidden') + relevantSubtype.style.removeProperty('display') // Re-add the name attribute so it gets submitted const selectElement = relevantSubtype.querySelector('select') if (selectElement) { @@ -65,4 +67,4 @@ export default class extends Controller { } } } -} \ No newline at end of file +} diff --git a/app/models/brex_account.rb b/app/models/brex_account.rb new file mode 100644 index 000000000..743fcc01a --- /dev/null +++ b/app/models/brex_account.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +class BrexAccount < ApplicationRecord + include CurrencyNormalizable, Encryptable + + CARD_PRIMARY_ACCOUNT_ID = "card_primary" + + if encryption_ready? + encrypts :raw_payload + encrypts :raw_transactions_payload + end + + belongs_to :brex_item + + has_one :account_provider, as: :provider, dependent: :destroy + has_one :account, through: :account_provider, source: :account + + validates :name, :currency, presence: true + validates :account_id, uniqueness: { scope: :brex_item_id } + validates :account_kind, inclusion: { in: %w[cash card] } + + def self.card_account_id + CARD_PRIMARY_ACCOUNT_ID + end + + def self.kind_for(account_data) + return account_data.account_kind if account_data.respond_to?(:account_kind) + + data = account_data.with_indifferent_access + kind = data[:account_kind].presence || data[:kind].presence || "cash" + kind.to_s == "credit_card" ? "card" : kind.to_s + end + + def self.name_for(account_data) + data = account_data.with_indifferent_access + kind = kind_for(data) + + if kind == "card" + data[:name].presence || I18n.t("brex_items.default_card_name", default: "Brex Card") + else + data[:name].presence || data[:display_name].presence || I18n.t("brex_items.default_cash_name", id: data[:id], default: "Brex Cash #{data[:id]}") + end + end + + def self.currency_for(account_data) + data = account_data.with_indifferent_access + currency_code_from_money(data[:current_balance] || data[:available_balance] || data[:account_limit]) + end + + def self.default_account_type_for(account_data) + kind_for(account_data) == "card" ? "CreditCard" : "Depository" + end + + def self.default_accountable_attributes(accountable_type) + case accountable_type + when "CreditCard" + { subtype: CreditCard::DEFAULT_SUBTYPE } + when "Depository" + { subtype: Depository::DEFAULT_SUBTYPE } + else + {} + end + end + + def self.money_to_decimal(money_payload) + return nil if money_payload.blank? + + payload = money_payload.is_a?(Hash) ? money_payload.with_indifferent_access : { amount: money_payload, currency: "USD" } + amount = payload[:amount] + return nil if amount.nil? + + currency = currency_code_from_money(payload) + divisor = Money::Currency.new(currency).minor_unit_conversion + BigDecimal(amount.to_s) / BigDecimal(divisor.to_s) + rescue Money::Currency::UnknownCurrencyError, ArgumentError + Rails.logger.warn("Invalid Brex money payload #{money_payload.inspect}, defaulting conversion to USD") + begin + safe_amount = BigDecimal(payload[:amount].to_s) + safe_amount / BigDecimal(Money::Currency.new("USD").minor_unit_conversion.to_s) + rescue ArgumentError, TypeError + BigDecimal("0") + end + end + + def self.currency_code_from_money(money_payload) + payload = money_payload.is_a?(Hash) ? money_payload.with_indifferent_access : {} + currency = payload[:currency].presence || "USD" + Money::Currency.new(currency).iso_code + rescue Money::Currency::UnknownCurrencyError + "USD" + end + + def self.sanitize_payload(payload) + case payload + when Array + payload.map { |value| sanitize_payload(value) } + when Hash + payload.each_with_object({}) do |(key, value), sanitized| + key_string = key.to_s + normalized_key = key_string.downcase + + if sensitive_number_key?(normalized_key) + sanitized["#{key_string}_last4"] = last_four(value) + elsif normalized_key == "card_metadata" + sanitized[key_string] = sanitize_card_metadata(value) + elsif sensitive_secret_key?(normalized_key) + sanitized[key_string] = "[FILTERED]" + else + sanitized[key_string] = sanitize_payload(value) + end + end + else + payload + end + end + + def self.last_four(value) + digits = value.to_s.gsub(/\D/, "") + digits.last(4) if digits.present? + end + + def self.sanitize_card_metadata(value) + return nil unless value.is_a?(Hash) + + metadata = value.with_indifferent_access + { + "card_id" => metadata[:card_id].presence || metadata[:id].presence, + "card_name" => metadata[:card_name].presence || metadata[:name].presence, + "card_type" => metadata[:card_type].presence || metadata[:type].presence, + "last_four" => last_four(metadata[:last_four].presence || metadata[:last4].presence || metadata[:card_last_four].presence) + }.compact + end + + def current_account + account + end + + def linked_account + account + end + + def cash? + account_kind == "cash" + end + + def card? + account_kind == "card" + end + + def upsert_brex_snapshot!(account_snapshot) + snapshot = account_snapshot.with_indifferent_access + kind = snapshot[:account_kind].presence || snapshot[:kind].presence || "cash" + kind = "card" if kind.to_s == "credit_card" + + update!( + current_balance: self.class.money_to_decimal(snapshot[:current_balance]), + available_balance: self.class.money_to_decimal(snapshot[:available_balance]), + account_limit: self.class.money_to_decimal(snapshot[:account_limit]), + currency: self.class.currency_code_from_money(snapshot[:current_balance] || snapshot[:available_balance] || snapshot[:account_limit]), + name: self.class.name_for(snapshot.merge(account_kind: kind)), + account_id: snapshot[:id]&.to_s, + account_kind: kind, + account_status: snapshot[:status], + account_type: snapshot[:type], + provider: "brex", + institution_metadata: build_institution_metadata(snapshot, kind), + raw_payload: self.class.sanitize_payload(account_snapshot) + ) + end + + def upsert_brex_transactions_snapshot!(transactions_snapshot) + update!( + raw_transactions_payload: self.class.sanitize_payload(transactions_snapshot) + ) + end + + private + + def self.sensitive_number_key?(normalized_key) + normalized_key.in?(%w[account_number routing_number pan primary_account_number card_number]) + end + + def self.sensitive_secret_key?(normalized_key) + normalized_key.include?("token") || + normalized_key.include?("secret") || + normalized_key.in?(%w[api_key access_key authorization cvc cvv security_code]) + end + private_class_method :sensitive_number_key?, :sensitive_secret_key? + + def build_institution_metadata(snapshot, kind) + { + name: "Brex", + domain: "brex.com", + url: "https://brex.com", + account_kind: kind, + account_type: snapshot[:type], + primary: snapshot[:primary], + account_number_last4: self.class.last_four(snapshot[:account_number]), + routing_number_last4: self.class.last_four(snapshot[:routing_number]), + status: snapshot[:status], + current_statement_period: self.class.sanitize_payload(snapshot[:current_statement_period]) + }.compact + end +end diff --git a/app/models/brex_account/processor.rb b/app/models/brex_account/processor.rb new file mode 100644 index 000000000..67c8a4a7b --- /dev/null +++ b/app/models/brex_account/processor.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +class BrexAccount::Processor + include CurrencyNormalizable + + attr_reader :brex_account + + def initialize(brex_account) + @brex_account = brex_account + end + + def process + unless brex_account.current_account.present? + Rails.logger.info "BrexAccount::Processor - No linked account for brex_account #{brex_account.id}, skipping processing" + return + end + + process_account! + process_transactions + rescue StandardError => e + Rails.logger.error "BrexAccount::Processor - Failed to process account #{brex_account.id}: #{e.message}" + report_exception(e, "account") + raise + end + + private + + def process_account! + account = brex_account.current_account + balance = brex_account.current_balance + currency = parse_currency(brex_account.currency) + + if balance.nil? + Rails.logger.warn "BrexAccount::Processor - current_balance is nil for brex_account #{brex_account.id}, defaulting to 0" + balance = 0 + end + + if currency.nil? + Rails.logger.warn "BrexAccount::Processor - currency parse failed for brex_account #{brex_account.id}: #{brex_account.currency.inspect}, defaulting to USD" + Sentry.capture_message("BrexAccount currency parse failed", level: :warning) do |scope| + scope.set_tags(brex_account_id: brex_account.id) + scope.set_context("brex_account", { + id: brex_account.id, + currency: brex_account.currency + }) + end + currency = "USD" + end + + account.update!( + balance: balance, + cash_balance: balance, + currency: currency + ) + + if account.accountable_type == "CreditCard" && brex_account.available_balance.present? + account.accountable.update!(available_credit: brex_account.available_balance) + end + end + + # Transaction import errors are logged and swallowed so balance sync can continue. + def process_transactions + BrexAccount::Transactions::Processor.new(brex_account).process + rescue StandardError => e + Rails.logger.error "BrexAccount::Processor - Failed to process transactions for brex_account #{brex_account.id}: #{e.message}" + Rails.logger.error Array(e.backtrace).first(10).join("\n") + report_exception(e, "transactions") + end + + def report_exception(error, context) + Sentry.capture_exception(error) do |scope| + scope.set_tags( + brex_account_id: brex_account.id, + context: context + ) + end + end +end diff --git a/app/models/brex_account/transactions/processor.rb b/app/models/brex_account/transactions/processor.rb new file mode 100644 index 000000000..da0a81e17 --- /dev/null +++ b/app/models/brex_account/transactions/processor.rb @@ -0,0 +1,83 @@ +class BrexAccount::Transactions::Processor + attr_reader :brex_account + + def initialize(brex_account) + @brex_account = brex_account + end + + def process + unless brex_account.raw_transactions_payload.present? + Rails.logger.info "BrexAccount::Transactions::Processor - No transactions in raw_transactions_payload for brex_account #{brex_account.id}" + return { success: true, total: 0, imported: 0, skipped: 0, failed: 0, errors: [], skipped_transactions: [] } + end + + total_count = brex_account.raw_transactions_payload.count + Rails.logger.info "BrexAccount::Transactions::Processor - Processing #{total_count} transactions for brex_account #{brex_account.id}" + + imported_count = 0 + failed_count = 0 + skipped_count = 0 + errors = [] + skipped = [] + + # Each entry is processed inside a transaction, but to avoid locking up the DB when + # there are hundreds or thousands of transactions, we process them individually. + brex_account.raw_transactions_payload.each_with_index do |transaction_data, index| + begin + result = BrexEntry::Processor.new( + transaction_data, + brex_account: brex_account + ).process + + if result == :skipped + skipped_count += 1 + skipped << { index: index, transaction_id: transaction_id_for(transaction_data), reason: "No linked account" } + elsif result.nil? + failed_count += 1 + errors << { index: index, transaction_id: transaction_id_for(transaction_data), error: "No transaction imported" } + else + imported_count += 1 + end + rescue ArgumentError => e + # Validation error - log and continue + failed_count += 1 + transaction_id = transaction_id_for(transaction_data) + error_message = "Validation error: #{e.message}" + Rails.logger.error "BrexAccount::Transactions::Processor - #{error_message} (transaction #{transaction_id})" + errors << { index: index, transaction_id: transaction_id, error: error_message } + rescue => e + # Unexpected error - log with full context and continue + failed_count += 1 + transaction_id = transaction_id_for(transaction_data) + error_message = "#{e.class}: #{e.message}" + Rails.logger.error "BrexAccount::Transactions::Processor - Error processing transaction #{transaction_id}: #{error_message}" + Rails.logger.error Array(e.backtrace).first(10).join("\n") + errors << { index: index, transaction_id: transaction_id, error: error_message } + end + end + + result = { + success: failed_count == 0, + total: total_count, + imported: imported_count, + skipped: skipped_count, + failed: failed_count, + errors: errors, + skipped_transactions: skipped + } + + if failed_count > 0 + Rails.logger.warn "BrexAccount::Transactions::Processor - Completed with #{failed_count} failures out of #{total_count} transactions" + else + Rails.logger.info "BrexAccount::Transactions::Processor - Successfully processed #{imported_count} transactions" + end + + result + end + + private + + def transaction_id_for(transaction_data) + transaction_data&.dig(:id) || transaction_data&.dig("id") || "unknown" + end +end diff --git a/app/models/brex_entry/processor.rb b/app/models/brex_entry/processor.rb new file mode 100644 index 000000000..03bbb8689 --- /dev/null +++ b/app/models/brex_entry/processor.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require "digest/md5" + +class BrexEntry::Processor + include CurrencyNormalizable + + def initialize(brex_transaction, brex_account:) + @brex_transaction = brex_transaction + @brex_account = brex_account + end + + def process + cached_external_id = nil + cached_external_id = external_id + + unless account.present? + Rails.logger.warn "BrexEntry::Processor - No linked account for brex_account #{brex_account.id}, skipping transaction #{cached_external_id}" + return :skipped + end + + import_adapter.import_transaction( + external_id: cached_external_id, + amount: amount, + currency: currency, + date: date, + name: name, + source: "brex", + merchant: merchant, + notes: notes, + extra: extra + ) + rescue ArgumentError => e + Rails.logger.error "BrexEntry::Processor - Validation error for transaction #{cached_external_id || safe_external_id}: #{e.message}" + raise + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e + Rails.logger.error "BrexEntry::Processor - Failed to save transaction #{cached_external_id || safe_external_id}: #{e.message}" + raise StandardError.new("Failed to import transaction: #{e.message}") + rescue => e + Rails.logger.error "BrexEntry::Processor - Unexpected error processing transaction #{cached_external_id || safe_external_id}: #{e.class} - #{e.message}" + Rails.logger.error Array(e.backtrace).join("\n") + raise StandardError.new("Unexpected error importing transaction: #{e.message}") + end + + private + attr_reader :brex_transaction, :brex_account + + def import_adapter + @import_adapter ||= Account::ProviderImportAdapter.new(account) + end + + def account + @account ||= brex_account.current_account + end + + def data + @data ||= brex_transaction.with_indifferent_access + end + + def external_id + id = data[:id].presence + raise ArgumentError, "Brex transaction missing required field 'id'" unless id + + "brex_#{id}" + end + + def safe_external_id + external_id + rescue ArgumentError + "brex_unknown" + end + + def name + data[:description].presence || + merchant_payload[:raw_descriptor].presence || + merchant_payload[:name].presence || + I18n.t("brex_items.entries.default_name") + end + + def notes + note_parts = [] + note_parts << data[:type] if data[:type].present? + note_parts << data[:expense_id] if data[:expense_id].present? + note_parts.any? ? note_parts.join(" - ") : nil + end + + def merchant + merchant_name = merchant_payload[:raw_descriptor].presence || merchant_payload[:name].presence + return @merchant if instance_variable_defined?(:@merchant) + return @merchant = nil if merchant_name.blank? + + merchant_name = merchant_name.to_s.strip + return @merchant = nil if merchant_name.blank? + + merchant_id = Digest::MD5.hexdigest(merchant_name.downcase) + + @merchant = import_adapter.find_or_create_merchant( + provider_merchant_id: "brex_merchant_#{merchant_id}", + name: merchant_name, + source: "brex" + ) + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error "BrexEntry::Processor - Failed to create merchant '#{merchant_name}': #{e.message}" + @merchant = nil + end + + def merchant_payload + @merchant_payload ||= begin + payload = data[:merchant] + payload.is_a?(Hash) ? payload.with_indifferent_access : {} + end + end + + def amount + BrexAccount.money_to_decimal(data[:amount]) || BigDecimal("0") + rescue ArgumentError => e + Rails.logger.error "Failed to parse Brex transaction amount: #{data[:amount].inspect} - #{e.message}" + raise + end + + def currency + amount_currency = transaction_amount_currency + log_invalid_currency(amount_currency) if amount_currency.blank? && data[:amount].present? + + parse_currency(amount_currency) || + parse_currency(brex_account.currency) || + "USD" + end + + def transaction_amount_currency + amount_payload = data[:amount] + return nil unless amount_payload.is_a?(Hash) + + amount_payload.with_indifferent_access[:currency] + end + + def log_invalid_currency(currency_value) + Rails.logger.warn( + "Invalid Brex currency #{currency_value.inspect} for transaction #{data[:id].presence || 'unknown'} " \ + "on brex_account #{brex_account.id} amount=#{data[:amount].inspect} account_currency=#{brex_account.currency.inspect}; defaulting to fallback" + ) + end + + def date + date_value = data[:posted_at_date].presence || data[:initiated_at_date].presence + + case date_value + when String + Date.parse(date_value) + when Integer, Float + Time.at(date_value).to_date + when Time, DateTime + date_value.to_date + when Date + date_value + else + raise ArgumentError, "Invalid date format: #{date_value.inspect}" + end + rescue ArgumentError, TypeError => e + Rails.logger.error("Failed to parse Brex transaction date '#{date_value}': #{e.message}") + raise ArgumentError, "Unable to parse transaction date: #{date_value.inspect}" + end + + def extra + { + brex: { + transaction_id: data[:id], + account_kind: brex_account.account_kind, + type: data[:type], + card_id: data[:card_id], + transfer_id: data[:transfer_id], + expense_id: data[:expense_id], + card_transaction_operation_reference_id: data[:card_transaction_operation_reference_id], + initiated_at_date: data[:initiated_at_date], + posted_at_date: data[:posted_at_date], + merchant: BrexAccount.sanitize_payload(data[:merchant]) + }.compact + } + end +end diff --git a/app/models/brex_item.rb b/app/models/brex_item.rb new file mode 100644 index 000000000..865797e65 --- /dev/null +++ b/app/models/brex_item.rb @@ -0,0 +1,197 @@ +class BrexItem < ApplicationRecord + include Syncable, Provided, Unlinking, Encryptable + + BLANK_TOKEN_SENTINELS = [ "", " ", " ", " ", "\t", "\n", "\r" ].freeze + + enum :status, { good: "good", requires_update: "requires_update" }, default: :good + + if encryption_ready? + encrypts :token, deterministic: true + encrypts :raw_payload + end + + validates :name, presence: true + validates :token, presence: true, on: :create + validate :base_url_must_be_official_brex_url + validate :token_cannot_be_blank_when_changed + before_validation :normalize_token + before_validation :normalize_base_url + + belongs_to :family + has_one_attached :logo, dependent: :purge_later + + has_many :brex_accounts, dependent: :destroy + has_many :accounts, through: :brex_accounts + + scope :active, -> { where(scheduled_for_deletion: false) } + scope :syncable, -> { active } + scope :ordered, -> { order(created_at: :desc) } + scope :needs_update, -> { where(status: :requires_update) } + scope :with_credentials, -> { where.not(token: [ nil, *BLANK_TOKEN_SENTINELS ]).where("BTRIM(token) <> ''") } + + def self.resolve_for(family:, brex_item_id: nil) + normalized_id = brex_item_id.to_s.strip.presence + + if normalized_id.present? + return family.brex_items.active.with_credentials.find_by(id: normalized_id) + end + + credentialed_items = family.brex_items.active.with_credentials.ordered + credentialed_items.first if credentialed_items.one? + end + + def destroy_later + update!(scheduled_for_deletion: true) + DestroyJob.perform_later(self) + end + + def import_latest_brex_data(sync_start_date: nil) + provider = brex_provider + unless provider + Rails.logger.error "BrexItem #{id} - Cannot import: provider is not configured" + raise Provider::Brex::BrexError.new("Brex provider is not configured", :not_configured) + end + + BrexItem::Importer.new(self, brex_provider: provider, sync_start_date: sync_start_date).import + rescue => e + Rails.logger.error "BrexItem #{id} - Failed to import data: #{e.message}" + raise + end + + def process_accounts + return [] if brex_accounts.empty? + + results = [] + brex_accounts.joins(:account).includes(:account).merge(Account.visible).each do |brex_account| + begin + result = BrexAccount::Processor.new(brex_account).process + results << { brex_account_id: brex_account.id, success: true, result: result } + rescue => e + Rails.logger.error "BrexItem #{id} - Failed to process account #{brex_account.id}: #{e.message}" + results << { brex_account_id: brex_account.id, success: false, error: e.message } + end + end + + results + end + + def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) + return [] if accounts.empty? + + results = [] + accounts.visible.each do |account| + begin + account.sync_later( + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: window_end_date + ) + results << { account_id: account.id, success: true } + rescue => e + Rails.logger.error "BrexItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" + results << { account_id: account.id, success: false, error: e.message } + end + end + + results + end + + def upsert_brex_snapshot!(accounts_snapshot) + update!(raw_payload: BrexAccount.sanitize_payload(accounts_snapshot)) + end + + def has_completed_initial_setup? + # Setup is complete if we have any linked accounts + accounts.any? + end + + def sync_status_summary + total_accounts = total_accounts_count + linked_count = linked_accounts_count + unlinked_count = unlinked_accounts_count + + if total_accounts == 0 + I18n.t("brex_items.sync_status.no_accounts") + elsif unlinked_count == 0 + I18n.t("brex_items.sync_status.all_synced", count: linked_count) + else + I18n.t("brex_items.sync_status.partial_setup", synced: linked_count, pending: unlinked_count) + end + end + + def linked_accounts_count + brex_accounts.joins(:account_provider).count + end + + def unlinked_accounts_count + brex_accounts.left_joins(:account_provider).where(account_providers: { id: nil }).count + end + + def total_accounts_count + brex_accounts.count + end + + def institution_display_name + institution_name.presence || institution_domain.presence || name + end + + def connected_institutions + brex_accounts.where.not(institution_metadata: nil) + .pluck(:institution_metadata) + .compact + .uniq { |inst| inst["name"] || inst["institution_name"] } + end + + def institution_summary + institutions = connected_institutions + case institutions.count + when 0 + I18n.t("brex_items.institution_summary.none") + when 1 + name = institutions.first["name"] || + institutions.first["institution_name"] || + I18n.t("brex_items.institution_summary.count", count: 1) + I18n.t("brex_items.institution_summary.one", name: name) + else + I18n.t("brex_items.institution_summary.count", count: institutions.count) + end + end + + def credentials_configured? + token.to_s.strip.present? + end + + def effective_base_url + return Provider::Brex::DEFAULT_BASE_URL if base_url.blank? + + Provider::Brex.normalize_base_url(base_url) + end + + private + def normalize_token + self.token = token&.strip + end + + def token_cannot_be_blank_when_changed + return unless persisted? && will_save_change_to_token? && token.blank? + + errors.add(:token, :blank) + end + + def normalize_base_url + stripped = base_url.to_s.strip + if stripped.blank? + self.base_url = nil + return + end + + normalized = Provider::Brex.normalize_base_url(stripped) + self.base_url = normalized if normalized.present? + end + + def base_url_must_be_official_brex_url + return if base_url.blank? || Provider::Brex.allowed_base_url?(base_url) + + errors.add(:base_url, :official_hosts_only) + end +end diff --git a/app/models/brex_item/account_flow.rb b/app/models/brex_item/account_flow.rb new file mode 100644 index 000000000..7fc08ed0d --- /dev/null +++ b/app/models/brex_item/account_flow.rb @@ -0,0 +1,425 @@ +# frozen_string_literal: true + +class BrexItem::AccountFlow + require_dependency "brex_item/account_flow/setup" + + include Setup + + CACHE_TTL = 5.minutes + + class NoApiTokenError < StandardError; end + class AccountNotFoundError < StandardError; end + class InvalidAccountNameError < StandardError; end + class AccountAlreadyLinkedError < StandardError; end + + NavigationResult = Data.define(:target, :flash_type, :message) + + SelectionResult = Data.define(:status, :brex_item, :available_accounts, :accountable_type, :message) do + def success? = status == :success + def setup_required? = status == :setup_required + def provider_error? = status.in?([ :api_error, :unexpected_error ]) + end + + LinkAccountsResult = Data.define(:created_accounts, :already_linked_names, :invalid_account_ids) do + def created_count = created_accounts.count + def already_linked_count = already_linked_names.count + def invalid_count = invalid_account_ids.count + end + + SetupResult = Data.define(:created_accounts, :skipped_count, :failed_count) do + def created_count = created_accounts.count + end + + SetupCompletion = Data.define(:success, :message) do + def success? = success + end + + attr_reader :family, :brex_item_id, :brex_item, :credentialed_items + + def initialize(family:, brex_item_id: nil, brex_item: nil) + @family = family + @brex_item_id = brex_item_id.to_s.strip.presence + @credentialed_items = family.brex_items.active.with_credentials.ordered + @brex_item = brex_item || BrexItem.resolve_for(family: family, brex_item_id: @brex_item_id) + end + + def self.cache_key(family, brex_item) + "brex_accounts_#{family.id}_#{brex_item.id}" + end + + def self.cache_sensitive_update?(permitted_params) + permitted_params.key?(:token) || permitted_params.key?(:base_url) + end + + def self.update_item_with_cache_expiration(brex_item, family:, attributes:) + expire_accounts_cache = cache_sensitive_update?(attributes) + updated = brex_item.update(attributes) + + Rails.cache.delete(cache_key(family, brex_item)) if updated && expire_accounts_cache + + updated + end + + def selected? + brex_item.present? + end + + def selection_required? + credentialed_items.count > 1 && brex_item_id.blank? + end + + def preload_payload + return selection_error_payload if !selected? + return { success: false, error: "no_credentials", has_accounts: false } unless brex_item.credentials_configured? + + cached_accounts = Rails.cache.read(cache_key) + cached = !cached_accounts.nil? + available_accounts = cached ? cached_accounts : fetch_and_cache_accounts + + { success: true, has_accounts: available_accounts.any?, cached: cached } + rescue NoApiTokenError + { success: false, error: "no_api_token", has_accounts: false } + rescue Provider::Brex::BrexError => e + Rails.logger.error("Brex preload error: #{e.message}") + { success: false, error: "api_error", error_message: e.message, has_accounts: nil } + rescue StandardError => e + Rails.logger.error("Unexpected error preloading Brex accounts: #{e.class}: #{e.message}") + { success: false, error: "unexpected_error", error_message: I18n.t("brex_items.errors.unexpected_error"), has_accounts: nil } + end + + def select_accounts_result(accountable_type:) + selection_result_for( + scope: "brex_items.select_accounts", + accountable_type: accountable_type, + empty_message_key: "no_accounts_found", + log_context: "select_accounts" + ) + end + + def select_existing_account_result(account:) + return linked_account_result if account.account_providers.exists? + + selection_result_for( + scope: "brex_items.select_existing_account", + accountable_type: account.accountable_type, + empty_message_key: "all_accounts_already_linked", + log_context: "select_existing_account" + ) + end + + def link_new_accounts_result(account_ids:, accountable_type:) + return navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.no_accounts_selected")) if account_ids.blank? + return navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.invalid_account_type")) unless supported_account_type?(accountable_type) + return navigation(:settings_providers, :alert, I18n.t("brex_items.link_accounts.select_connection")) unless selected? + + link_navigation_result(link_new_accounts!(account_ids: account_ids, accountable_type: accountable_type)) + rescue NoApiTokenError + navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.no_api_token")) + rescue Provider::Brex::BrexError => e + navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.api_error", message: e.message)) + rescue StandardError => e + Rails.logger.error("Brex account linking failed: #{e.class} - #{e.message}") + Rails.logger.error(Array(e.backtrace).first(10).join("\n")) + navigation(:new_account, :alert, I18n.t("brex_items.errors.unexpected_error")) + end + + def link_existing_account_result(account:, brex_account_id:) + return navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.missing_parameters")) if account.blank? || brex_account_id.blank? + return navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.account_already_linked")) if account.account_providers.exists? + return navigation(:settings_providers, :alert, I18n.t("brex_items.link_existing_account.select_connection")) unless selected? + + link_existing_account!(account: account, brex_account_id: brex_account_id) + + navigation(:return_to_or_accounts, :notice, I18n.t("brex_items.link_existing_account.success", account_name: account.name)) + rescue NoApiTokenError + navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.no_api_token")) + rescue AccountNotFoundError + navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.provider_account_not_found")) + rescue InvalidAccountNameError + navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.invalid_account_name")) + rescue AccountAlreadyLinkedError + navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.provider_account_already_linked")) + rescue Provider::Brex::BrexError => e + navigation(:accounts, :alert, I18n.t("brex_items.link_existing_account.api_error", message: e.message)) + rescue StandardError => e + Rails.logger.error("Brex existing account linking failed: #{e.class} - #{e.message}") + Rails.logger.error(Array(e.backtrace).first(10).join("\n")) + navigation(:accounts, :alert, I18n.t("brex_items.errors.unexpected_error")) + end + + def link_new_accounts!(account_ids:, accountable_type:) + raise ArgumentError, "Unsupported Brex account type: #{accountable_type}" unless supported_account_type?(accountable_type) + + created_accounts = [] + already_linked_names = [] + invalid_account_ids = [] + accounts_by_id = indexed_accounts + + ActiveRecord::Base.transaction do + account_ids.each do |account_id| + account_data = accounts_by_id[account_id.to_s] + next unless account_data + + account_name = BrexAccount.name_for(account_data) + + if account_name.blank? + invalid_account_ids << account_id + Rails.logger.warn "BrexItem::AccountFlow - Skipping account #{account_id} with blank name" + next + end + + brex_account = upsert_brex_account!(account_id, account_data) + + if brex_account.account_provider.present? + already_linked_names << account_name + next + end + + account = Account.create_and_sync( + { + family: family, + name: account_name, + balance: 0, + currency: BrexAccount.currency_for(account_data), + accountable_type: accountable_type, + accountable_attributes: BrexAccount.default_accountable_attributes(accountable_type) + }, + skip_initial_sync: true + ) + + AccountProvider.create!(account: account, provider: brex_account) + created_accounts << account + end + end + + brex_item.sync_later if created_accounts.any? + + LinkAccountsResult.new( + created_accounts: created_accounts, + already_linked_names: already_linked_names, + invalid_account_ids: invalid_account_ids + ) + end + + def link_existing_account!(account:, brex_account_id:) + account_data = indexed_accounts[brex_account_id.to_s] + raise AccountNotFoundError unless account_data + + account_name = BrexAccount.name_for(account_data) + raise InvalidAccountNameError if account_name.blank? + + brex_account = nil + + ActiveRecord::Base.transaction do + brex_account = upsert_brex_account!(brex_account_id, account_data) + raise AccountAlreadyLinkedError if brex_account.account_provider.present? + + AccountProvider.create!(account: account, provider: brex_account) + end + + brex_item.sync_later + + brex_account + end + + private + + def selection_error_payload + if brex_item_id.present? + return { + success: false, + error: "select_connection", + error_message: I18n.t("brex_items.select_accounts.select_connection"), + has_accounts: nil + } + end + + return { success: false, error: "no_credentials", has_accounts: false } unless selection_required? + + { + success: false, + error: "select_connection", + error_message: I18n.t("brex_items.select_accounts.select_connection"), + has_accounts: nil + } + end + + def selection_failure_result(scope, accountable_type: nil) + if selection_required? + SelectionResult.new( + status: :select_connection, + brex_item: nil, + available_accounts: [], + accountable_type: accountable_type, + message: I18n.t("#{scope}.select_connection") + ) + else + SelectionResult.new( + status: :setup_required, + brex_item: nil, + available_accounts: [], + accountable_type: accountable_type, + message: I18n.t("#{scope}.no_credentials_configured") + ) + end + end + + def selection_result_for(scope:, accountable_type:, empty_message_key:, log_context:) + return selection_failure_result(scope, accountable_type: accountable_type) unless selected? + + available_accounts = filter_accounts(unlinked_available_accounts, accountable_type) + if available_accounts.empty? + return selection_result( + status: :empty, + accountable_type: accountable_type, + message: I18n.t("#{scope}.#{empty_message_key}") + ) + end + + selection_result(status: :success, accountable_type: accountable_type, available_accounts: available_accounts) + rescue NoApiTokenError + selection_result( + status: :no_api_token, + accountable_type: accountable_type, + message: I18n.t("#{scope}.no_api_token") + ) + rescue Provider::Brex::BrexError => e + Rails.logger.error("Brex API error in #{log_context}: #{e.message}") + selection_result(status: :api_error, accountable_type: accountable_type, message: e.message) + rescue StandardError => e + Rails.logger.error("Unexpected error in #{log_context}: #{e.class}: #{e.message}") + selection_result( + status: :unexpected_error, + accountable_type: accountable_type, + message: I18n.t("#{scope}.unexpected_error") + ) + end + + def selection_result(status:, accountable_type:, available_accounts: [], message: nil) + SelectionResult.new( + status: status, + brex_item: brex_item, + available_accounts: available_accounts, + accountable_type: accountable_type, + message: message + ) + end + + def linked_account_result + SelectionResult.new( + status: :account_already_linked, + brex_item: brex_item, + available_accounts: [], + accountable_type: nil, + message: I18n.t("brex_items.select_existing_account.account_already_linked") + ) + end + + def link_navigation_result(result) + if result.invalid_count.positive? && result.created_count.zero? && result.already_linked_count.zero? + navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.invalid_account_names", count: result.invalid_count)) + elsif result.invalid_count.positive? && (result.created_count.positive? || result.already_linked_count.positive?) + navigation( + :return_to_or_accounts, + :alert, + I18n.t( + "brex_items.link_accounts.partial_invalid", + created_count: result.created_count, + already_linked_count: result.already_linked_count, + invalid_count: result.invalid_count + ) + ) + elsif result.created_count.positive? && result.already_linked_count.positive? + navigation( + :return_to_or_accounts, + :notice, + I18n.t( + "brex_items.link_accounts.partial_success", + created_count: result.created_count, + already_linked_count: result.already_linked_count, + already_linked_names: result.already_linked_names.join(", ") + ) + ) + elsif result.created_count.positive? + navigation(:return_to_or_accounts, :notice, I18n.t("brex_items.link_accounts.success", count: result.created_count)) + elsif result.already_linked_count.positive? + navigation( + :return_to_or_accounts, + :alert, + I18n.t( + "brex_items.link_accounts.all_already_linked", + count: result.already_linked_count, + names: result.already_linked_names.join(", ") + ) + ) + else + navigation(:new_account, :alert, I18n.t("brex_items.link_accounts.link_failed")) + end + end + + def navigation(target, flash_type, message) + NavigationResult.new(target: target, flash_type: flash_type, message: message) + end + + def cache_key + self.class.cache_key(family, brex_item) + end + + def fetch_accounts + provider = brex_item&.brex_provider + raise NoApiTokenError unless provider.present? + + accounts_data = provider.get_accounts + accounts_data[:accounts] || [] + end + + def accounts + cached_accounts = Rails.cache.read(cache_key) + return cached_accounts unless cached_accounts.nil? + + fetch_and_cache_accounts + end + + def fetch_and_cache_accounts + available_accounts = fetch_accounts + Rails.cache.write(cache_key, available_accounts, expires_in: CACHE_TTL) + available_accounts + end + + def unlinked_available_accounts + linked_account_ids = brex_item.brex_accounts + .joins(:account_provider) + .pluck("#{BrexAccount.table_name}.account_id") + .map(&:to_s) + accounts.reject { |account| linked_account_ids.include?(account.with_indifferent_access[:id].to_s) } + end + + def filter_accounts(accounts, accountable_type) + return [] unless Provider::BrexAdapter.supported_account_types.include?(accountable_type) + + accounts.select do |account| + case accountable_type + when "CreditCard" + BrexAccount.kind_for(account) == "card" + when "Depository" + BrexAccount.kind_for(account) == "cash" + else + true + end + end + end + + def indexed_accounts + accounts.index_by { |account| account.with_indifferent_access[:id].to_s } + end + + def upsert_brex_account!(account_id, account_data) + brex_account = brex_item.brex_accounts.find_or_initialize_by(account_id: account_id.to_s) + brex_account.upsert_brex_snapshot!(account_data) + brex_account + end + + def supported_account_type?(accountable_type) + Provider::BrexAdapter.supported_account_types.include?(accountable_type) + end +end diff --git a/app/models/brex_item/account_flow/setup.rb b/app/models/brex_item/account_flow/setup.rb new file mode 100644 index 000000000..730892b4e --- /dev/null +++ b/app/models/brex_item/account_flow/setup.rb @@ -0,0 +1,242 @@ +# frozen_string_literal: true + +class BrexItem::AccountFlow + module Setup + def import_accounts_from_api_if_needed + raise NoApiTokenError unless brex_item&.credentials_configured? + + available_accounts = fetch_accounts + return nil if available_accounts.empty? + + existing_accounts = brex_item.brex_accounts.index_by(&:account_id) + + available_accounts.each do |account_data| + account_id = account_data.with_indifferent_access[:id].to_s + account_name = BrexAccount.name_for(account_data) + next if account_id.blank? || account_name.blank? + + brex_account = existing_accounts[account_id] + next if brex_account.present? && !brex_account_snapshot_changed?(brex_account, account_data) + + upsert_brex_account!(account_id, account_data) + end + + nil + end + + def unlinked_brex_accounts + brex_item.brex_accounts + .left_joins(:account_provider) + .where(account_providers: { id: nil }) + end + + def account_type_options + supported_types = Provider::BrexAdapter.supported_account_types + account_type_keys = { + "depository" => "Depository", + "credit_card" => "CreditCard", + "investment" => "Investment", + "loan" => "Loan", + "other_asset" => "OtherAsset" + } + + options = account_type_keys.filter_map do |key, type| + next unless supported_types.include?(type) + + [ I18n.t("brex_items.setup_accounts.account_types.#{key}"), type ] + end + + [ [ I18n.t("brex_items.setup_accounts.account_types.skip"), "skip" ] ] + options + end + + def displayable_account_type_options + account_type_options.reject { |_, type| type == "skip" } + end + + def subtype_options + supported_types = Provider::BrexAdapter.supported_account_types + all_subtype_options = { + "Depository" => { + label: I18n.t("brex_items.setup_accounts.subtype_labels.depository"), + options: translate_subtypes("depository", Depository::SUBTYPES) + }, + "CreditCard" => { + label: I18n.t("brex_items.setup_accounts.subtype_labels.credit_card"), + options: [], + message: I18n.t("brex_items.setup_accounts.subtype_messages.credit_card") + }, + "Investment" => { + label: I18n.t("brex_items.setup_accounts.subtype_labels.investment"), + options: translate_subtypes("investment", Investment::SUBTYPES) + }, + "Loan" => { + label: I18n.t("brex_items.setup_accounts.subtype_labels.loan"), + options: translate_subtypes("loan", Loan::SUBTYPES) + }, + "OtherAsset" => { + label: I18n.t("brex_items.setup_accounts.subtype_labels.other_asset", default: "Other asset"), + options: [], + message: I18n.t("brex_items.setup_accounts.subtype_messages.other_asset") + } + } + + all_subtype_options.slice(*supported_types) + end + + def complete_setup!(account_types:, account_subtypes:) + created_accounts = [] + skipped_count = 0 + valid_types = Provider::BrexAdapter.supported_account_types + failed_count = 0 + + submitted_brex_accounts = brex_item.brex_accounts + .where(id: account_types.keys) + .includes(:account_provider) + .index_by { |brex_account| brex_account.id.to_s } + + account_types.each do |brex_account_id, selected_type| + if selected_type == "skip" || selected_type.blank? + skipped_count += 1 + next + end + + unless valid_types.include?(selected_type) + Rails.logger.warn("Invalid account type '#{selected_type}' submitted for Brex account #{brex_account_id}") + skipped_count += 1 + next + end + + brex_account = submitted_brex_accounts[brex_account_id.to_s] + unless brex_account + Rails.logger.warn("Brex account #{brex_account_id} not found for item #{brex_item.id}") + next + end + + if brex_account.account_provider.present? + Rails.logger.info("Brex account #{brex_account_id} already linked, skipping") + next + end + + selected_subtype = selected_subtype_for( + selected_type: selected_type, + submitted_subtype: account_subtypes[brex_account_id] + ) + + begin + ActiveRecord::Base.transaction do + account = Account.create_and_sync( + { + family: family, + name: brex_account.name, + balance: brex_account.current_balance || 0, + currency: brex_account.currency.presence || family.currency, + accountable_type: selected_type, + accountable_attributes: selected_subtype.present? ? { subtype: selected_subtype } : {} + }, + skip_initial_sync: true + ) + + AccountProvider.create!(account: account, provider: brex_account) + created_accounts << account + end + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e + failed_count += 1 + Rails.logger.error("Brex account setup failed for #{brex_account_id}: #{e.class} - #{e.message}") + Rails.logger.error(Array(e.backtrace).first(10).join("\n")) + end + end + + brex_item.sync_later if created_accounts.any? + + SetupResult.new(created_accounts: created_accounts, skipped_count: skipped_count, failed_count: failed_count) + end + + def import_accounts_with_user_facing_error + import_accounts_from_api_if_needed + rescue NoApiTokenError + I18n.t("brex_items.setup_accounts.no_api_token") + rescue Provider::Brex::BrexError => e + Rails.logger.error("Brex API error: #{e.message}") + I18n.t("brex_items.setup_accounts.api_error", message: e.message) + rescue StandardError => e + Rails.logger.error("Unexpected error fetching Brex accounts: #{e.class}: #{e.message}") + I18n.t("brex_items.setup_accounts.api_error", message: I18n.t("brex_items.errors.unexpected_error")) + end + + def complete_setup_result(account_types:, account_subtypes:) + result = complete_setup!(account_types: account_types, account_subtypes: account_subtypes) + + SetupCompletion.new(success: result.failed_count.zero? && result.created_count.positive?, message: setup_notice(result)) + rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotSaved => e + Rails.logger.error("Brex account setup failed: #{e.class} - #{e.message}") + Rails.logger.error(Array(e.backtrace).first(10).join("\n")) + SetupCompletion.new( + success: false, + message: I18n.t("brex_items.complete_account_setup.creation_failed", error: e.message) + ) + rescue StandardError => e + Rails.logger.error("Brex account setup failed unexpectedly: #{e.class} - #{e.message}") + Rails.logger.error(Array(e.backtrace).first(10).join("\n")) + SetupCompletion.new( + success: false, + message: I18n.t( + "brex_items.complete_account_setup.creation_failed", + error: I18n.t("brex_items.complete_account_setup.unexpected_error") + ) + ) + end + + private + + def setup_notice(result) + if result.failed_count.positive? && result.created_count.positive? + I18n.t("brex_items.complete_account_setup.partial_success", created_count: result.created_count, failed_count: result.failed_count) + elsif result.skipped_count.positive? && result.created_count.positive? + I18n.t("brex_items.complete_account_setup.partial_skipped", created_count: result.created_count, skipped_count: result.skipped_count) + elsif result.failed_count.positive? + I18n.t("brex_items.complete_account_setup.creation_failed_count", count: result.failed_count) + elsif result.created_count.positive? + I18n.t("brex_items.complete_account_setup.success", count: result.created_count) + elsif result.skipped_count.positive? + I18n.t("brex_items.complete_account_setup.all_skipped") + else + I18n.t("brex_items.complete_account_setup.no_accounts") + end + end + + def brex_account_snapshot_changed?(brex_account, account_data) + snapshot = account_data.with_indifferent_access + balances = snapshot.slice(:current_balance, :available_balance, :account_limit) + + expected = { + account_kind: BrexAccount.kind_for(snapshot), + account_status: snapshot[:status], + account_type: snapshot[:type], + available_balance: BrexAccount.money_to_decimal(balances[:available_balance]), + current_balance: BrexAccount.money_to_decimal(balances[:current_balance]), + account_limit: BrexAccount.money_to_decimal(balances[:account_limit]), + currency: BrexAccount.currency_code_from_money(balances[:current_balance] || balances[:available_balance] || balances[:account_limit]), + name: BrexAccount.name_for(snapshot), + raw_payload: BrexAccount.sanitize_payload(account_data) + } + + expected.any? { |attribute, value| brex_account.public_send(attribute) != value } + end + + def translate_subtypes(type_key, subtypes_hash) + subtypes_hash.map do |key, value| + [ + I18n.t("brex_items.setup_accounts.subtypes.#{type_key}.#{key}", default: value[:long] || key.to_s.humanize), + key + ] + end + end + + def selected_subtype_for(selected_type:, submitted_subtype:) + return CreditCard::DEFAULT_SUBTYPE if selected_type == "CreditCard" && submitted_subtype.blank? + return Depository::DEFAULT_SUBTYPE if selected_type == "Depository" && submitted_subtype.blank? + + submitted_subtype + end + end +end diff --git a/app/models/brex_item/importer.rb b/app/models/brex_item/importer.rb new file mode 100644 index 000000000..a053c16e2 --- /dev/null +++ b/app/models/brex_item/importer.rb @@ -0,0 +1,245 @@ +# frozen_string_literal: true + +class BrexItem::Importer + attr_reader :brex_item, :brex_provider, :sync_start_date + + def initialize(brex_item, brex_provider:, sync_start_date: nil) + @brex_item = brex_item + @brex_provider = brex_provider + @sync_start_date = sync_start_date + end + + def import + Rails.logger.info "BrexItem::Importer - Starting import for item #{brex_item.id}" + + accounts_data = fetch_accounts_data + return failed_result("Failed to fetch accounts data") unless accounts_data + + store_item_snapshot(accounts_data) + + account_result = import_accounts(accounts_data[:accounts].to_a) + transaction_result = import_transactions + + brex_item.update!(status: :good) if account_result[:accounts_failed].zero? && transaction_result[:transactions_failed].zero? + + { + success: account_result[:accounts_failed].zero? && transaction_result[:transactions_failed].zero?, + **account_result, + **transaction_result + } + end + + private + + def fetch_accounts_data + accounts_data = brex_provider.get_accounts + + unless accounts_data.is_a?(Hash) + Rails.logger.error "BrexItem::Importer - Invalid accounts_data format: expected Hash, got #{accounts_data.class}" + return nil + end + + accounts_data + rescue Provider::Brex::BrexError => e + mark_requires_update_if_credentials_error(e) + Rails.logger.error "BrexItem::Importer - Brex API error: #{e.message} trace_id=#{e.trace_id}" + nil + rescue JSON::ParserError => e + Rails.logger.error "BrexItem::Importer - Failed to parse Brex API response: #{e.message}" + nil + rescue => e + Rails.logger.error "BrexItem::Importer - Unexpected error fetching accounts: #{e.class} - #{e.message}" + Rails.logger.error Array(e.backtrace).join("\n") + nil + end + + def store_item_snapshot(accounts_data) + brex_item.upsert_brex_snapshot!(accounts_data) + rescue => e + Rails.logger.error "BrexItem::Importer - Failed to store accounts snapshot: #{e.message}" + Sentry.capture_exception(e) do |scope| + scope.set_tags(brex_item_id: brex_item.id) + scope.set_context("brex_item_snapshot", { + brex_item_id: brex_item.id, + accounts_data: BrexAccount.sanitize_payload(accounts_data) + }) + end + raise + end + + def import_accounts(accounts) + accounts_updated = 0 + accounts_created = 0 + accounts_failed = 0 + + all_existing_ids = brex_item.brex_accounts.pluck("#{BrexAccount.table_name}.account_id").map(&:to_s) + + accounts.each do |account_data| + snapshot = account_data.with_indifferent_access + account_id = snapshot[:id].to_s + account_name = BrexAccount.name_for(snapshot) + next if account_id.blank? || account_name.blank? + + if all_existing_ids.include?(account_id) + import_account(snapshot) + accounts_updated += 1 + else + import_account(snapshot) + accounts_created += 1 + all_existing_ids << account_id + end + rescue => e + accounts_failed += 1 + Rails.logger.error "BrexItem::Importer - Failed to import account #{account_id.presence || 'unknown'}: #{e.message}" + end + + { + accounts_updated: accounts_updated, + accounts_created: accounts_created, + accounts_failed: accounts_failed + } + end + + def import_account(account_data) + account_id = account_data[:id].to_s + raise ArgumentError, "Account ID is required" if account_id.blank? + + brex_account = brex_item.brex_accounts.find_or_initialize_by(account_id: account_id) + brex_account.name ||= BrexAccount.name_for(account_data) + brex_account.currency ||= BrexAccount.currency_code_from_money(account_data[:current_balance] || account_data[:available_balance] || account_data[:account_limit]) + brex_account.upsert_brex_snapshot!(account_data) + brex_account + end + + def import_transactions + transactions_imported = 0 + transactions_failed = 0 + + brex_item.brex_accounts.joins(:account).merge(Account.visible).find_each do |brex_account| + result = fetch_and_store_transactions(brex_account) + if result[:success] + transactions_imported += result[:transactions_count] + else + transactions_failed += 1 + end + rescue => e + transactions_failed += 1 + Rails.logger.error "BrexItem::Importer - Failed to fetch/store transactions for account #{brex_account.account_id}: #{e.message}" + end + + { + transactions_imported: transactions_imported, + transactions_failed: transactions_failed + } + end + + def fetch_and_store_transactions(brex_account) + start_date = determine_sync_start_date(brex_account) + Rails.logger.info "BrexItem::Importer - Fetching #{brex_account.account_kind} transactions for account #{brex_account.account_id} from #{start_date}" + + transactions_data = if brex_account.card? + brex_provider.get_primary_card_transactions(start_date: start_date) + else + brex_provider.get_cash_transactions(brex_account.account_id, start_date: start_date) + end + + unless transactions_data.is_a?(Hash) + Rails.logger.error "BrexItem::Importer - Invalid transactions_data format for account #{brex_account.account_id}" + return { success: false, transactions_count: 0, error: "Invalid response format" } + end + + transactions = transactions_data[:transactions].to_a + created_count = store_new_transactions(brex_account, transactions, window_start_date: start_date) + + { success: true, transactions_count: created_count } + rescue Provider::Brex::BrexError => e + mark_requires_update_if_credentials_error(e) + Rails.logger.error "BrexItem::Importer - Brex API error for account #{brex_account.account_id}: #{e.message} trace_id=#{e.trace_id}" + { success: false, transactions_count: 0, error: e.message } + rescue JSON::ParserError => e + Rails.logger.error "BrexItem::Importer - Failed to parse transaction response for account #{brex_account.account_id}: #{e.message}" + { success: false, transactions_count: 0, error: "Failed to parse response" } + rescue => e + Rails.logger.error "BrexItem::Importer - Unexpected error fetching transactions for account #{brex_account.account_id}: #{e.class} - #{e.message}" + Rails.logger.error Array(e.backtrace).join("\n") + { success: false, transactions_count: 0, error: "Unexpected error: #{e.message}" } + end + + def store_new_transactions(brex_account, transactions, window_start_date:) + existing_payload = brex_account.raw_transactions_payload.to_a + existing_transactions = transactions_in_window(existing_payload, window_start_date) + existing_ids = existing_transactions.map { |tx| tx.with_indifferent_access[:id] }.to_set + + new_transactions = transactions.select do |tx| + tx_id = tx.with_indifferent_access[:id] + tx_id.present? && !existing_ids.include?(tx_id) && transaction_in_window?(tx, window_start_date) + end + + return 0 if new_transactions.empty? && existing_transactions.count == existing_payload.count + + brex_account.upsert_brex_transactions_snapshot!(existing_transactions + new_transactions) + new_transactions.count + end + + def transactions_in_window(transactions, window_start_date) + transactions.select { |transaction| transaction_in_window?(transaction, window_start_date) } + end + + def transaction_in_window?(transaction, window_start_date) + return true if window_start_date.blank? + + transaction_date = transaction_date_for(transaction) + return true if transaction_date.blank? + + transaction_date >= window_start_date.to_date + end + + def transaction_date_for(transaction) + data = transaction.with_indifferent_access + date_value = data[:posted_at_date].presence || data[:initiated_at_date].presence || data[:posted_at].presence || data[:created_at].presence + + case date_value + when Date + date_value + when Time, DateTime + date_value.to_date + when String + Date.parse(date_value) + else + nil + end + rescue ArgumentError, TypeError + nil + end + + def determine_sync_start_date(brex_account) + return sync_start_date if sync_start_date.present? + + if brex_account.raw_transactions_payload.to_a.any? + brex_item.last_synced_at ? brex_item.last_synced_at - 7.days : 90.days.ago + else + account_baseline = brex_account.created_at || Time.current + [ account_baseline - 7.days, 90.days.ago ].max + end + end + + def mark_requires_update_if_credentials_error(error) + return unless error.error_type.in?([ :unauthorized, :access_forbidden ]) + + brex_item.update!(status: :requires_update) + rescue => update_error + Rails.logger.error "BrexItem::Importer - Failed to update item status: #{update_error.message}" + end + + def failed_result(error) + { + success: false, + error: error, + accounts_updated: 0, + accounts_created: 0, + accounts_failed: 0, + transactions_imported: 0, + transactions_failed: 0 + } + end +end diff --git a/app/models/brex_item/provided.rb b/app/models/brex_item/provided.rb new file mode 100644 index 000000000..6e4b22d14 --- /dev/null +++ b/app/models/brex_item/provided.rb @@ -0,0 +1,16 @@ +module BrexItem::Provided + extend ActiveSupport::Concern + + def brex_provider + return nil unless credentials_configured? + + base_url = effective_base_url + return nil unless base_url.present? + + Provider::Brex.new(token.to_s.strip, base_url: base_url) + end + + def syncer + BrexItem::Syncer.new(self) + end +end diff --git a/app/models/brex_item/syncer.rb b/app/models/brex_item/syncer.rb new file mode 100644 index 000000000..3e5de1686 --- /dev/null +++ b/app/models/brex_item/syncer.rb @@ -0,0 +1,148 @@ +class BrexItem::Syncer + include SyncStats::Collector + + SafeSyncError = Class.new(StandardError) + + attr_reader :brex_item + + def initialize(brex_item) + @brex_item = brex_item + end + + def perform_sync(sync) + sync_errors = [] + + # Phase 1: Import data from Brex API + update_status(sync, :importing_accounts) + import_result = brex_item.import_latest_brex_data(sync_start_date: sync.window_start_date) + sync_errors.concat(import_result_errors(import_result)) + + # Phase 2: Collect setup statistics + update_status(sync, :checking_account_configuration) + + linked_count = brex_item.brex_accounts.joins(:account_provider).count + unlinked_count = brex_item.brex_accounts + .left_joins(:account_provider) + .where(account_providers: { id: nil }) + .count + total_count = linked_count + unlinked_count + collect_brex_setup_stats( + sync, + total_count: total_count, + linked_count: linked_count, + unlinked_count: unlinked_count + ) + + # Set pending_account_setup if there are unlinked accounts + if unlinked_count.positive? + brex_item.update!(pending_account_setup: true) + update_status(sync, :accounts_need_setup, count: unlinked_count) + else + brex_item.update!(pending_account_setup: false) + end + + # Phase 3: Process transactions for linked accounts only + if linked_count.positive? + linked_accounts = brex_item.brex_accounts.joins(:account_provider) + update_status(sync, :processing_transactions) + mark_import_started(sync) + Rails.logger.info "BrexItem::Syncer - Processing #{linked_count} linked accounts" + process_results = brex_item.process_accounts + sync_errors.concat(result_failure_errors(process_results, category: :account_processing_error, message_key: :account_processing_failed)) + Rails.logger.info "BrexItem::Syncer - Finished processing accounts" + + # Phase 4: Schedule balance calculations for linked accounts + update_status(sync, :calculating_balances) + schedule_results = brex_item.schedule_account_syncs( + parent_sync: sync, + window_start_date: sync.window_start_date, + window_end_date: sync.window_end_date + ) + sync_errors.concat(result_failure_errors(schedule_results, category: :account_sync_error, message_key: :account_sync_failed)) + + # Phase 5: Collect transaction statistics + account_ids = linked_accounts + .includes(account_provider: :account) + .filter_map { |ma| ma.current_account&.id } + collect_transaction_stats(sync, account_ids: account_ids, source: "brex") + else + Rails.logger.info "BrexItem::Syncer - No linked accounts to process" + end + + # Mark sync health + collect_health_stats(sync, errors: sync_errors.presence) + rescue => e + safe_message = user_safe_error_message(e) + Rails.logger.error "BrexItem::Syncer - sync failed for Brex item #{brex_item.id}: #{e.class} - #{e.message}" + Rails.logger.error Array(e.backtrace).first(10).join("\n") + Sentry.capture_exception(e) do |scope| + scope.set_tags(brex_item_id: brex_item.id) + end + collect_health_stats(sync, errors: [ { message: safe_message, category: "sync_error" } ]) + raise SafeSyncError, safe_message + end + + def perform_post_sync + # no-op + end + + private + + def update_status(sync, key, **options) + return unless sync.respond_to?(:status_text) + + sync.update!(status_text: I18n.t("brex_items.syncer.#{key}", **options)) + end + + def collect_brex_setup_stats(sync, total_count:, linked_count:, unlinked_count:) + return {} unless sync.respond_to?(:sync_stats) + + setup_stats = { + "total_accounts" => total_count, + "linked_accounts" => linked_count, + "unlinked_accounts" => unlinked_count + } + + merge_sync_stats(sync, setup_stats) + setup_stats + end + + def import_result_errors(result) + return [] if result.is_a?(Hash) && result[:success] + + unless result.is_a?(Hash) + return [ sync_error(:import_error, :import_failed) ] + end + + errors = [] + accounts_failed = result[:accounts_failed].to_i + transactions_failed = result[:transactions_failed].to_i + + errors << sync_error(:account_import_error, :accounts_failed, count: accounts_failed) if accounts_failed.positive? + errors << sync_error(:transaction_import_error, :transactions_failed, count: transactions_failed) if transactions_failed.positive? + errors << sync_error(:import_error, :import_failed) if errors.empty? + errors + end + + def result_failure_errors(results, category:, message_key:) + failed_count = Array(results).count { |result| result.is_a?(Hash) && result[:success] == false } + return [] unless failed_count.positive? + + [ sync_error(category, message_key, count: failed_count) ] + end + + def sync_error(category, message_key, **options) + { + message: I18n.t("brex_items.syncer.#{message_key}", **options), + category: category.to_s + } + end + + def user_safe_error_message(error) + if error.is_a?(Provider::Brex::BrexError) && error.error_type.in?([ :unauthorized, :access_forbidden ]) + I18n.t("brex_items.syncer.credentials_invalid") + else + I18n.t("brex_items.syncer.failed") + end + end +end diff --git a/app/models/brex_item/unlinking.rb b/app/models/brex_item/unlinking.rb new file mode 100644 index 000000000..a2c1d3703 --- /dev/null +++ b/app/models/brex_item/unlinking.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module BrexItem::Unlinking + # Concern that encapsulates unlinking logic for a Brex item. + extend ActiveSupport::Concern + + # Idempotently remove all connections between this Brex item and local accounts. + # - Detaches any AccountProvider links for each BrexAccount + # - Detaches Holdings that point at the AccountProvider links + # Returns a per-account result payload for observability + def unlink_all!(dry_run: false) + results = [] + + brex_accounts.find_each do |provider_account| + result = { + provider_account_id: provider_account.id, + name: provider_account.name, + provider_link_ids: [] + } + results << result + + if dry_run + result[:provider_link_ids] = AccountProvider.where(provider_type: "BrexAccount", provider_id: provider_account.id).ids + next + end + + link_ids = [] + + begin + ActiveRecord::Base.transaction do + links = AccountProvider.where(provider_type: "BrexAccount", provider_id: provider_account.id).to_a + link_ids = links.map(&:id) + result[:provider_link_ids] = link_ids + + # Detach holdings for any provider links found + if link_ids.any? + Holding.where(account_provider_id: link_ids).update_all(account_provider_id: nil) + end + + # Destroy all provider links + links.each do |ap| + ap.destroy! + end + end + rescue StandardError => e + Rails.logger.warn( + "BrexItem Unlinker: failed to fully unlink provider account ##{provider_account.id} (links=#{link_ids.inspect}): #{e.class} - #{e.message}" + ) + # Record error for observability; continue with other accounts + result[:error] = e.message + end + end + + results + end +end diff --git a/app/models/concerns/encryptable.rb b/app/models/concerns/encryptable.rb index 0ec5ae923..a04c773de 100644 --- a/app/models/concerns/encryptable.rb +++ b/app/models/concerns/encryptable.rb @@ -6,11 +6,7 @@ module Encryptable # This allows encryption to be optional - if not configured, sensitive fields # are stored in plaintext (useful for development or legacy deployments). def encryption_ready? - creds_ready = Rails.application.credentials.active_record_encryption.present? - env_ready = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"].present? && - ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"].present? && - ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"].present? - creds_ready || env_ready + ActiveRecordEncryptionConfig.explicitly_configured? end end end diff --git a/app/models/credit_card.rb b/app/models/credit_card.rb index 05bf7746a..4f42beaee 100644 --- a/app/models/credit_card.rb +++ b/app/models/credit_card.rb @@ -1,6 +1,8 @@ class CreditCard < ApplicationRecord include Accountable + DEFAULT_SUBTYPE = "credit_card" + SUBTYPES = { "credit_card" => { short: "Credit Card", long: "Credit Card" } }.freeze diff --git a/app/models/data_enrichment.rb b/app/models/data_enrichment.rb index 817b05fb2..d14b47eb5 100644 --- a/app/models/data_enrichment.rb +++ b/app/models/data_enrichment.rb @@ -11,6 +11,7 @@ class DataEnrichment < ApplicationRecord enable_banking: "enable_banking", coinstats: "coinstats", mercury: "mercury", + brex: "brex", indexa_capital: "indexa_capital", sophtron: "sophtron", ibkr: "ibkr" diff --git a/app/models/depository.rb b/app/models/depository.rb index b788a6d4e..e78e70a8a 100644 --- a/app/models/depository.rb +++ b/app/models/depository.rb @@ -1,6 +1,8 @@ class Depository < ApplicationRecord include Accountable + DEFAULT_SUBTYPE = "checking" + SUBTYPES = { "checking" => { short: "Checking", long: "Checking" }, "savings" => { short: "Savings", long: "Savings" }, diff --git a/app/models/family.rb b/app/models/family.rb index 7ebec6c5d..fb211d78c 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -1,7 +1,7 @@ class Family < ApplicationRecord include Syncable, AutoTransferMatchable, Subscribeable, VectorSearchable include PlaidConnectable, SimplefinConnectable, LunchflowConnectable, EnableBankingConnectable - include CoinbaseConnectable, BinanceConnectable, KrakenConnectable, CoinstatsConnectable, SnaptradeConnectable, MercuryConnectable, SophtronConnectable + include CoinbaseConnectable, BinanceConnectable, KrakenConnectable, CoinstatsConnectable, SnaptradeConnectable, MercuryConnectable, BrexConnectable, SophtronConnectable include IndexaCapitalConnectable, IbkrConnectable DATE_FORMATS = [ diff --git a/app/models/family/brex_connectable.rb b/app/models/family/brex_connectable.rb new file mode 100644 index 000000000..49fe3e560 --- /dev/null +++ b/app/models/family/brex_connectable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Family::BrexConnectable + extend ActiveSupport::Concern + + included do + has_many :brex_items, dependent: :destroy + end + + def can_connect_brex? + true + end + + def create_brex_item!(token:, base_url: nil, item_name: nil) + brex_item = brex_items.create!( + name: item_name.presence || I18n.t("brex_items.default_connection_name"), + token: token, + base_url: base_url + ) + + brex_item.sync_later + + brex_item + end + + def has_brex_credentials? + brex_items.active.with_credentials.exists? + end +end diff --git a/app/models/family/syncer.rb b/app/models/family/syncer.rb index 6b909ebcb..7873c5ff0 100644 --- a/app/models/family/syncer.rb +++ b/app/models/family/syncer.rb @@ -17,6 +17,7 @@ class Family::Syncer coinbase_items coinstats_items mercury_items + brex_items binance_items snaptrade_items sophtron_items diff --git a/app/models/provider/brex.rb b/app/models/provider/brex.rb new file mode 100644 index 000000000..969dfee50 --- /dev/null +++ b/app/models/provider/brex.rb @@ -0,0 +1,271 @@ +# frozen_string_literal: true + +class Provider::Brex + include HTTParty + extend SslConfigurable + + DEFAULT_BASE_URL = "https://api.brex.com" + STAGING_BASE_URL = "https://api-staging.brex.com" + ALLOWED_BASE_URLS = [ DEFAULT_BASE_URL, STAGING_BASE_URL ].freeze + DEFAULT_LIMIT = 1000 + # Transaction syncs are date-window bounded; this is only a runaway cursor guard. + MAX_PAGES = 25 + + headers "User-Agent" => "Sure Finance Brex Client" + default_options.merge!({ timeout: 120 }.merge(httparty_ssl_options)) + + attr_reader :token, :base_url + + def initialize(token, base_url: DEFAULT_BASE_URL) + @token = token.to_s.strip + @base_url = self.class.normalize_base_url(base_url) + raise ArgumentError, "Brex base URL must be blank or one of: #{ALLOWED_BASE_URLS.join(', ')}" unless @base_url.present? + end + + def self.normalize_base_url(value) + stripped = value.to_s.strip + return DEFAULT_BASE_URL if stripped.blank? + + uri = URI.parse(stripped) + return nil unless uri.is_a?(URI::HTTPS) + return nil if uri.userinfo.present? + return nil if uri.query.present? || uri.fragment.present? + return nil unless uri.path.blank? || uri.path == "/" + return nil unless uri.port == 443 + + # This exact allowlist is the SSRF boundary; arbitrary Brex-like hosts are never accepted. + normalized = "#{uri.scheme.downcase}://#{uri.host.to_s.downcase}" + ALLOWED_BASE_URLS.include?(normalized) ? normalized : nil + rescue URI::InvalidURIError + nil + end + + def self.allowed_base_url?(value) + normalize_base_url(value).present? + end + + def get_accounts + cash_accounts = get_cash_accounts + card_accounts = get_card_accounts + + accounts = cash_accounts.dup + accounts << aggregate_card_account(card_accounts) if card_accounts.any? + + { + accounts: accounts, + cash_accounts: cash_accounts, + card_accounts: card_accounts + } + end + + def get_cash_accounts + get_paginated("/v2/accounts/cash").map { |account| account.with_indifferent_access.merge(account_kind: "cash") } + end + + def get_card_accounts + get_paginated("/v2/accounts/card").map { |account| account.with_indifferent_access.merge(account_kind: "card") } + end + + def get_cash_transactions(account_id, start_date: nil) + path = "/v2/transactions/cash/#{ERB::Util.url_encode(account_id.to_s)}" + { + transactions: get_paginated(path, params: posted_at_start_params(start_date)) + } + end + + def get_primary_card_transactions(start_date: nil) + { + transactions: get_paginated("/v2/transactions/card/primary", params: posted_at_start_params(start_date)) + } + end + + private + + def aggregate_card_account(card_accounts) + totals = %i[current_balance available_balance account_limit].index_with do |field| + sum_money(card_accounts.filter_map { |account| account.with_indifferent_access[field] }) + end + + { + id: BrexAccount.card_account_id, + name: "Brex Card", + account_kind: "card", + status: card_accounts.map { |account| account.with_indifferent_access[:status] }.compact.first, + card_accounts_count: card_accounts.count, + current_balance: totals[:current_balance], + available_balance: totals[:available_balance], + account_limit: totals[:account_limit], + raw_card_accounts: BrexAccount.sanitize_payload(card_accounts) + }.compact + end + + def sum_money(money_values) + normalized = money_values.compact + return nil if normalized.empty? + + currencies = normalized.map { |money| BrexAccount.currency_code_from_money(money) }.uniq + if currencies.many? + Rails.logger.warn "Brex API: Cannot aggregate card balances with mixed currencies: #{currencies.join(', ')}" + return nil + end + + currency = currencies.first + total = normalized.sum do |money| + money.with_indifferent_access[:amount].to_i + end + + { amount: total, currency: currency } + end + + def posted_at_start_params(start_date) + return {} if start_date.blank? + + { posted_at_start: rfc3339_start_date(start_date) } + end + + def get_paginated(path, params: {}) + records = [] + cursor = nil + seen_cursors = Set.new + page_count = 0 + + loop do + page_count += 1 + raise BrexError.new("Brex pagination exceeded #{MAX_PAGES} pages", :pagination_error) if page_count > MAX_PAGES + + page_params = params.compact.merge(limit: DEFAULT_LIMIT) + page_params[:cursor] = cursor if cursor.present? + + response_payload = get_json(path, params: page_params) + if response_payload.is_a?(Array) + records.concat(response_payload) + break + end + + page_records = extract_records(response_payload) + records.concat(page_records) + + next_cursor = response_payload.with_indifferent_access[:next_cursor] + break if next_cursor.blank? + + if seen_cursors.include?(next_cursor) + raise BrexError.new("Brex pagination returned a repeated cursor", :pagination_error) + end + + seen_cursors.add(next_cursor) + cursor = next_cursor + end + + records + end + + def get_json(path, params: {}) + query = params.present? ? "?#{URI.encode_www_form(params)}" : "" + request_path = "#{path}#{query}" + + response = self.class.get( + "#{base_url}#{request_path}", + headers: auth_headers + ) + + handle_response(response, path: path) + rescue BrexError + raise + rescue SocketError, Net::OpenTimeout, Net::ReadTimeout => e + Rails.logger.error "Brex API: GET #{path} failed: #{e.class}: #{e.message}" + raise BrexError.new("Exception during GET request: #{e.message}", :request_failed) + rescue JSON::ParserError => e + Rails.logger.error "Brex API: invalid JSON for GET #{path}: #{e.message}" + raise BrexError.new("Invalid response from Brex API", :invalid_response) + rescue => e + Rails.logger.error "Brex API: Unexpected error during GET #{path}: #{e.class}: #{e.message}" + raise BrexError.new("Exception during GET request: #{e.message}", :request_failed) + end + + def extract_records(response_payload) + return response_payload if response_payload.is_a?(Array) + + payload = response_payload.with_indifferent_access + payload[:items] || + payload[:data] || + payload[:accounts] || + payload[:transactions] || + [] + end + + def auth_headers + { + "Authorization" => "Bearer #{token}", + "Content-Type" => "application/json", + "Accept" => "application/json" + } + end + + def handle_response(response, path:) + trace_id = brex_trace_id(response) + + case response.code + when 200 + parse_json(response.body) + when 400 + Rails.logger.error "Brex API: bad request for #{path} trace_id=#{trace_id}" + raise BrexError.new("Bad request to Brex API", :bad_request, http_status: 400, trace_id: trace_id) + when 401 + Rails.logger.warn "Brex API: unauthorized for #{path} trace_id=#{trace_id}" + raise BrexError.new("Invalid Brex API token or account permissions", :unauthorized, http_status: 401, trace_id: trace_id) + when 403 + Rails.logger.warn "Brex API: access forbidden for #{path} trace_id=#{trace_id}" + raise BrexError.new("Access forbidden - check Brex API token scopes", :access_forbidden, http_status: 403, trace_id: trace_id) + when 404 + Rails.logger.warn "Brex API: resource not found for #{path} trace_id=#{trace_id}" + raise BrexError.new("Brex resource not found", :not_found, http_status: 404, trace_id: trace_id) + when 429 + Rails.logger.warn "Brex API: rate limited for #{path} trace_id=#{trace_id}" + raise BrexError.new("Brex rate limit exceeded. Please try again later.", :rate_limited, http_status: 429, trace_id: trace_id) + else + Rails.logger.error "Brex API: unexpected response code=#{response.code} path=#{path} trace_id=#{trace_id}" + raise BrexError.new("Failed to fetch data from Brex API: HTTP #{response.code}", :fetch_failed, http_status: response.code, trace_id: trace_id) + end + end + + def parse_json(body) + return {} if body.blank? + + JSON.parse(body, symbolize_names: true) + end + + def rfc3339_start_date(start_date) + time = + case start_date + when Time + start_date + when DateTime + start_date.to_time + when Date + start_date.to_time(:utc) + else + Time.zone.parse(start_date.to_s) + end + + raise ArgumentError, "Invalid start_date: #{start_date.inspect}" if time.nil? + + time.utc.iso8601 + end + + def brex_trace_id(response) + headers = response.respond_to?(:headers) ? response.headers : {} + headers["X-Brex-Trace-Id"].presence || + headers["x-brex-trace-id"].presence + end + + class BrexError < StandardError + attr_reader :error_type, :http_status, :trace_id + + def initialize(message, error_type = :unknown, http_status: nil, trace_id: nil) + super(message) + @error_type = error_type + @http_status = http_status + @trace_id = trace_id + end + end +end diff --git a/app/models/provider/brex_adapter.rb b/app/models/provider/brex_adapter.rb new file mode 100644 index 000000000..dbed8c7ca --- /dev/null +++ b/app/models/provider/brex_adapter.rb @@ -0,0 +1,119 @@ +class Provider::BrexAdapter < Provider::Base + include Provider::Syncable + include Provider::InstitutionMetadata + + # Register this adapter with the factory + Provider::Factory.register("BrexAccount", self) + + def self.supported_account_types + %w[Depository CreditCard] + end + + # Returns connection configurations for this provider + def self.connection_configs(family:) + return [] unless family.can_connect_brex? + + brex_items = family.brex_items.active.with_credentials.ordered + + return [ connection_config_for(nil) ] if brex_items.empty? + + brex_items.map { |brex_item| connection_config_for(brex_item) } + end + + def provider_name + "brex" + end + + # Build a Brex provider instance with family-specific credentials + # @param family [Family] The family to get credentials for (required) + # @return [Provider::Brex, nil] Returns nil if credentials are not configured + def self.build_provider(family: nil, brex_item_id: nil) + return nil unless family.present? + + brex_item = BrexItem.resolve_for(family: family, brex_item_id: brex_item_id) + return nil unless brex_item&.credentials_configured? + + base_url = brex_item.effective_base_url + return nil unless base_url.present? + + Provider::Brex.new( + brex_item.token.to_s.strip, + base_url: base_url + ) + end + + def self.connection_config_for(brex_item) + path_params = ->(extra = {}) do + brex_item.present? ? extra.merge(brex_item_id: brex_item.id) : extra + end + + { + key: brex_item.present? ? "brex_#{brex_item.id}" : "brex", + name: brex_item.present? ? I18n.t("brex_items.provider_connection.name", name: brex_item.name) : I18n.t("brex_items.provider_connection.default_name"), + description: brex_item.present? ? I18n.t("brex_items.provider_connection.description", name: brex_item.name) : I18n.t("brex_items.provider_connection.default_description"), + can_connect: true, + new_account_path: ->(accountable_type, return_to) { + Rails.application.routes.url_helpers.select_accounts_brex_items_path( + path_params.call(accountable_type: accountable_type, return_to: return_to) + ) + }, + existing_account_path: ->(account_id) { + Rails.application.routes.url_helpers.select_existing_account_brex_items_path( + path_params.call(account_id: account_id) + ) + } + } + end + private_class_method :connection_config_for + + def sync_path + Rails.application.routes.url_helpers.sync_brex_item_path(item) + end + + def item + provider_account.brex_item + end + + def can_delete_holdings? + false + end + + def institution_domain + metadata = provider_account.institution_metadata + return nil unless metadata.present? + + domain = metadata["domain"] + url = metadata["url"] + + # Derive domain from URL if missing + if domain.blank? && url.present? + begin + parsed_host = URI.parse(url).host + Rails.logger.warn("Brex account #{provider_account.id} institution URL has no host: #{url}") if parsed_host.nil? + domain = parsed_host&.gsub(/^www\./, "") + rescue URI::InvalidURIError + Rails.logger.warn("Invalid institution URL for Brex account #{provider_account.id}: #{url}") + end + end + + domain + end + + def institution_name + metadata = provider_account.institution_metadata + + metadata&.dig("name") || item&.institution_name + end + + def institution_url + metadata = provider_account.institution_metadata + + metadata&.dig("url") || item&.institution_url + end + + def institution_color + metadata = provider_account.institution_metadata + + metadata&.dig("color") || item&.institution_color + end +end diff --git a/app/models/provider/metadata.rb b/app/models/provider/metadata.rb index 3d8472d70..4c8263b8a 100644 --- a/app/models/provider/metadata.rb +++ b/app/models/provider/metadata.rb @@ -6,6 +6,7 @@ class Provider enable_banking: { region: "EU", kind: "Bank", maturity: :beta, logo_text: "EB", logo_bg: "bg-purple-600" }, coinstats: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "CS", logo_bg: "bg-pink-600" }, mercury: { region: "US", kind: "Bank", maturity: :beta, logo_text: "ME", logo_bg: "bg-cyan-600" }, + brex: { region: "US", kind: "Bank", maturity: :beta, logo_text: "BX", logo_bg: "bg-emerald-600" }, coinbase: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "CB", logo_bg: "bg-blue-500" }, binance: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "BI", logo_bg: "bg-yellow-600" }, kraken: { region: "Global", kind: "Crypto", maturity: :beta, logo_text: "KR", logo_bg: "bg-violet-600" }, diff --git a/app/models/provider_connection_status.rb b/app/models/provider_connection_status.rb index 47ce145ea..0563d6f77 100644 --- a/app/models/provider_connection_status.rb +++ b/app/models/provider_connection_status.rb @@ -13,6 +13,7 @@ class ProviderConnectionStatus { key: "snaptrade", type: "SnaptradeItem", association: :snaptrade_items, accounts: :snaptrade_accounts, linked_accounts: :linked_accounts }, { key: "ibkr", type: "IbkrItem", association: :ibkr_items, accounts: :ibkr_accounts }, { key: "mercury", type: "MercuryItem", association: :mercury_items, accounts: :mercury_accounts }, + { key: "brex", type: "BrexItem", association: :brex_items, accounts: :brex_accounts }, { key: "sophtron", type: "SophtronItem", association: :sophtron_items, accounts: :sophtron_accounts }, { key: "indexa_capital", type: "IndexaCapitalItem", association: :indexa_capital_items, accounts: :indexa_capital_accounts } ].freeze diff --git a/app/models/provider_merchant.rb b/app/models/provider_merchant.rb index 089d937eb..5cfb2fdf9 100644 --- a/app/models/provider_merchant.rb +++ b/app/models/provider_merchant.rb @@ -1,5 +1,5 @@ class ProviderMerchant < Merchant - enum :source, { plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats", mercury: "mercury", indexa_capital: "indexa_capital", sophtron: "sophtron" } + enum :source, { plaid: "plaid", simplefin: "simplefin", lunchflow: "lunchflow", synth: "synth", ai: "ai", enable_banking: "enable_banking", coinstats: "coinstats", mercury: "mercury", brex: "brex", indexa_capital: "indexa_capital", sophtron: "sophtron" } validates :name, uniqueness: { scope: [ :source ] } validates :source, presence: true diff --git a/app/views/accounts/index.html.erb b/app/views/accounts/index.html.erb index bd9d4b3af..38ba4d888 100644 --- a/app/views/accounts/index.html.erb +++ b/app/views/accounts/index.html.erb @@ -17,7 +17,7 @@ ) %> <% end %> -<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @snaptrade_items.empty? && @ibkr_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? %> +<% if @manual_accounts.empty? && @plaid_items.empty? && @simplefin_items.empty? && @lunchflow_items.empty? && @enable_banking_items.empty? && @coinstats_items.empty? && @coinbase_items.empty? && @mercury_items.empty? && @brex_items.empty? && @ibkr_items.empty? && @snaptrade_items.empty? && @indexa_capital_items.empty? && @sophtron_items.empty? %> <%= render "empty" %> <% else %>
@@ -49,6 +49,10 @@ <%= render @mercury_items.sort_by(&:created_at) %> <% end %> + <% if @brex_items.any? %> + <%= render @brex_items.sort_by(&:created_at) %> + <% end %> + <% if @coinbase_items.any? %> <%= render @coinbase_items.sort_by(&:created_at) %> <% end %> @@ -62,8 +66,8 @@ <% end %> <% if @indexa_capital_items.any? %> - <%= render @indexa_capital_items.sort_by(&:created_at) %> -<% end %> + <%= render @indexa_capital_items.sort_by(&:created_at) %> + <% end %> <% if @manual_accounts.any? %>
diff --git a/app/views/brex_items/_api_error.html.erb b/app/views/brex_items/_api_error.html.erb new file mode 100644 index 000000000..8f05f813b --- /dev/null +++ b/app/views/brex_items/_api_error.html.erb @@ -0,0 +1,36 @@ +<%# locals: (error_message:, return_path:) %> +<%= turbo_frame_tag "modal" do %> + <%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title")) %> + <% dialog.with_body do %> +
+
+ <%= icon("alert-circle", class: "text-destructive w-5 h-5 shrink-0 mt-0.5") %> +
+

<%= t(".heading") %>

+

<%= error_message %>

+
+
+ +
+

<%= t(".common_issues") %>

+
    +
  • <%= t(".invalid_token_label") %> <%= t(".invalid_token") %>
    • +
  • <%= t(".expired_credentials_label") %> <%= t(".expired_credentials") %>
    • +
  • <%= t(".permissions_label") %> <%= t(".permissions") %>
    • +
  • <%= t(".network_label") %> <%= t(".network") %>
    • +
  • <%= t(".service_label") %> <%= t(".service") %>
    • +
+
+ +
+ <%= link_to return_path.presence || settings_providers_path, + class: "inline-flex items-center justify-center rounded-lg px-4 py-2 text-sm font-medium text-inverse button-bg-primary hover:button-bg-primary-hover focus:outline-none focus:ring-2 focus:ring-primary focus:ring-offset-2 transition-colors", + data: { turbo: false } do %> + <%= t(".settings_link") %> + <% end %> +
+
+ <% end %> + <% end %> +<% end %> diff --git a/app/views/brex_items/_brex_item.html.erb b/app/views/brex_items/_brex_item.html.erb new file mode 100644 index 000000000..4b97ceb84 --- /dev/null +++ b/app/views/brex_items/_brex_item.html.erb @@ -0,0 +1,132 @@ +<%# locals: (brex_item:) %> +<% render_locals = brex_item_render_locals( + brex_item, + sync_stats_map: @brex_sync_stats_map, + account_counts_map: @brex_account_counts_map, + institutions_count_map: @brex_institutions_count_map + ) %> +<% stats = render_locals[:stats] %> +<% unlinked_count = render_locals[:unlinked_count] %> +<% linked_count = render_locals[:linked_count] %> +<% total_count = render_locals[:total_count] %> +<% institutions_count = render_locals[:institutions_count] %> + +<%= tag.div id: dom_id(brex_item) do %> +
+ +
+ <%= icon "chevron-right", class: "group-open:transform group-open:rotate-90" %> + +
+ <% if brex_item.logo.attached? %> + <%= image_tag brex_item.logo, class: "rounded-full h-full w-full", loading: "lazy" %> + <% else %> +
+ <%= tag.p brex_item.name.first.upcase, class: "text-primary text-xs font-medium" %> +
+ <% end %> +
+ +
+
+ <%= tag.p brex_item.name, class: "font-medium text-primary" %> + <% if brex_item.scheduled_for_deletion? %> + <%= tag.p t(".deletion_in_progress"), class: "text-destructive text-sm animate-pulse" %> + <% end %> +
+ <% if brex_item.accounts.any? %> +

+ <%= brex_item.institution_summary %> +

+ <% end %> + <% if brex_item.syncing? %> +
+ <%= icon "loader", size: "sm", class: "animate-spin" %> + <%= tag.span t(".syncing") %> +
+ <% elsif brex_item.sync_error.present? %> +
+ <%= render DS::Tooltip.new(text: brex_item.sync_error, icon: "alert-circle", size: "sm", color: "destructive") %> + <%= tag.span t(".error"), class: "text-destructive" %> +
+ <% else %> +

+ <% if brex_item.last_synced_at %> + <% if brex_item.sync_status_summary %> + <%= t(".status_with_summary", timestamp: time_ago_in_words(brex_item.last_synced_at), summary: brex_item.sync_status_summary) %> + <% else %> + <%= t(".status", timestamp: time_ago_in_words(brex_item.last_synced_at)) %> + <% end %> + <% else %> + <%= t(".status_never") %> + <% end %> +

+ <% end %> +
+
+ + <% if Current.user&.admin? %> +
+ <% if Rails.env.development? %> + <%= icon( + "refresh-cw", + as_button: true, + href: sync_brex_item_path(brex_item) + ) %> + <% end %> + + <%= render DS::Menu.new do |menu| %> + <% menu.with_item( + variant: "button", + text: t(".delete"), + icon: "trash-2", + href: brex_item_path(brex_item), + method: :delete, + confirm: CustomConfirm.for_resource_deletion(brex_item.name, high_severity: true) + ) %> + <% end %> +
+ <% end %> + + + <% unless brex_item.scheduled_for_deletion? %> +
+ <% if brex_item.accounts.any? %> + <%= render "accounts/index/account_groups", accounts: brex_item.accounts %> + <% end %> + + <%= render ProviderSyncSummary.new( + stats: stats, + provider_item: brex_item, + institutions_count: institutions_count + ) %> + + <% if unlinked_count > 0 %> +
+

<%= t(".setup_needed") %>

+

<%= t(".setup_description", linked: linked_count, total: total_count) %>

+ <%= render DS::Link.new( + text: t(".setup_action"), + icon: "settings", + variant: "primary", + href: setup_accounts_brex_item_path(brex_item), + frame: :modal + ) %> +
+ <% elsif brex_item.accounts.empty? && total_count == 0 %> +
+

<%= t(".no_accounts_title") %>

+

<%= t(".no_accounts_description") %>

+ <%= render DS::Link.new( + text: t(".setup_action"), + icon: "settings", + variant: "primary", + href: setup_accounts_brex_item_path(brex_item), + frame: :modal + ) %> +
+ <% end %> +
+ <% end %> +
+<% end %> diff --git a/app/views/brex_items/_setup_required.html.erb b/app/views/brex_items/_setup_required.html.erb new file mode 100644 index 000000000..cce66fce2 --- /dev/null +++ b/app/views/brex_items/_setup_required.html.erb @@ -0,0 +1,34 @@ +<%= turbo_frame_tag "modal" do %> + <%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title")) %> + <% dialog.with_body do %> +
+
+ <%= icon("alert-circle", class: "text-warning w-5 h-5 shrink-0 mt-0.5") %> +
+

<%= t(".heading") %>

+

<%= t(".description") %>

+
+
+ +
+

<%= t(".setup_steps") %>

+
    +
  1. <%= t(".steps.open_settings_html") %>
    2. +
  2. <%= t(".steps.find_section_html") %>
    3. +
  3. <%= t(".steps.enter_token") %>
    4. +
  4. <%= t(".steps.return_to_link") %>
    5. +
+
+ +
+ <%= link_to settings_providers_path, + class: "inline-flex items-center justify-center rounded-lg px-4 py-2 text-sm font-medium text-inverse button-bg-primary hover:button-bg-primary-hover focus:outline-none focus:ring-2 focus:ring-primary focus:ring-offset-2 transition-colors", + data: { turbo: false } do %> + <%= t(".settings_link") %> + <% end %> +
+
+ <% end %> + <% end %> +<% end %> diff --git a/app/views/brex_items/_subtype_select.html.erb b/app/views/brex_items/_subtype_select.html.erb new file mode 100644 index 000000000..9653265a6 --- /dev/null +++ b/app/views/brex_items/_subtype_select.html.erb @@ -0,0 +1,16 @@ + diff --git a/app/views/brex_items/select_accounts.html.erb b/app/views/brex_items/select_accounts.html.erb new file mode 100644 index 000000000..fdc3e25f9 --- /dev/null +++ b/app/views/brex_items/select_accounts.html.erb @@ -0,0 +1,59 @@ +<%= turbo_frame_tag "modal" do %> + <%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title")) %> + + <% dialog.with_body do %> +
+

+ <%= t(".description", product_name: product_name) %> +

+ + <%= form_with url: link_accounts_brex_items_path, + method: :post, + data: { turbo_frame: "_top" }, + class: "space-y-4" do %> + <%= hidden_field_tag :brex_item_id, @brex_item.id %> + <%= hidden_field_tag :accountable_type, @accountable_type %> + <%= hidden_field_tag :return_to, @return_to %> + + <% account_displays = @available_accounts.map { |account| brex_account_display(account) } %> + <% has_selectable = account_displays.any? { |account_display| !account_display.blank_name? } %> + +
+ <% account_displays.each do |account_display| %> + + <% end %> +
+ +
+ <%= link_to t(".cancel"), @return_to || new_account_path, + class: "inline-flex items-center gap-1 px-3 py-2 text-sm font-medium rounded-lg text-primary button-bg-secondary hover:button-bg-secondary-hover", + data: { turbo_frame: "_top" } %> + <%= submit_tag t(".link_accounts"), + disabled: !has_selectable, + class: "inline-flex items-center gap-1 px-3 py-2 text-sm font-medium rounded-lg text-inverse bg-inverse hover:bg-inverse-hover disabled:button-bg-disabled disabled:cursor-not-allowed" %> +
+ <% end %> +
+ <% end %> + <% end %> +<% end %> diff --git a/app/views/brex_items/select_existing_account.html.erb b/app/views/brex_items/select_existing_account.html.erb new file mode 100644 index 000000000..734db5a1f --- /dev/null +++ b/app/views/brex_items/select_existing_account.html.erb @@ -0,0 +1,59 @@ +<%= turbo_frame_tag "modal" do %> + <%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title", account_name: @account.name)) %> + + <% dialog.with_body do %> +
+

+ <%= t(".description") %> +

+ + <%= form_with url: link_existing_account_brex_items_path, + method: :post, + data: { turbo_frame: "_top" }, + class: "space-y-4" do %> + <%= hidden_field_tag :brex_item_id, @brex_item.id %> + <%= hidden_field_tag :account_id, @account.id %> + <%= hidden_field_tag :return_to, @return_to %> + + <% account_displays = @available_accounts.map { |account| brex_account_display(account) } %> + <% has_selectable = account_displays.any? { |account_display| !account_display.blank_name? } %> + +
+ <% account_displays.each do |account_display| %> + + <% end %> +
+ +
+ <%= link_to t(".cancel"), @return_to || accounts_path, + class: "inline-flex items-center gap-1 px-3 py-2 text-sm font-medium rounded-lg text-primary button-bg-secondary hover:button-bg-secondary-hover", + data: { turbo_frame: "_top" } %> + <%= submit_tag t(".link_account"), + disabled: !has_selectable, + class: "inline-flex items-center gap-1 px-3 py-2 text-sm font-medium rounded-lg text-inverse bg-inverse hover:bg-inverse-hover disabled:button-bg-disabled disabled:cursor-not-allowed" %> +
+ <% end %> +
+ <% end %> + <% end %> +<% end %> diff --git a/app/views/brex_items/setup_accounts.html.erb b/app/views/brex_items/setup_accounts.html.erb new file mode 100644 index 000000000..88206fb46 --- /dev/null +++ b/app/views/brex_items/setup_accounts.html.erb @@ -0,0 +1,106 @@ +<% content_for :title, t(".title") %> + +<%= render DS::Dialog.new do |dialog| %> + <% dialog.with_header(title: t(".title")) do %> +
+ <%= icon "building-2", class: "text-primary" %> + <%= t(".subtitle") %> +
+ <% end %> + + <% dialog.with_body do %> + <%= form_with url: complete_account_setup_brex_item_path(@brex_item), + method: :post, + local: true, + data: { + controller: "loading-button", + action: "submit->loading-button#showLoading", + loading_button_loading_text_value: t(".creating_accounts"), + turbo_frame: "_top" + }, + class: "space-y-6" do |form| %> + +
+ <% if @api_error.present? %> +
+ <%= icon "alert-circle", size: "lg", class: "text-destructive" %> +

<%= t(".fetch_failed") %>

+

<%= @api_error %>

+
+ <% elsif @brex_accounts.empty? %> +
+ <%= icon "check-circle", size: "lg", class: "text-success" %> +

<%= t(".no_accounts_to_setup") %>

+

<%= t(".all_accounts_linked") %>

+
+ <% else %> +
+
+ <%= icon "info", size: "sm", class: "text-primary mt-0.5 flex-shrink-0" %> +
+

+ <%= t(".choose_account_type") %> +

+
    + <% @displayable_account_type_options.each do |label, type| %> +
  • <%= label %>
    • + <% end %> +
+
+
+
+ + <% @brex_accounts.each do |brex_account| %> +
+
+
+

+ <%= brex_account.name %> +

+
+
+ +
+
+ <%= label_tag "account_types[#{brex_account.id}]", t(".account_type_label"), + class: "block text-sm font-medium text-primary mb-2" %> + <% default_account_type = brex_account.card? ? "CreditCard" : "Depository" %> + <%= select_tag "account_types[#{brex_account.id}]", + options_for_select(@account_type_options, default_account_type), + { class: "appearance-none bg-container border border-primary rounded-md px-3 py-2 text-sm leading-6 text-primary focus:border-primary focus:ring-1 focus:ring-primary focus:outline-none w-full", + data: { + action: "change->account-type-selector#updateSubtype" + } } %> +
+ + +
+ <% @subtype_options.each do |account_type, subtype_config| %> + <%= render "brex_items/subtype_select", account_type: account_type, subtype_config: subtype_config, brex_account: brex_account %> + <% end %> +
+
+
+ <% end %> + <% end %> +
+ +
+ <%= render DS::Button.new( + text: t(".create_accounts"), + variant: "primary", + icon: "plus", + type: "submit", + class: "flex-1", + disabled: @api_error.present? || @brex_accounts.empty?, + data: { loading_button_target: "button" } + ) %> + <%= render DS::Link.new( + text: t(".cancel"), + variant: "secondary", + href: accounts_path + ) %> +
+ <% end %> + <% end %> +<% end %> diff --git a/app/views/settings/providers/_brex_panel.html.erb b/app/views/settings/providers/_brex_panel.html.erb new file mode 100644 index 000000000..3954c60f3 --- /dev/null +++ b/app/views/settings/providers/_brex_panel.html.erb @@ -0,0 +1,154 @@ +
+ <% active_items = local_assigns[:brex_items] || @brex_items || Current.family.brex_items.active.ordered %> + <% credentialed_items = active_items.select(&:credentials_configured?) %> + +
+

<%= t("brex_items.provider_panel.setup_title") %>

+
    +
  1. <%= t("brex_items.provider_panel.instructions.sign_in_html", link: link_to("Brex", "https://brex.com", target: "_blank", rel: "noopener noreferrer", class: "link")) %>
    2. +
  2. <%= t("brex_items.provider_panel.instructions.open_tokens") %>
    3. +
  3. <%= t("brex_items.provider_panel.instructions.create_token") %>
    4. +
  4. <%= t("brex_items.provider_panel.instructions.copy_token_html") %>
    5. +
+ +

+ <%= t("brex_items.provider_panel.sandbox_note_html") %> +

+
+ + <% unless BrexItem.encryption_ready? %> +
+
+ <%= icon "shield-alert", size: "sm", class: "mt-0.5 shrink-0" %> +
+

<%= t("brex_items.provider_panel.encryption_warning.title") %>

+

<%= t("brex_items.provider_panel.encryption_warning.message") %>

+
+
+
+ <% end %> + + <% error_msg = local_assigns[:error_message] || @error_message %> + <% if error_msg.present? %> +
+

<%= error_msg %>

+
+ <% end %> + + <% if active_items.any? %> +
+ <% active_items.each do |item| %> +
+ +
+
+

<%= item.name.to_s.first.to_s.upcase %>

+
+
+

<%= item.name %>

+

<%= item.sync_status_summary %>

+
+
+ + +
+
+ <%= button_to sync_brex_item_path(item), + method: :post, + class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-secondary hover:text-primary border border-secondary rounded-lg hover:border-primary", + disabled: item.syncing? do %> + <%= icon "refresh-cw", size: "sm" %> + <%= t("brex_items.provider_panel.sync") %> + <% end %> + <%= button_to brex_item_path(item), + method: :delete, + class: "inline-flex items-center gap-1 px-3 py-1.5 text-sm font-medium text-destructive hover:bg-destructive/10 rounded-lg", + aria: { label: t("brex_items.provider_panel.disconnect_label", name: item.name) }, + data: { turbo_confirm: t("brex_items.provider_panel.disconnect_confirm", name: item.name) } do %> + <%= icon "trash-2", size: "sm" %> + <% end %> +
+ + <%= styled_form_with model: item, + url: brex_item_path(item), + scope: :brex_item, + method: :patch, + data: { turbo: true }, + class: "space-y-3" do |form| %> + <%= form.text_field :name, + label: t("brex_items.provider_panel.connection_name_label"), + placeholder: t("brex_items.provider_panel.connection_name_placeholder") %> + + <%= form.text_field :token, + label: t("brex_items.provider_panel.token_label"), + placeholder: t("brex_items.provider_panel.keep_token_placeholder"), + type: :password, + value: nil %> + + <%= form.text_field :base_url, + label: t("brex_items.provider_panel.base_url_label"), + placeholder: t("brex_items.provider_panel.base_url_placeholder"), + value: item.base_url %> + +
+ <%= render DS::Link.new( + text: t("brex_items.provider_panel.setup_accounts"), + icon: "settings", + variant: "secondary", + href: setup_accounts_brex_item_path(item), + frame: :modal + ) %> + <%= form.submit t("brex_items.provider_panel.update_connection"), + class: "inline-flex items-center justify-center rounded-lg px-4 py-2 text-sm font-medium text-inverse bg-inverse hover:bg-inverse-hover focus:outline-none focus:ring-2 focus:ring-primary transition-colors" %> +
+ <% end %> +
+
+ <% end %> +
+ <% end %> + +
class="group bg-container p-4 shadow-border-xs rounded-xl"> + + <%= icon "plus" %> + <%= t("brex_items.provider_panel.add_connection") %> + + + <% brex_item = Current.family.brex_items.build(name: t("brex_items.provider_panel.default_connection_name")) %> + <%= styled_form_with model: brex_item, + url: brex_items_path, + scope: :brex_item, + method: :post, + data: { turbo: true }, + class: "space-y-3 mt-4" do |form| %> + <%= form.text_field :name, + label: t("brex_items.provider_panel.connection_name_label"), + placeholder: t("brex_items.provider_panel.connection_name_placeholder") %> + + <%= form.text_field :token, + label: t("brex_items.provider_panel.token_label"), + placeholder: t("brex_items.provider_panel.token_placeholder"), + type: :password, + value: nil %> + + <%= form.text_field :base_url, + label: t("brex_items.provider_panel.base_url_label"), + placeholder: t("brex_items.provider_panel.base_url_placeholder") %> + +
+ <%= form.submit t("brex_items.provider_panel.add_connection"), + class: "inline-flex items-center justify-center rounded-lg px-4 py-2 text-sm font-medium text-inverse bg-inverse hover:bg-inverse-hover focus:outline-none focus:ring-2 focus:ring-primary transition-colors" %> +
+ <% end %> +
+ +
+ <% if credentialed_items.any? %> +
+

<%= t("brex_items.provider_panel.configured_html", accounts_link: link_to(t("brex_items.provider_panel.accounts_link"), accounts_path, class: "link")) %>

+ <% else %> +
+

<%= t("brex_items.provider_panel.not_configured") %>

+ <% end %> +
+
diff --git a/app/views/settings/providers/_drawer_header.html.erb b/app/views/settings/providers/_drawer_header.html.erb index df439dafd..20c3eb611 100644 --- a/app/views/settings/providers/_drawer_header.html.erb +++ b/app/views/settings/providers/_drawer_header.html.erb @@ -15,8 +15,8 @@ variant: "icon", class: "ml-auto hidden lg:flex", icon: "x", - title: t("common.close"), - aria_label: t("common.close"), + title: t("defaults.common.close"), + aria_label: t("defaults.common.close"), data: { action: "DS--dialog#close" } ) %>
diff --git a/config/initializers/active_record_encryption.rb b/config/initializers/active_record_encryption.rb index 0c6da99ef..35e6b865f 100644 --- a/config/initializers/active_record_encryption.rb +++ b/config/initializers/active_record_encryption.rb @@ -1,3 +1,5 @@ +require Rails.root.join("lib/active_record_encryption_config").to_s + # Configure Active Record encryption keys # Priority order: # 1. Environment variables (works for both managed and self-hosted modes) @@ -9,8 +11,12 @@ primary_key = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"] deterministic_key = ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"] key_derivation_salt = ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"] +if ActiveRecordEncryptionConfig.partial_env? + raise ActiveRecordEncryptionConfig.partial_env_message +end + # If all environment variables are present, use them (works for both managed and self-hosted) -if primary_key.present? && deterministic_key.present? && key_derivation_salt.present? +if ActiveRecordEncryptionConfig.complete_env? Rails.application.config.active_record.encryption.primary_key = primary_key Rails.application.config.active_record.encryption.deterministic_key = deterministic_key Rails.application.config.active_record.encryption.key_derivation_salt = key_derivation_salt diff --git a/config/locales/defaults/en.yml b/config/locales/defaults/en.yml index bf860dde4..2a3b5285a 100644 --- a/config/locales/defaults/en.yml +++ b/config/locales/defaults/en.yml @@ -3,6 +3,8 @@ en: defaults: brand_name: "%{brand_name}" product_name: "%{product_name}" + common: + close: "Close" global: expand: "Expand" activerecord: diff --git a/config/locales/models/brex_item/en.yml b/config/locales/models/brex_item/en.yml new file mode 100644 index 000000000..4d8bf067b --- /dev/null +++ b/config/locales/models/brex_item/en.yml @@ -0,0 +1,14 @@ +--- +en: + activerecord: + attributes: + brex_item: + base_url: Base URL + name: Connection name + token: Token + errors: + models: + brex_item: + attributes: + base_url: + official_hosts_only: must be blank, https://api.brex.com, or https://api-staging.brex.com diff --git a/config/locales/views/brex_items/en.yml b/config/locales/views/brex_items/en.yml new file mode 100644 index 000000000..d090cb6dd --- /dev/null +++ b/config/locales/views/brex_items/en.yml @@ -0,0 +1,277 @@ +--- +en: + brex_items: + default_connection_name: Brex Connection + account_metadata: + provider: Brex + separator: " • " + kinds: + cash: Cash + card: Card + statuses: + ACTIVE: Active + active: Active + CLOSED: Closed + closed: Closed + frozen: Frozen + FROZEN: Frozen + create: + success: Brex connection created successfully + default_card_name: Brex Card + default_cash_name: "Brex Cash %{id}" + destroy: + success: Brex connection removed + index: + title: Brex Connections + institution_summary: + none: No institutions connected + one: "%{name}" + count: + one: "%{count} institution" + other: "%{count} institutions" + sync_status: + no_accounts: No accounts found + all_synced: + one: "%{count} account synced" + other: "%{count} accounts synced" + partial_setup: "%{synced} synced, %{pending} need setup" + api_error: + common_issues: "Common issues:" + expired_credentials: Generate a new API token from Brex. + expired_credentials_label: "Expired credentials:" + heading: Unable to connect to Brex + invalid_token: Check your API token in Provider Settings. + invalid_token_label: "Invalid API token:" + network: Check your internet connection. + network_label: "Network issue:" + permissions: Ensure your token has the required read-only account and transaction scopes. + permissions_label: "Insufficient permissions:" + service: Brex API may be temporarily unavailable. + service_label: "Service down:" + settings_link: Check Provider Settings + title: Brex Connection Error + errors: + unexpected_error: An unexpected error occurred. Please try again later. + entries: + default_name: Brex transaction + loading: + loading_message: Loading Brex accounts... + loading_title: Loading + link_accounts: + all_already_linked: + one: "The selected account (%{names}) is already linked" + other: "All %{count} selected accounts are already linked: %{names}" + api_error: "API error: %{message}" + invalid_account_names: + one: "Cannot link account with blank name" + other: "Cannot link %{count} accounts with blank names" + invalid_account_type: Unsupported Brex account type + link_failed: Failed to link accounts + no_accounts_selected: Please select at least one account + no_api_token: Brex API token not found. Please configure it in Provider Settings. + partial_invalid: "Successfully linked %{created_count} account(s), %{already_linked_count} account(s) were already linked, %{invalid_count} account(s) had invalid names" + partial_success: "Successfully linked %{created_count} account(s). %{already_linked_count} account(s) were already linked: %{already_linked_names}" + select_connection: Choose a Brex connection before linking accounts. + success: + one: "Successfully linked %{count} account" + other: "Successfully linked %{count} accounts" + brex_item: + accounts_need_setup: Accounts need setup + delete: Delete connection + deletion_in_progress: deletion in progress... + error: Error + no_accounts_description: This connection has no linked accounts yet. + no_accounts_title: No accounts + setup_action: Set Up New Accounts + setup_description: "%{linked} of %{total} accounts linked. Choose account types for your newly imported Brex accounts." + setup_needed: New accounts ready to set up + status: "Synced %{timestamp} ago" + status_never: Never synced + status_with_summary: "Last synced %{timestamp} ago - %{summary}" + syncing: Syncing... + total: Total + unlinked: Unlinked + provider_panel: + accounts_link: Accounts + add_connection: Add Brex connection + base_url_label: Base URL (optional) + base_url_placeholder: https://api.brex.com + configured_html: "Configured and ready to use. Visit the %{accounts_link} tab to manage and set up accounts." + connection_name_label: Connection name + connection_name_placeholder: Business checking + default_connection_name: Brex Connection + disconnect_label: "Disconnect %{name}" + disconnect_confirm: "Disconnect %{name}?" + encryption_warning: + title: Database encryption is not configured + message: Configure Active Record encryption keys before adding Brex tokens in production. Without encryption keys, Sure stores Brex provider credentials and snapshots in plaintext like other provider records. + instructions: + copy_token_html: "Copy the token and add it as a named connection below. Sure stores the token only for syncing this family." + create_token: "Create an API token with these read-only scopes: accounts.cash.readonly, accounts.card.readonly, transactions.cash.readonly, transactions.card.readonly" + open_tokens: Go to the Brex developer/API token settings for the company you want to connect + sign_in_html: "Visit %{link} and log in to the account you want to connect" + keep_token_placeholder: Leave blank to keep the current token + not_configured: Not configured + sandbox_note_html: "Use a separate named connection for each Brex company/API token you want to sync. Leave Base URL blank for production. Staging is limited to Brex-approved testing and does not work with customer tokens." + setup_accounts: Set up accounts + setup_title: "Setup instructions:" + sync: Sync + token_label: Token + token_placeholder: Paste token here + update_connection: Update connection + provider_connection: + default_description: Connect to your Brex account + default_name: Brex + description: "Connect using %{name}" + name: "Brex - %{name}" + select_accounts: + accounts_selected: accounts selected + api_error: "API error: %{message}" + cancel: Cancel + configure_name_in_brex: Cannot import - please configure account name in Brex + description: Select the accounts you want to link to your %{product_name} account. + link_accounts: Link selected accounts + no_accounts_found: No accounts found. Please check your API token configuration. + no_api_token: Brex API token not found. Please configure it in Provider Settings. + no_credentials_configured: Please configure your Brex API token first in Provider Settings. + no_name_placeholder: "(No name)" + select_connection: Choose a Brex connection in Provider Settings. + title: Select Brex Accounts + unexpected_error: An unexpected error occurred. Please try again later. + select_existing_account: + account_already_linked: This account is already linked to a provider + all_accounts_already_linked: All Brex accounts are already linked + api_error: "API error: %{message}" + cancel: Cancel + configure_name_in_brex: Cannot import - please configure account name in Brex + description: Select a Brex account to link with this account. Transactions will be synced and deduplicated automatically. + link_account: Link account + no_account_specified: No account specified + no_accounts_found: No Brex accounts found. Please check your API token configuration. + no_api_token: Brex API token not found. Please configure it in Provider Settings. + no_credentials_configured: Please configure your Brex API token first in Provider Settings. + no_name_placeholder: "(No name)" + select_connection: Choose a Brex connection in Provider Settings. + title: "Link %{account_name} with Brex" + unexpected_error: An unexpected error occurred. Please try again later. + setup_required: + description: Before you can link Brex accounts, you need to configure your Brex API token. + heading: API Token Not Configured + settings_link: Go to Provider Settings + setup_steps: "Setup steps:" + steps: + enter_token: Enter your Brex API token + find_section_html: "Find the Brex section" + open_settings_html: "Go to Settings > Providers" + return_to_link: Return here to link your accounts + title: Brex Setup Required + subtype_select: + placeholder: + subtype: Select subtype + type: Select type + link_existing_account: + account_already_linked: This account is already linked to a provider + api_error: "API error: %{message}" + invalid_account_name: Cannot link account with blank name + missing_parameters: Missing required parameters + no_account_specified: No account specified + no_api_token: Brex API token not found. Please configure it in Provider Settings. + provider_account_already_linked: This Brex account is already linked to another account + provider_account_not_found: Brex account not found + select_connection: Choose a Brex connection before linking accounts. + success: "Successfully linked %{account_name} with Brex" + setup_accounts: + account_type_label: "Account Type:" + all_accounts_linked: "All your Brex accounts have already been set up." + api_error: "API error: %{message}" + fetch_failed: "Failed to Fetch Accounts" + no_accounts_to_setup: "No Accounts to Set Up" + no_api_token: Brex API token not found. Please configure it in Provider Settings. + account_types: + skip: Skip this account + depository: Checking or Savings Account + credit_card: Credit Card + investment: Investment Account + loan: Loan or Mortgage + other_asset: Other Asset + subtype_labels: + depository: "Account Subtype:" + credit_card: "" + investment: "Investment Type:" + loan: "Loan Type:" + other_asset: "" + subtype_messages: + credit_card: "Credit cards will be automatically set up as credit card accounts." + other_asset: "No additional options needed for Other Assets." + subtypes: + depository: + checking: Checking + savings: Savings + hsa: Health Savings Account + cd: Certificate of Deposit + money_market: Money Market + investment: + brokerage: Brokerage + pension: Pension + retirement: Retirement + "401k": "401(k)" + roth_401k: "Roth 401(k)" + "403b": "403(b)" + tsp: Thrift Savings Plan + "529_plan": "529 Plan" + hsa: Health Savings Account + mutual_fund: Mutual Fund + ira: Traditional IRA + roth_ira: Roth IRA + angel: Angel + loan: + mortgage: Mortgage + student: Student Loan + auto: Auto Loan + other: Other Loan + balance: Balance + cancel: Cancel + choose_account_type: "Choose the correct account type for each Brex account:" + create_accounts: Create Accounts + creating_accounts: Creating Accounts... + historical_data_range: "Historical Data Range:" + subtitle: Choose the correct account types for your imported accounts + sync_start_date_help: Select how far back you want to sync transaction history. Maximum 3 years of history available. + sync_start_date_label: "Start syncing transactions from:" + title: Set Up Your Brex Accounts + complete_account_setup: + all_skipped: "All accounts were skipped. No accounts were created." + creation_failed: "Failed to create accounts: %{error}" + creation_failed_count: "Failed to create %{count} account(s)." + no_accounts: "No accounts to set up." + partial_skipped: "Successfully created %{created_count} account(s); %{skipped_count} account(s) were skipped." + partial_success: "Successfully created %{created_count} account(s), but %{failed_count} account(s) failed." + success: "Successfully created %{count} account(s)." + unexpected_error: An unexpected error occurred. + sync: + success: Sync started + syncer: + account_processing_failed: + one: "%{count} Brex account failed while processing." + other: "%{count} Brex accounts failed while processing." + account_sync_failed: + one: "%{count} Brex account sync could not be scheduled." + other: "%{count} Brex account syncs could not be scheduled." + accounts_need_setup: + one: "%{count} account needs setup..." + other: "%{count} accounts need setup..." + accounts_failed: + one: "%{count} Brex account failed to import." + other: "%{count} Brex accounts failed to import." + calculating_balances: Calculating balances... + checking_account_configuration: Checking account configuration... + credentials_invalid: Invalid Brex API token or account permissions + failed: Sync failed. Please try again or contact support. + import_failed: Brex import failed. + importing_accounts: Importing accounts from Brex... + processing_transactions: Processing transactions... + transactions_failed: + one: "%{count} Brex account had transaction import failures." + other: "%{count} Brex accounts had transaction import failures." + update: + success: Brex connection updated diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index 13887befc..4ac65a9f3 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -233,6 +233,7 @@ en: enable_banking: Sync European bank accounts via PSD2 open banking. coinstats: Track your entire crypto portfolio across wallets and exchanges. mercury: Sync your Mercury business banking accounts automatically. + brex: Sync Brex cash and corporate card activity with read-only access. coinbase: Import your Coinbase crypto holdings and track performance. binance: Sync your Binance spot balances using a read-only API key. kraken: Sync Kraken balances and spot trade fills using a read-only API key. diff --git a/config/locales/views/valuations/nb.yml b/config/locales/views/valuations/nb.yml index f2d311307..9564af840 100644 --- a/config/locales/views/valuations/nb.yml +++ b/config/locales/views/valuations/nb.yml @@ -27,5 +27,5 @@ nb: note_label: Notater note_placeholder: Legg til eventuelle tilleggsdetaljer om denne oppføringen overview: Oversikt - settings: Innstillinger - opening_balance: Startsaldo \ No newline at end of file + settings: Innstillinger + opening_balance: Startsaldo diff --git a/config/routes.rb b/config/routes.rb index ba00f86f4..ac61dcecb 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -33,6 +33,22 @@ Rails.application.routes.draw do end end + resources :brex_items, only: %i[index new create show edit update destroy] do + collection do + get :preload_accounts, to: "brex_items/account_flows#preload_accounts" + get :select_accounts, to: "brex_items/account_flows#select_accounts" + post :link_accounts, to: "brex_items/account_flows#link_accounts" + get :select_existing_account, to: "brex_items/account_flows#select_existing_account" + post :link_existing_account, to: "brex_items/account_flows#link_existing_account" + end + + member do + post :sync + get :setup_accounts, to: "brex_items/account_setups#setup_accounts" + post :complete_account_setup, to: "brex_items/account_setups#complete_account_setup" + end + end + resources :coinbase_items, only: [ :index, :new, :create, :show, :edit, :update, :destroy ] do collection do get :preload_accounts diff --git a/db/migrate/20260505010000_create_brex_items_and_accounts.rb b/db/migrate/20260505010000_create_brex_items_and_accounts.rb new file mode 100644 index 000000000..a76820b11 --- /dev/null +++ b/db/migrate/20260505010000_create_brex_items_and_accounts.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +class CreateBrexItemsAndAccounts < ActiveRecord::Migration[7.2] + def change + create_table :brex_items, id: :uuid do |t| + t.references :family, null: false, foreign_key: true, type: :uuid + t.string :name, null: false + + t.string :institution_id + t.string :institution_name + t.string :institution_domain + t.string :institution_url + t.string :institution_color + + t.string :status, null: false, default: "good" + t.boolean :scheduled_for_deletion, null: false, default: false + t.boolean :pending_account_setup, null: false, default: false + + t.datetime :sync_start_date + + t.jsonb :raw_payload + t.jsonb :raw_institution_payload + + t.text :token, null: false + t.string :base_url + + t.timestamps + end + + add_index :brex_items, :status + + create_table :brex_accounts, id: :uuid do |t| + t.references :brex_item, null: false, foreign_key: true, type: :uuid + + t.string :name + t.string :account_id, null: false + t.string :account_kind, null: false, default: "cash" + + t.string :currency, null: false, default: "USD" + t.decimal :current_balance, precision: 19, scale: 4 + t.decimal :available_balance, precision: 19, scale: 4 + t.decimal :account_limit, precision: 19, scale: 4 + t.string :account_status + t.string :account_type + t.string :provider + + t.jsonb :institution_metadata + t.jsonb :raw_payload + t.jsonb :raw_transactions_payload + + t.timestamps + end + + add_index :brex_accounts, + [ :brex_item_id, :account_id ], + unique: true, + name: "index_brex_accounts_on_item_and_account_id" + end +end diff --git a/db/schema.rb b/db/schema.rb index f013a0ed2..8b3cef737 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -214,6 +214,49 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_12_211200) do t.index ["status"], name: "index_binance_items_on_status" end + create_table "brex_accounts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "brex_item_id", null: false + t.string "name" + t.string "account_id", null: false + t.string "account_kind", default: "cash", null: false + t.string "currency", default: "USD", null: false + t.decimal "current_balance", precision: 19, scale: 4 + t.decimal "available_balance", precision: 19, scale: 4 + t.decimal "account_limit", precision: 19, scale: 4 + t.string "account_status" + t.string "account_type" + t.string "provider" + t.jsonb "institution_metadata" + t.jsonb "raw_payload" + t.jsonb "raw_transactions_payload" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["brex_item_id", "account_id"], name: "index_brex_accounts_on_item_and_account_id", unique: true + t.index ["brex_item_id"], name: "index_brex_accounts_on_brex_item_id" + end + + create_table "brex_items", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "family_id", null: false + t.string "name", null: false + t.string "institution_id" + t.string "institution_name" + t.string "institution_domain" + t.string "institution_url" + t.string "institution_color" + t.string "status", default: "good", null: false + t.boolean "scheduled_for_deletion", default: false, null: false + t.boolean "pending_account_setup", default: false, null: false + t.datetime "sync_start_date" + t.jsonb "raw_payload" + t.jsonb "raw_institution_payload" + t.text "token", null: false + t.string "base_url" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["family_id"], name: "index_brex_items_on_family_id" + t.index ["status"], name: "index_brex_items_on_status" + end + create_table "budget_categories", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "budget_id", null: false t.uuid "category_id", null: false @@ -1766,6 +1809,8 @@ ActiveRecord::Schema[7.2].define(version: 2026_05_12_211200) do add_foreign_key "balances", "accounts", on_delete: :cascade add_foreign_key "binance_accounts", "binance_items" add_foreign_key "binance_items", "families" + add_foreign_key "brex_accounts", "brex_items" + add_foreign_key "brex_items", "families" add_foreign_key "budget_categories", "budgets" add_foreign_key "budget_categories", "categories" add_foreign_key "budgets", "families" diff --git a/lib/active_record_encryption_config.rb b/lib/active_record_encryption_config.rb new file mode 100644 index 000000000..463976adc --- /dev/null +++ b/lib/active_record_encryption_config.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module ActiveRecordEncryptionConfig + ENV_KEYS = %w[ + ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY + ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY + ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT + ].freeze + + CONFIG_KEYS = %i[ + primary_key + deterministic_key + key_derivation_salt + ].freeze + + module_function + + def complete_env?(env = ENV) + ENV_KEYS.all? { |key| env_value_present?(env, key) } + end + + def partial_env?(env = ENV) + present_count = ENV_KEYS.count { |key| env_value_present?(env, key) } + present_count.positive? && present_count < ENV_KEYS.count + end + + def missing_env_keys(env = ENV) + ENV_KEYS.reject { |key| env_value_present?(env, key) } + end + + def partial_env_message(env = ENV) + "Active Record encryption environment variables are partially configured. Missing: #{missing_env_keys(env).join(', ')}" + end + + def credentials_configured?(credentials = Rails.application.credentials) + credentials.active_record_encryption.present? + rescue NoMethodError + false + end + + def runtime_configured?(config = Rails.application.config.active_record.encryption) + CONFIG_KEYS.all? { |key| config.public_send(key).present? } + rescue NoMethodError + false + end + + def explicitly_configured? + complete_env? || credentials_configured? + end + + def ready? + explicitly_configured? || runtime_configured? + end + + def env_value_present?(env, key) + env[key].present? + end +end diff --git a/test/controllers/api/v1/provider_connections_controller_test.rb b/test/controllers/api/v1/provider_connections_controller_test.rb index ebd637380..957b5aa31 100644 --- a/test/controllers/api/v1/provider_connections_controller_test.rb +++ b/test/controllers/api/v1/provider_connections_controller_test.rb @@ -158,6 +158,24 @@ class Api::V1::ProviderConnectionsControllerTest < ActionDispatch::IntegrationTe assert_response :success end + test "lists Brex provider connection status" do + brex_item = brex_items(:one) + + get api_v1_provider_connections_url, headers: api_headers(@api_key) + assert_response :success + + brex_connection = JSON.parse(response.body)["data"].detect do |connection| + connection["id"] == brex_item.id && connection["provider"] == "brex" + end + + assert_not_nil brex_connection + assert_equal "BrexItem", brex_connection["provider_type"] + assert_equal brex_item.name, brex_connection["name"] + assert_equal brex_item.brex_accounts.count, brex_connection["accounts"]["total_count"] + assert_equal brex_item.linked_accounts_count, brex_connection["accounts"]["linked_count"] + assert_equal brex_item.unlinked_accounts_count, brex_connection["accounts"]["unlinked_count"] + end + test "returns an empty list when no provider connections exist" do ProviderConnectionStatus.stub(:for_family, []) do get api_v1_provider_connections_url, headers: api_headers(@api_key) diff --git a/test/controllers/api/v1/usage_controller_test.rb b/test/controllers/api/v1/usage_controller_test.rb index 272358261..02f4ee8c2 100644 --- a/test/controllers/api/v1/usage_controller_test.rb +++ b/test/controllers/api/v1/usage_controller_test.rb @@ -111,26 +111,28 @@ class Api::V1::UsageControllerTest < ActionDispatch::IntegrationTest end test "should work correctly when approaching rate limit" do - # Make 98 requests to get close to the limit - 98.times do + travel_to Time.zone.local(2026, 1, 1, 12, 15, 0) do + # Make 98 requests to get close to the limit + 98.times do + get "/api/v1/test", headers: { "X-Api-Key" => @api_key.display_key } + assert_response :success + end + + # Check usage - this should be request 99 + get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key } + assert_response :success + + response_body = JSON.parse(response.body) + assert_equal 99, response_body["rate_limit"]["current_count"] + assert_equal 1, response_body["rate_limit"]["remaining"] + + # One more request should hit the limit get "/api/v1/test", headers: { "X-Api-Key" => @api_key.display_key } assert_response :success + + # Now we should be rate limited + get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key } + assert_response :too_many_requests end - - # Check usage - this should be request 99 - get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key } - assert_response :success - - response_body = JSON.parse(response.body) - assert_equal 99, response_body["rate_limit"]["current_count"] - assert_equal 1, response_body["rate_limit"]["remaining"] - - # One more request should hit the limit - get "/api/v1/test", headers: { "X-Api-Key" => @api_key.display_key } - assert_response :success - - # Now we should be rate limited - get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key } - assert_response :too_many_requests end end diff --git a/test/controllers/brex_items_controller_test.rb b/test/controllers/brex_items_controller_test.rb new file mode 100644 index 000000000..b443f2023 --- /dev/null +++ b/test/controllers/brex_items_controller_test.rb @@ -0,0 +1,488 @@ +# frozen_string_literal: true + +require "test_helper" + +class BrexItemsControllerTest < ActionDispatch::IntegrationTest + setup do + sign_in users(:family_admin) + SyncJob.stubs(:perform_later) + + @family = families(:dylan_family) + clear_brex_cache_entries + @existing_item = brex_items(:one) + @second_item = BrexItem.create!( + family: @family, + name: "Business Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + end + + teardown do + clear_brex_cache_entries + end + + test "create adds a new brex connection without overwriting existing credentials" do + existing_token = @existing_item.token + + assert_difference "BrexItem.count", 1 do + post brex_items_url, params: { + brex_item: { + name: "Joint Brex", + token: "joint_brex_token", + base_url: "https://api.brex.com" + } + } + end + + assert_redirected_to accounts_path + assert_equal existing_token, @existing_item.reload.token + assert_equal "joint_brex_token", @family.brex_items.find_by!(name: "Joint Brex").token + end + + test "create uses localized default name when submitted name is blank" do + assert_difference "BrexItem.count", 1 do + post brex_items_url, params: { + brex_item: { + name: " ", + token: "default_name_token", + base_url: "https://api.brex.com" + } + } + end + + assert_redirected_to accounts_path + assert_equal I18n.t("brex_items.default_connection_name"), @family.brex_items.order(:created_at).last.name + end + + test "update changes only the selected brex connection" do + existing_token = @existing_item.token + + patch brex_item_url(@second_item), params: { + brex_item: { + name: "Renamed Business Brex", + token: "updated_second_token", + base_url: "https://api-staging.brex.com" + } + } + + assert_redirected_to accounts_path + assert_equal existing_token, @existing_item.reload.token + assert_equal "Renamed Business Brex", @second_item.reload.name + assert_equal "updated_second_token", @second_item.token + assert_equal "https://api-staging.brex.com", @second_item.base_url + end + + test "update rejects arbitrary brex base url" do + patch brex_item_url(@second_item), params: { + brex_item: { + name: "Renamed Business Brex", + token: "updated_second_token", + base_url: "https://evil.example.test" + } + } + + assert_redirected_to settings_providers_path + assert_includes flash[:alert], "https://api.brex.com" + assert_equal "https://api.brex.com", @second_item.reload.base_url + assert_equal "second_brex_token", @second_item.token + end + + test "blank token update preserves the selected brex token" do + original_token = @second_item.token + + patch brex_item_url(@second_item), params: { + brex_item: { + name: "Renamed Business Brex", + token: "", + base_url: "https://api.brex.com" + } + } + + assert_redirected_to accounts_path + assert_equal "Renamed Business Brex", @second_item.reload.name + assert_equal original_token, @second_item.token + end + + test "update expires selected brex account cache when credentials change" do + Rails.cache.expects(:delete).with(brex_cache_key(@existing_item)).never + Rails.cache.expects(:delete).with(brex_cache_key(@second_item)).once + + patch brex_item_url(@second_item), params: { + brex_item: { + name: "Renamed Business Brex", + token: "updated_second_token", + base_url: "https://api-staging.brex.com" + } + } + + assert_redirected_to accounts_path + end + + test "update does not expire selected brex account cache for name-only changes" do + Rails.cache.expects(:delete).never + + patch brex_item_url(@second_item), params: { + brex_item: { + name: "Renamed Business Brex" + } + } + + assert_redirected_to accounts_path + assert_equal "Renamed Business Brex", @second_item.reload.name + end + + test "preload accounts uses selected brex item cache key" do + Rails.cache.expects(:read).with(brex_cache_key(@second_item)).returns(nil) + Rails.cache.expects(:write).with(brex_cache_key(@second_item), brex_accounts_payload, expires_in: 5.minutes) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: brex_accounts_payload) + Provider::Brex.expects(:new) + .with(@second_item.token, base_url: @second_item.effective_base_url) + .returns(provider) + + get preload_accounts_brex_items_url, params: { brex_item_id: @second_item.id }, as: :json + + assert_response :success + response = JSON.parse(@response.body) + assert_equal true, response["success"] + assert_equal true, response["has_accounts"] + end + + test "select accounts requires an explicit connection when multiple brex items exist" do + get select_accounts_brex_items_url, params: { accountable_type: "Depository" } + + assert_redirected_to settings_providers_path + assert_equal I18n.t("brex_items.select_accounts.select_connection"), flash[:alert] + end + + test "select accounts renders the selected brex item id" do + Rails.cache.expects(:read).with(brex_cache_key(@second_item)).returns(nil) + Rails.cache.expects(:write).with(brex_cache_key(@second_item), brex_accounts_payload, expires_in: 5.minutes) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: brex_accounts_payload) + Provider::Brex.expects(:new) + .with(@second_item.token, base_url: @second_item.effective_base_url) + .returns(provider) + + get select_accounts_brex_items_url, params: { + brex_item_id: @second_item.id, + accountable_type: "Depository" + } + + assert_response :success + assert_includes @response.body, %(name="brex_item_id") + assert_includes @response.body, %(value="#{@second_item.id}") + end + + test "select accounts rejects protocol relative return paths" do + Rails.cache.expects(:read).with(brex_cache_key(@second_item)).returns(brex_accounts_payload) + + get select_accounts_brex_items_url, params: { + brex_item_id: @second_item.id, + accountable_type: "Depository", + return_to: "//evil.example/accounts" + } + + assert_response :success + refute_includes @response.body, "//evil.example/accounts" + end + + test "select accounts rejects backslash and unsafe local return paths" do + [ + "/\\evil.example/accounts", + "/%2fevil.example/accounts", + "/%2Fevil.example/accounts", + "/%5cevil.example/accounts", + "/%5Cevil.example/accounts", + "/\naccounts", + "/ accounts", + "/" + ].each do |return_to| + Rails.cache.expects(:read).with(brex_cache_key(@second_item)).returns(brex_accounts_payload) + + get select_accounts_brex_items_url, params: { + brex_item_id: @second_item.id, + accountable_type: "Depository", + return_to: return_to + } + + assert_response :success + assert_select %(input[name="return_to"]) do |fields| + assert fields.first["value"].blank? + end + end + end + + test "select existing account rejects unsafe return paths" do + account = @family.accounts.create!( + name: "Manual Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + + [ + "//evil.example/accounts", + "\\evil.example/accounts", + "/\\evil.example/accounts", + "/%2fevil.example/accounts", + "/%2Fevil.example/accounts", + "/%5cevil.example/accounts", + "/%5Cevil.example/accounts", + "/\naccounts", + "/ accounts", + " ", + "/" + ].each do |return_to| + Rails.cache.expects(:read).with(brex_cache_key(@second_item)).returns(brex_accounts_payload) + + get select_existing_account_brex_items_url, params: { + brex_item_id: @second_item.id, + account_id: account.id, + return_to: return_to + } + + assert_response :success + assert_select %(input[name="return_to"]) do |fields| + assert fields.first["value"].blank? + end + end + end + + test "select existing account preserves safe local return path" do + account = @family.accounts.create!( + name: "Manual Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + return_to = "/accounts?tab=manual" + + Rails.cache.expects(:read).with(brex_cache_key(@second_item)).returns(brex_accounts_payload) + + get select_existing_account_brex_items_url, params: { + brex_item_id: @second_item.id, + account_id: account.id, + return_to: return_to + } + + assert_response :success + assert_select %(input[name="return_to"][value="#{return_to}"]) + end + + test "select existing account redirects when account id is invalid" do + get select_existing_account_brex_items_url, params: { + brex_item_id: @second_item.id, + account_id: SecureRandom.uuid + } + + assert_redirected_to accounts_path + assert_equal I18n.t("brex_items.select_existing_account.no_account_specified"), flash[:alert] + end + + test "select existing account renders the selected brex item id" do + account = @family.accounts.create!( + name: "Manual Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + + Rails.cache.expects(:read).with(brex_cache_key(@second_item)).returns(nil) + Rails.cache.expects(:write).with(brex_cache_key(@second_item), brex_accounts_payload, expires_in: 5.minutes) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: brex_accounts_payload) + Provider::Brex.expects(:new) + .with(@second_item.token, base_url: @second_item.effective_base_url) + .returns(provider) + + get select_existing_account_brex_items_url, params: { + brex_item_id: @second_item.id, + account_id: account.id + } + + assert_response :success + assert_includes @response.body, %(name="brex_item_id") + assert_includes @response.body, %(value="#{@second_item.id}") + end + + test "link accounts uses selected brex item and allows duplicate upstream ids across items" do + @existing_item.brex_accounts.create!( + account_id: "shared_brex_account", + name: "Shared Checking", + currency: "USD", + current_balance: 1000 + ) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: brex_accounts_payload) + Provider::Brex.expects(:new) + .with(@second_item.token, base_url: @second_item.effective_base_url) + .returns(provider) + + assert_difference -> { @second_item.brex_accounts.where(account_id: "shared_brex_account").count }, 1 do + assert_difference "AccountProvider.count", 1 do + post link_accounts_brex_items_url, params: { + brex_item_id: @second_item.id, + account_ids: [ "shared_brex_account" ], + accountable_type: "Depository" + } + end + end + + assert_redirected_to accounts_path + assert_equal 1, @existing_item.brex_accounts.where(account_id: "shared_brex_account").count + end + + test "link accounts does not silently use the first connection when multiple items exist" do + assert_no_difference "BrexAccount.count" do + assert_no_difference "Account.count" do + post link_accounts_brex_items_url, params: { + account_ids: [ "shared_brex_account" ], + accountable_type: "Depository" + } + end + end + + assert_redirected_to settings_providers_path + assert_equal I18n.t("brex_items.link_accounts.select_connection"), flash[:alert] + end + + test "link existing account does not silently use the first connection when multiple items exist" do + account = @family.accounts.create!( + name: "Manual Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + + assert_no_difference "BrexAccount.count" do + assert_no_difference "AccountProvider.count" do + post link_existing_account_brex_items_url, params: { + account_id: account.id, + brex_account_id: "shared_brex_account" + } + end + end + + assert_redirected_to settings_providers_path + assert_equal I18n.t("brex_items.link_existing_account.select_connection"), flash[:alert] + end + + test "link existing account requires account id" do + assert_no_difference "AccountProvider.count" do + post link_existing_account_brex_items_url, params: { + brex_item_id: @second_item.id, + brex_account_id: "shared_brex_account" + } + end + + assert_redirected_to accounts_path + assert_equal I18n.t("brex_items.link_existing_account.no_account_specified"), flash[:alert] + end + + test "link existing account redirects when account id is invalid" do + assert_no_difference "AccountProvider.count" do + post link_existing_account_brex_items_url, params: { + brex_item_id: @second_item.id, + account_id: SecureRandom.uuid, + brex_account_id: "shared_brex_account" + } + end + + assert_redirected_to accounts_path + assert_equal I18n.t("brex_items.link_existing_account.no_account_specified"), flash[:alert] + end + + test "sync only queues a sync for the selected brex item" do + assert_difference -> { Sync.where(syncable: @second_item).count }, 1 do + assert_no_difference -> { Sync.where(syncable: @existing_item).count } do + post sync_brex_item_url(@second_item) + end + end + + assert_response :redirect + end + + test "complete account setup ignores unsupported account type and subtype params" do + valid_brex_account = @second_item.brex_accounts.create!( + account_id: "setup_valid", + account_kind: "cash", + name: "Setup Valid", + currency: "USD", + current_balance: 100 + ) + unsupported_brex_account = @second_item.brex_accounts.create!( + account_id: "setup_unsupported", + account_kind: "cash", + name: "Setup Unsupported", + currency: "USD", + current_balance: 100 + ) + + assert_difference "AccountProvider.count", 1 do + post complete_account_setup_brex_item_url(@second_item), params: { + account_types: { + valid_brex_account.id => "Depository", + unsupported_brex_account.id => "Investment", + "not-a-brex-account" => "Depository" + }, + account_subtypes: { + valid_brex_account.id => "savings", + unsupported_brex_account.id => "brokerage", + "not-a-brex-account" => "checking" + } + } + end + + assert_redirected_to accounts_path + assert_equal "savings", valid_brex_account.reload.account.accountable.subtype + assert_nil unsupported_brex_account.reload.account_provider + assert_match(/skipped/i, flash[:notice]) + end + + test "complete account setup treats scalar setup params as empty" do + assert_no_difference "AccountProvider.count" do + post complete_account_setup_brex_item_url(@second_item), params: { + account_types: "not-a-hash", + account_subtypes: "also-not-a-hash" + } + end + + assert_redirected_to accounts_path + assert_equal I18n.t("brex_items.complete_account_setup.no_accounts"), flash[:alert] + end + + private + + def brex_accounts_payload + [ + { + id: "shared_brex_account", + name: "Shared Checking", + account_kind: "cash", + status: "active", + current_balance: { amount: 100_000, currency: "USD" }, + available_balance: { amount: 95_000, currency: "USD" } + } + ] + end + + def brex_cache_key(brex_item) + BrexItem::AccountFlow.cache_key(@family, brex_item) + end + + def clear_brex_cache_entries + return unless defined?(@family) && @family.present? + return unless Rails.cache.respond_to?(:delete_matched) + + Rails.cache.delete_matched("brex_accounts_#{@family.id}_*") + rescue NotImplementedError + # Some test cache stores do not implement delete_matched; tests that depend + # on cache state stub exact Brex cache keys instead of relying on globals. + end +end diff --git a/test/controllers/settings/providers_controller_test.rb b/test/controllers/settings/providers_controller_test.rb index f000bed7b..0f9bce42e 100644 --- a/test/controllers/settings/providers_controller_test.rb +++ b/test/controllers/settings/providers_controller_test.rb @@ -32,6 +32,27 @@ class Settings::ProvidersControllerTest < ActionDispatch::IntegrationTest end end + test "shows configured Brex connections in bank sync settings" do + get settings_providers_url + + assert_response :success + assert_includes response.body, "Brex" + assert_includes response.body, "Test Brex Connection" + assert_includes response.body, "brex-providers-panel" + end + + test "shows Brex as available when family has no Brex connections" do + sign_in users(:empty) + + get settings_providers_url + + assert_response :success + assert_includes response.body, "Brex" + assert_includes response.body, I18n.t("settings.providers.taglines.brex") + assert_includes response.body, connect_form_settings_providers_path(provider_key: "brex") + refute_includes response.body, "Test Brex Connection" + end + test "correctly identifies declared vs dynamic fields" do # All current provider fields are dynamic, but the logic should correctly # distinguish between declared and dynamic fields @@ -355,6 +376,21 @@ class Settings::ProvidersControllerTest < ActionDispatch::IntegrationTest assert_match(/Sync started/i, response.body) end + test "POST sync for brex without an active Brex sync enqueues SyncJob" do + item = brex_items(:one) + Sync.where(syncable_type: "BrexItem", syncable_id: item.id).delete_all + + assert_enqueued_jobs 1, only: SyncJob do + post sync_provider_settings_providers_path(provider_key: "brex") + end + + assert_redirected_to settings_providers_path + + follow_redirect! + assert_response :success + assert_match(/Sync started/i, response.body) + end + test "GET show includes Interactive Brokers in bank sync providers" do get settings_providers_url diff --git a/test/fixtures/brex_accounts.yml b/test/fixtures/brex_accounts.yml new file mode 100644 index 000000000..ce5214b47 --- /dev/null +++ b/test/fixtures/brex_accounts.yml @@ -0,0 +1,7 @@ +checking_account: + brex_item: one + account_id: "cash_acc_checking_1" + account_kind: cash + name: "Brex Checking" + currency: USD + current_balance: 10000.00 diff --git a/test/fixtures/brex_items.yml b/test/fixtures/brex_items.yml new file mode 100644 index 000000000..492f464df --- /dev/null +++ b/test/fixtures/brex_items.yml @@ -0,0 +1,7 @@ +one: + family: dylan_family + + name: "Test Brex Connection" + token: "test_brex_token_123" + base_url: "https://api-staging.brex.com" + status: good diff --git a/test/helpers/brex_items_helper_test.rb b/test/helpers/brex_items_helper_test.rb new file mode 100644 index 000000000..124881220 --- /dev/null +++ b/test/helpers/brex_items_helper_test.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require "test_helper" + +class BrexItemsHelperTest < ActionView::TestCase + test "metadata uses translations with titleized fallback" do + display = BrexItemsHelper::BrexAccountDisplay.new( + id: "cash_1", + name: "Operating Cash", + kind: "cash", + currency: "USD", + status: "ACTIVE", + blank_name: false + ) + + assert_equal "Brex • USD • Cash • Active", brex_account_metadata(display) + + fallback_display = BrexItemsHelper::BrexAccountDisplay.new( + id: "unknown_1", + name: "Unknown", + kind: "custom_kind", + currency: "USD", + status: "custom_status", + blank_name: false + ) + + assert_equal "Brex • USD • Custom Kind • Custom Status", brex_account_metadata(fallback_display) + end +end diff --git a/test/lib/active_record_encryption_config_test.rb b/test/lib/active_record_encryption_config_test.rb new file mode 100644 index 000000000..825d357d7 --- /dev/null +++ b/test/lib/active_record_encryption_config_test.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require "test_helper" + +class ActiveRecordEncryptionConfigTest < ActiveSupport::TestCase + test "detects complete encryption environment" do + env = { + "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY" => "primary", + "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY" => "deterministic", + "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT" => "salt" + } + + assert ActiveRecordEncryptionConfig.complete_env?(env) + refute ActiveRecordEncryptionConfig.partial_env?(env) + assert_empty ActiveRecordEncryptionConfig.missing_env_keys(env) + end + + test "detects partially configured encryption environment" do + env = { + "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY" => "primary", + "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY" => nil, + "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT" => "salt" + } + + refute ActiveRecordEncryptionConfig.complete_env?(env) + assert ActiveRecordEncryptionConfig.partial_env?(env) + assert_equal [ "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY" ], ActiveRecordEncryptionConfig.missing_env_keys(env) + assert_includes ActiveRecordEncryptionConfig.partial_env_message(env), "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY" + end + + test "does not treat absent encryption environment as partial" do + env = { + "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY" => nil, + "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY" => nil, + "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT" => nil + } + + refute ActiveRecordEncryptionConfig.complete_env?(env) + refute ActiveRecordEncryptionConfig.partial_env?(env) + end + + test "detects runtime encryption configuration" do + config = Struct.new(:primary_key, :deterministic_key, :key_derivation_salt).new("primary", "deterministic", "salt") + + assert ActiveRecordEncryptionConfig.runtime_configured?(config) + end + + test "explicit configuration excludes runtime generated config" do + ActiveRecordEncryptionConfig.stubs(:complete_env?).returns(false) + ActiveRecordEncryptionConfig.stubs(:credentials_configured?).returns(false) + ActiveRecordEncryptionConfig.stubs(:runtime_configured?).returns(true) + + refute ActiveRecordEncryptionConfig.explicitly_configured? + assert ActiveRecordEncryptionConfig.ready? + end +end diff --git a/test/models/brex_account/transactions/processor_test.rb b/test/models/brex_account/transactions/processor_test.rb new file mode 100644 index 000000000..74c4a2a3a --- /dev/null +++ b/test/models/brex_account/transactions/processor_test.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require "test_helper" + +class BrexAccount::Transactions::ProcessorTest < ActiveSupport::TestCase + setup do + @brex_item = brex_items(:one) + @brex_account = @brex_item.brex_accounts.create!( + account_id: "cash_unlinked", + account_kind: "cash", + name: "Unlinked Cash", + currency: "USD", + raw_transactions_payload: [ + { + id: "tx_skipped", + amount: { amount: 1_00, currency: "USD" }, + description: "Skipped transaction", + posted_at_date: "2026-01-02" + } + ] + ) + end + + test "counts intentionally skipped transactions separately from failures" do + result = BrexAccount::Transactions::Processor.new(@brex_account).process + + assert result[:success] + assert_equal 1, result[:total] + assert_equal 0, result[:imported] + assert_equal 1, result[:skipped] + assert_equal 0, result[:failed] + assert_equal "No linked account", result[:skipped_transactions].first[:reason] + assert_empty result[:errors] + end + + test "imports linked transactions successfully" do + link_brex_account! + + result = BrexAccount::Transactions::Processor.new(@brex_account).process + + assert result[:success] + assert_equal 1, result[:total] + assert_equal 1, result[:imported] + assert_equal 0, result[:skipped] + assert_equal 0, result[:failed] + assert_empty result[:skipped_transactions] + assert_empty result[:errors] + end + + test "aggregates partial transaction failures" do + link_brex_account! + @brex_account.update!( + raw_transactions_payload: [ + { + id: "tx_success", + amount: { amount: 1_00, currency: "USD" }, + description: "Successful transaction", + posted_at_date: "2026-01-02" + }, + { + id: "tx_failure", + amount: { amount: 2_00, currency: "USD" }, + description: "Failed transaction", + posted_at_date: "not-a-date" + } + ] + ) + + result = BrexAccount::Transactions::Processor.new(@brex_account).process + + assert_not result[:success] + assert_equal 2, result[:total] + assert_equal 1, result[:imported] + assert_equal 0, result[:skipped] + assert_equal 1, result[:failed] + assert_empty result[:skipped_transactions] + assert_equal "tx_failure", result[:errors].first[:transaction_id] + assert_match(/Unable to parse transaction date/, result[:errors].first[:error]) + end + + private + + def link_brex_account! + account = @brex_item.family.accounts.create!( + name: "Linked Cash", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + AccountProvider.create!(account: account, provider: @brex_account) + end +end diff --git a/test/models/brex_account_test.rb b/test/models/brex_account_test.rb new file mode 100644 index 000000000..6d0d60769 --- /dev/null +++ b/test/models/brex_account_test.rb @@ -0,0 +1,210 @@ +require "test_helper" + +class BrexAccountTest < ActiveSupport::TestCase + setup do + @family_a = families(:dylan_family) + @family_b = families(:empty) + + @item_a = BrexItem.create!( + family: @family_a, + name: "Family A Brex", + token: "token_a", + base_url: "https://api-staging.brex.com", + status: "good" + ) + + @item_b = BrexItem.create!( + family: @family_b, + name: "Family B Brex", + token: "token_b", + base_url: "https://api-staging.brex.com", + status: "good" + ) + end + + test "same account_id can be linked under different brex_items" do + BrexAccount.create!( + brex_item: @item_a, + account_id: "shared_brex_acc_1", + name: "Checking", + currency: "USD", + current_balance: 5000 + ) + + # A second family connecting the same Brex account must succeed and produce + # an independent ledger (separate BrexAccount row, separate Account). + assert_difference "BrexAccount.count", 1 do + BrexAccount.create!( + brex_item: @item_b, + account_id: "shared_brex_acc_1", + name: "Checking", + currency: "USD", + current_balance: 5000 + ) + end + end + + test "declares raw Brex payloads as encrypted" do + skip "Encryption not configured" unless BrexAccount.encryption_ready? + + encrypted_attributes = BrexAccount.encrypted_attributes.map(&:to_s) + + assert_includes encrypted_attributes, "raw_payload" + assert_includes encrypted_attributes, "raw_transactions_payload" + end + + test "same account_id can be linked under different brex_items in the same family" do + item_a_2 = BrexItem.create!( + family: @family_a, + name: "Family A Second Brex", + token: "token_a_2", + base_url: "https://api-staging.brex.com", + status: "good" + ) + + BrexAccount.create!( + brex_item: @item_a, + account_id: "shared_brex_acc_1", + name: "Checking", + currency: "USD", + current_balance: 5000 + ) + + assert_difference "BrexAccount.count", 1 do + BrexAccount.create!( + brex_item: item_a_2, + account_id: "shared_brex_acc_1", + name: "Checking", + currency: "USD", + current_balance: 5000 + ) + end + end + + test "same account_id cannot appear twice under the same brex_item" do + BrexAccount.create!( + brex_item: @item_a, + account_id: "duplicate_acc", + name: "Checking", + currency: "USD", + current_balance: 1000 + ) + + duplicate = BrexAccount.new( + brex_item: @item_a, + account_id: "duplicate_acc", + name: "Checking", + currency: "USD", + current_balance: 1000 + ) + refute duplicate.valid? + assert_includes duplicate.errors[:account_id], "has already been taken" + + assert_raises(ActiveRecord::RecordInvalid) do + BrexAccount.create!( + brex_item: @item_a, + account_id: "duplicate_acc", + name: "Checking", + currency: "USD", + current_balance: 1000 + ) + end + end + + test "minor-unit money converts to decimal account balances" do + brex_account = @item_a.brex_accounts.create!( + account_id: "cash_1", + name: "Operating", + currency: "USD", + account_kind: "cash" + ) + + brex_account.upsert_brex_snapshot!( + { + id: "cash_1", + name: "Operating", + account_kind: "cash", + current_balance: { amount: 123_456, currency: "USD" }, + available_balance: { amount: 120_000, currency: "USD" } + } + ) + + assert_equal BigDecimal("1234.56"), brex_account.current_balance + assert_equal BigDecimal("1200.0"), brex_account.available_balance + end + + test "invalid Brex money amount falls back to zero" do + assert_equal BigDecimal("0"), BrexAccount.money_to_decimal(amount: "not-a-number", currency: "USD") + end + + test "snapshot sanitizes full account and routing numbers" do + brex_account = @item_a.brex_accounts.create!( + account_id: "cash_2", + name: "Operating", + currency: "USD", + account_kind: "cash" + ) + + brex_account.upsert_brex_snapshot!( + { + id: "cash_2", + name: "Operating", + account_kind: "cash", + current_balance: { amount: 100, currency: "USD" }, + account_number: "account-last4-9012", + routing_number: "routing-last4-0021", + token: "test-token-placeholder" + } + ) + + payload = brex_account.raw_payload + refute_includes payload.values.compact.map(&:to_s).join(" "), "account-last4-9012" + refute_includes payload.values.compact.map(&:to_s).join(" "), "routing-last4-0021" + assert_equal "9012", payload["account_number_last4"] + assert_equal "0021", payload["routing_number_last4"] + assert_equal "[FILTERED]", payload["token"] + end + + test "transaction payload sanitizer drops arbitrary card metadata" do + sanitized = BrexAccount.sanitize_payload( + { + id: "tx_1", + card_metadata: { + card_id: "card_1", + pan: "test-pan-placeholder", + private_note: "private", + last_four: "card ending 1111" + } + } + ) + + assert_equal({ "card_id" => "card_1", "last_four" => "1111" }, sanitized["card_metadata"]) + refute_includes sanitized.to_s, "test-pan-placeholder" + refute_includes sanitized.to_s, "private" + end + + test "transaction payload sanitizer limits card metadata last four to digits" do + sanitized = BrexAccount.sanitize_payload(card_metadata: { card_last_four: "card id abc9876" }) + + assert_equal "9876", sanitized["card_metadata"]["last_four"] + refute_includes sanitized.to_s, "abc9876" + end + + test "linked_account uses the cached account association" do + brex_account = @item_a.brex_accounts.create!( + account_id: "cash_linked_alias", + name: "Linked Alias", + currency: "USD", + account_kind: "cash" + ) + account = @family_a.accounts.create!( + name: "Linked Alias", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + AccountProvider.create!(account: account, provider: brex_account) + + assert_equal brex_account.account, brex_account.linked_account + end +end diff --git a/test/models/brex_entry/processor_test.rb b/test/models/brex_entry/processor_test.rb new file mode 100644 index 000000000..aa36765dc --- /dev/null +++ b/test/models/brex_entry/processor_test.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +require "test_helper" + +class BrexEntry::ProcessorTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @brex_item = brex_items(:one) + @account = @family.accounts.create!( + name: "Brex Card", + balance: 0, + currency: "USD", + accountable: CreditCard.new + ) + @brex_account = @brex_item.brex_accounts.create!( + account_id: BrexAccount.card_account_id, + account_kind: "card", + name: "Brex Card", + currency: "USD", + current_balance: 0 + ) + AccountProvider.create!(account: @account, provider: @brex_account) + end + + test "imports card purchase with Brex signed amount preserved" do + entry = BrexEntry::Processor.new(card_transaction(amount: 12_34), brex_account: @brex_account).process + + assert_equal BigDecimal("12.34"), entry.amount + assert_equal "USD", entry.currency + assert_equal "brex", entry.source + assert_equal Date.new(2026, 1, 2), entry.date + assert_equal "STAPLES", entry.transaction.merchant.name + assert_equal "card_1", entry.transaction.extra.dig("brex", "card_id") + assert_equal "STAPLES", entry.transaction.extra.dig("brex", "merchant", "raw_descriptor") + refute_includes entry.transaction.extra.dig("brex", "merchant").to_s, "test-pan-placeholder" + refute_includes entry.transaction.extra.dig("brex", "merchant").to_s, "pan" + end + + test "imports card payment as negative amount" do + entry = BrexEntry::Processor.new(card_transaction(id: "tx_payment", amount: -50_00, type: "COLLECTION"), brex_account: @brex_account).process + + assert_equal BigDecimal("-50.0"), entry.amount + assert_equal "cc_payment", entry.transaction.kind + end + + test "is idempotent by external id and source" do + transaction = card_transaction(id: "tx_duplicate", amount: 12_34) + + assert_difference -> { @account.entries.where(source: "brex", external_id: "brex_tx_duplicate").count }, 1 do + BrexEntry::Processor.new(transaction, brex_account: @brex_account).process + BrexEntry::Processor.new(transaction, brex_account: @brex_account).process + end + end + + test "tolerates nullable Brex fields and unknown types" do + transaction = { + id: "tx_nullable", + amount: nil, + description: "Cash movement", + posted_at_date: "2026-01-03", + initiated_at_date: "2026-01-02", + type: "NEW_BREX_TYPE" + } + + entry = BrexEntry::Processor.new(transaction, brex_account: @brex_account).process + + assert_equal BigDecimal("0"), entry.amount + assert_equal "Cash movement", entry.name + assert_equal "NEW_BREX_TYPE", entry.transaction.extra.dig("brex", "type") + end + + test "uses localized default transaction name" do + transaction = card_transaction(id: "tx_default_name", amount: 12_34) + transaction.delete(:description) + transaction.delete(:merchant) + + entry = BrexEntry::Processor.new(transaction, brex_account: @brex_account).process + + assert_equal I18n.t("brex_items.entries.default_name"), entry.name + end + + test "logs validation failure without re-reading missing external id" do + Rails.logger.expects(:error).with(regexp_matches(/Validation error for transaction brex_unknown/)).once + + assert_raises(ArgumentError) do + BrexEntry::Processor.new(card_transaction(id: nil, amount: 12_34), brex_account: @brex_account).process + end + end + + test "logs save failure with cached external id" do + Account::ProviderImportAdapter.any_instance + .expects(:import_transaction) + .raises(ActiveRecord::RecordInvalid.new(Entry.new)) + Rails.logger.expects(:error).with(regexp_matches(/Failed to save transaction brex_tx_save_failure/)).once + + assert_raises(StandardError) do + BrexEntry::Processor.new(card_transaction(id: "tx_save_failure", amount: 12_34), brex_account: @brex_account).process + end + end + + test "logs missing transaction currency before using account fallback" do + Rails.logger.expects(:warn).with(regexp_matches(/Invalid Brex currency nil for transaction tx_missing_currency/)).once + + entry = BrexEntry::Processor.new( + card_transaction(id: "tx_missing_currency", amount: 12_34).tap { |transaction| transaction[:amount].delete(:currency) }, + brex_account: @brex_account + ).process + + assert_equal "USD", entry.currency + end + + private + + def card_transaction(id: "tx_1", amount:, type: "CARD_EXPENSE") + { + id: id, + amount: { amount: amount, currency: "USD" }, + description: "Office supplies", + posted_at_date: "2026-01-02", + initiated_at_date: "2026-01-01", + type: type, + card_id: "card_1", + merchant: { + raw_descriptor: "STAPLES", + card_metadata: { + pan: "test-pan-placeholder" + } + } + } + end +end diff --git a/test/models/brex_item/account_flow_test.rb b/test/models/brex_item/account_flow_test.rb new file mode 100644 index 000000000..60a2207dd --- /dev/null +++ b/test/models/brex_item/account_flow_test.rb @@ -0,0 +1,394 @@ +# frozen_string_literal: true + +require "test_helper" + +class BrexItem::AccountFlowTest < ActiveSupport::TestCase + setup do + SyncJob.stubs(:perform_later) + @family = families(:dylan_family) + @brex_item = brex_items(:one) + end + + test "requires explicit item when multiple credentialed connections exist" do + BrexItem.create!( + family: @family, + name: "Second Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + + flow = BrexItem::AccountFlow.new(family: @family) + + assert_not flow.selected? + assert flow.selection_required? + end + + test "preload payload returns explicit selection error when multiple connections exist" do + BrexItem.create!( + family: @family, + name: "Second Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + + payload = BrexItem::AccountFlow.new(family: @family).preload_payload + + assert_equal false, payload[:success] + assert_equal "select_connection", payload[:error] + assert_nil payload[:has_accounts] + end + + test "preload payload treats cached empty accounts as a cache hit" do + cache_key = BrexItem::AccountFlow.cache_key(@family, @brex_item) + Rails.cache.expects(:read).with(cache_key).returns([]) + Rails.cache.expects(:write).never + @brex_item.expects(:brex_provider).never + + payload = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item).preload_payload + + assert payload[:success] + assert_equal false, payload[:has_accounts] + assert_equal true, payload[:cached] + end + + test "account cache keys isolate multiple credentialed connections with shared upstream ids" do + second_item = BrexItem.create!( + family: @family, + name: "Second Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + first_cache_key = BrexItem::AccountFlow.cache_key(@family, @brex_item) + second_cache_key = BrexItem::AccountFlow.cache_key(@family, second_item) + + refute_equal first_cache_key, second_cache_key + + Rails.cache.expects(:read).with(first_cache_key).never + Rails.cache.expects(:read).with(second_cache_key).returns( + [ { id: BrexAccount.card_account_id, name: "Second Brex Card", account_kind: "card" } ] + ) + Rails.cache.expects(:write).never + + result = BrexItem::AccountFlow.new(family: @family, brex_item: second_item).select_accounts_result(accountable_type: "CreditCard") + + assert result.success? + assert_equal [ "Second Brex Card" ], result.available_accounts.map { |account| account.with_indifferent_access[:name] } + end + + test "preload payload reports invalid explicit connection as selection error" do + payload = BrexItem::AccountFlow.new( + family: @family, + brex_item_id: " #{SecureRandom.uuid} " + ).preload_payload + + assert_equal false, payload[:success] + assert_equal "select_connection", payload[:error] + assert_nil payload[:has_accounts] + end + + test "import accounts reports missing selected item as no api token" do + flow = BrexItem::AccountFlow.new(family: @family, brex_item_id: SecureRandom.uuid) + + assert_raises BrexItem::AccountFlow::NoApiTokenError do + flow.import_accounts_from_api_if_needed + end + end + + test "link result returns navigation instead of raising expected selection errors" do + BrexItem.create!( + family: @family, + name: "Second Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + + result = BrexItem::AccountFlow.new(family: @family).link_new_accounts_result( + account_ids: [ "cash_import_1" ], + accountable_type: "Depository" + ) + + assert_equal :settings_providers, result.target + assert_equal :alert, result.flash_type + assert_equal I18n.t("brex_items.link_accounts.select_connection"), result.message + end + + test "link new accounts rejects unsupported account type before creating accounts" do + flow = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item) + @brex_item.expects(:brex_provider).never + + assert_no_difference [ "Account.count", "BrexAccount.count", "AccountProvider.count" ] do + result = flow.link_new_accounts_result( + account_ids: [ "cash_import_1" ], + accountable_type: "Investment" + ) + + assert_equal :new_account, result.target + assert_equal :alert, result.flash_type + assert_equal I18n.t("brex_items.link_accounts.invalid_account_type"), result.message + end + end + + test "link new accounts converts unexpected errors into navigation alerts" do + flow = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item) + flow.expects(:link_new_accounts!).raises(StandardError, "link failure") + + result = flow.link_new_accounts_result( + account_ids: [ "cash_import_1" ], + accountable_type: "Depository" + ) + + assert_equal :new_account, result.target + assert_equal :alert, result.flash_type + assert_equal I18n.t("brex_items.errors.unexpected_error"), result.message + end + + test "link existing account converts unexpected errors into navigation alerts" do + account = @family.accounts.create!( + name: "Manual Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + flow = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item) + flow.expects(:link_existing_account!).raises(StandardError, "link existing failure") + + result = flow.link_existing_account_result(account: account, brex_account_id: "cash_import_1") + + assert_equal :accounts, result.target + assert_equal :alert, result.flash_type + assert_equal I18n.t("brex_items.errors.unexpected_error"), result.message + end + + test "imports provider accounts into the selected item" do + brex_item = BrexItem.create!( + family: @family, + name: "Import Brex", + token: "import_brex_token", + base_url: "https://api.brex.com" + ) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns( + accounts: [ + { + id: "cash_import_1", + name: "Imported Cash", + account_kind: "cash", + current_balance: { amount: 12_345, currency: "USD" }, + account_number: "account-last4-3456" + } + ] + ) + brex_item.expects(:brex_provider).returns(provider) + + flow = BrexItem::AccountFlow.new(family: @family, brex_item: brex_item) + + assert_difference -> { brex_item.brex_accounts.count }, 1 do + assert_nil flow.import_accounts_from_api_if_needed + end + + brex_account = brex_item.brex_accounts.find_by!(account_id: "cash_import_1") + assert_equal "Imported Cash", brex_account.name + assert_equal "3456", brex_account.raw_payload["account_number_last4"] + refute_includes brex_account.raw_payload.to_s, "account-last4-3456" + end + + test "refreshes existing provider accounts during setup discovery" do + brex_item = BrexItem.create!( + family: @family, + name: "Refresh Brex", + token: "refresh_brex_token", + base_url: "https://api.brex.com" + ) + brex_item.brex_accounts.create!( + account_id: "cash_import_1", + name: "Old Cash", + currency: "USD", + account_kind: "cash", + current_balance: 1 + ) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns( + accounts: [ + { + id: "cash_import_1", + name: "Updated Cash", + account_kind: "cash", + current_balance: { amount: 12_345, currency: "USD" } + } + ] + ) + brex_item.expects(:brex_provider).returns(provider) + + flow = BrexItem::AccountFlow.new(family: @family, brex_item: brex_item) + + assert_no_difference -> { brex_item.brex_accounts.count } do + assert_nil flow.import_accounts_from_api_if_needed + end + + brex_account = brex_item.brex_accounts.find_by!(account_id: "cash_import_1") + assert_equal "Updated Cash", brex_account.name + assert_equal BigDecimal("123.45"), brex_account.current_balance + end + + test "complete setup result is unsuccessful when any account creation fails" do + first_brex_account = @brex_item.brex_accounts.create!( + account_id: "setup_result_partial_1", + account_kind: "cash", + name: "Setup Result Partial One", + currency: "USD", + current_balance: 100 + ) + second_brex_account = @brex_item.brex_accounts.create!( + account_id: "setup_result_partial_2", + account_kind: "cash", + name: "Setup Result Partial Two", + currency: "USD", + current_balance: 100 + ) + second_brex_account.update_column(:name, nil) + + result = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item).complete_setup_result( + account_types: { + first_brex_account.id => "Depository", + second_brex_account.id => "Depository" + }, + account_subtypes: {} + ) + + refute result.success? + assert_match(/failed/i, result.message) + assert first_brex_account.reload.account_provider.present? + assert_nil second_brex_account.reload.account_provider + end + + test "complete setup creates account links with default subtype" do + brex_account = @brex_item.brex_accounts.create!( + account_id: "setup_cash_1", + account_kind: "cash", + name: "Setup Cash", + currency: "USD", + current_balance: 100 + ) + + flow = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item) + + assert_difference "AccountProvider.count", 1 do + result = flow.complete_setup!( + account_types: { brex_account.id => "Depository" }, + account_subtypes: {} + ) + + assert_equal 1, result.created_count + assert_equal 0, result.skipped_count + end + + account = brex_account.reload.account + assert_equal "Setup Cash", account.name + assert_equal Depository::DEFAULT_SUBTYPE, account.accountable.subtype + end + + test "complete setup keeps prior accounts when one account creation fails" do + first_brex_account = @brex_item.brex_accounts.create!( + account_id: "setup_partial_1", + account_kind: "cash", + name: "Setup Partial One", + currency: "USD", + current_balance: 100 + ) + second_brex_account = @brex_item.brex_accounts.create!( + account_id: "setup_partial_2", + account_kind: "cash", + name: "Setup Partial Two", + currency: "USD", + current_balance: 100 + ) + second_brex_account.update_column(:name, nil) + + result = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item).complete_setup!( + account_types: { + first_brex_account.id => "Depository", + second_brex_account.id => "Depository" + }, + account_subtypes: {} + ) + + assert_equal 1, result.created_count + assert_equal 1, result.failed_count + assert first_brex_account.reload.account_provider.present? + assert_nil second_brex_account.reload.account_provider + end + + test "link new accounts rolls back account creation when provider link fails" do + provider = mock("brex_provider") + provider.expects(:get_accounts).returns( + accounts: [ + { + id: "rollback_cash_1", + name: "Rollback Cash", + account_kind: "cash", + current_balance: { amount: 12_345, currency: "USD" } + } + ] + ) + @brex_item.expects(:brex_provider).returns(provider) + AccountProvider.expects(:create!).raises(ActiveRecord::RecordInvalid.new(AccountProvider.new)) + + flow = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item) + + assert_no_difference [ "Account.count", "BrexAccount.count", "AccountProvider.count" ] do + assert_raises(ActiveRecord::RecordInvalid) do + flow.link_new_accounts!(account_ids: [ "rollback_cash_1" ], accountable_type: "Depository") + end + end + end + + test "link existing account rolls back provider account when link creation fails" do + account = @family.accounts.create!( + name: "Existing Cash", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + provider = mock("brex_provider") + provider.expects(:get_accounts).returns( + accounts: [ + { + id: "rollback_existing_cash_1", + name: "Rollback Existing Cash", + account_kind: "cash", + current_balance: { amount: 12_345, currency: "USD" } + } + ] + ) + @brex_item.expects(:brex_provider).returns(provider) + AccountProvider.expects(:create!).raises(ActiveRecord::RecordInvalid.new(AccountProvider.new)) + + flow = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item) + + assert_no_difference [ "BrexAccount.count", "AccountProvider.count" ] do + assert_raises(ActiveRecord::RecordInvalid) do + flow.link_existing_account!(account: account, brex_account_id: "rollback_existing_cash_1") + end + end + end + + test "complete setup result returns localized notice" do + brex_account = @brex_item.brex_accounts.create!( + account_id: "setup_result_cash_1", + account_kind: "cash", + name: "Setup Result Cash", + currency: "USD", + current_balance: 100 + ) + + result = BrexItem::AccountFlow.new(family: @family, brex_item: @brex_item).complete_setup_result( + account_types: { brex_account.id => "Depository" }, + account_subtypes: {} + ) + + assert result.success? + assert_equal I18n.t("brex_items.complete_account_setup.success", count: 1), result.message + end +end diff --git a/test/models/brex_item/importer_test.rb b/test/models/brex_item/importer_test.rb new file mode 100644 index 000000000..ed64f5e81 --- /dev/null +++ b/test/models/brex_item/importer_test.rb @@ -0,0 +1,331 @@ +# frozen_string_literal: true + +require "test_helper" + +class BrexItem::ImporterTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @brex_item = brex_items(:one) + @account = @family.accounts.create!( + name: "Operating Cash", + balance: 0, + currency: "USD", + accountable: Depository.new(subtype: "checking") + ) + @brex_account = @brex_item.brex_accounts.create!( + account_id: "cash_1", + account_kind: "cash", + name: "Operating Cash", + currency: "USD", + current_balance: 0 + ) + AccountProvider.create!(account: @account, provider: @brex_account) + end + + test "imports account discovery and fetches transactions only for linked accounts" do + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: [ cash_account_payload, card_account_payload ]) + provider.expects(:get_cash_transactions).with("cash_1", start_date: Date.new(2026, 1, 1)).returns( + transactions: [ + { + id: "cash_tx_1", + amount: { amount: 12_34, currency: "USD" }, + description: "Wire fee", + posted_at_date: "2026-01-02" + } + ] + ) + provider.expects(:get_primary_card_transactions).never + + result = BrexItem::Importer.new(@brex_item, brex_provider: provider, sync_start_date: Date.new(2026, 1, 1)).import + + assert result[:success] + assert_equal 1, result[:accounts_updated] + assert_equal 1, result[:accounts_created] + assert_equal [ "cash_tx_1" ], @brex_account.reload.raw_transactions_payload.map { |tx| tx["id"] } + assert_equal "card", @brex_item.brex_accounts.find_by!(account_id: BrexAccount.card_account_id).account_kind + end + + test "counts only newly stored transactions as imported" do + @brex_account.update!( + raw_transactions_payload: [ + { + id: "cash_tx_1", + amount: { amount: 12_34, currency: "USD" }, + description: "Existing wire fee", + posted_at_date: "2026-01-02" + } + ] + ) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: [ cash_account_payload ]) + provider.expects(:get_cash_transactions).with("cash_1", start_date: anything).returns( + transactions: [ + { + id: "cash_tx_1", + amount: { amount: 12_34, currency: "USD" }, + description: "Existing wire fee", + posted_at_date: "2026-01-02" + }, + { + id: "cash_tx_2", + amount: { amount: 56_78, currency: "USD" }, + description: "New wire fee", + posted_at_date: "2026-01-03" + } + ] + ) + + result = BrexItem::Importer.new(@brex_item, brex_provider: provider, sync_start_date: Date.new(2026, 1, 1)).import + + assert result[:success] + assert_equal 1, result[:transactions_imported] + assert_equal [ "cash_tx_1", "cash_tx_2" ], @brex_account.reload.raw_transactions_payload.map { |tx| tx["id"] } + end + + test "keeps raw transaction snapshots bounded to the sync window" do + @brex_account.update!( + raw_transactions_payload: [ + { + id: "old_cash_tx", + amount: { amount: 12_34, currency: "USD" }, + description: "Old wire fee", + posted_at_date: "2025-12-01" + }, + { + id: "recent_cash_tx", + amount: { amount: 56_78, currency: "USD" }, + description: "Recent wire fee", + posted_at_date: "2026-01-02" + } + ] + ) + + sync_start_date = Date.new(2026, 1, 1) + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: [ cash_account_payload ]) + provider.expects(:get_cash_transactions).with("cash_1", start_date: sync_start_date).returns( + transactions: [ + { + id: "ignored_before_window", + amount: { amount: 1_00, currency: "USD" }, + description: "Ignored old transaction", + posted_at_date: "2025-12-31" + }, + { + id: "new_cash_tx", + amount: { amount: 2_00, currency: "USD" }, + description: "New transaction", + posted_at_date: "2026-01-03" + } + ] + ) + + result = BrexItem::Importer.new(@brex_item, brex_provider: provider, sync_start_date: sync_start_date).import + + assert result[:success] + assert_equal 1, result[:transactions_imported] + assert_equal [ "recent_cash_tx", "new_cash_tx" ], @brex_account.reload.raw_transactions_payload.map { |tx| tx["id"] } + end + + test "uses explicit sync start date for cash and card transaction fetches" do + card_account = @family.accounts.create!( + name: "Brex Card", + balance: 0, + currency: "USD", + accountable: CreditCard.new + ) + brex_card_account = @brex_item.brex_accounts.create!( + account_id: BrexAccount.card_account_id, + account_kind: "card", + name: "Brex Card", + currency: "USD", + current_balance: 0 + ) + AccountProvider.create!(account: card_account, provider: brex_card_account) + + sync_start_date = Date.new(2026, 2, 1) + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: [ cash_account_payload, card_account_payload ]) + provider.expects(:get_cash_transactions).with("cash_1", start_date: sync_start_date).returns(transactions: []) + provider.expects(:get_primary_card_transactions).with(start_date: sync_start_date).returns(transactions: []) + + result = BrexItem::Importer.new( + @brex_item, + brex_provider: provider, + sync_start_date: sync_start_date + ).import + + assert result[:success] + end + + test "imports aggregate card transactions only into the selected connection" do + first_card_account = @family.accounts.create!( + name: "First Brex Card", + balance: 0, + currency: "USD", + accountable: CreditCard.new + ) + first_brex_card_account = @brex_item.brex_accounts.create!( + account_id: BrexAccount.card_account_id, + account_kind: "card", + name: "First Brex Card", + currency: "USD", + current_balance: 0 + ) + AccountProvider.create!(account: first_card_account, provider: first_brex_card_account) + + second_item = BrexItem.create!( + family: @family, + name: "Second Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + second_card_account = @family.accounts.create!( + name: "Second Brex Card", + balance: 0, + currency: "USD", + accountable: CreditCard.new + ) + second_brex_card_account = second_item.brex_accounts.create!( + account_id: BrexAccount.card_account_id, + account_kind: "card", + name: "Second Brex Card", + currency: "USD", + current_balance: 0, + raw_transactions_payload: [ + { + id: "second_connection_card_tx", + amount: { amount: 42_00, currency: "USD" }, + description: "Existing second connection card transaction", + posted_at_date: "2026-02-01" + } + ] + ) + AccountProvider.create!(account: second_card_account, provider: second_brex_card_account) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: [ cash_account_payload, card_account_payload ]) + provider.expects(:get_cash_transactions).with("cash_1", start_date: anything).returns(transactions: []) + provider.expects(:get_primary_card_transactions).with(start_date: anything).returns( + transactions: [ + { + id: "first_connection_card_tx", + amount: { amount: 21_00, currency: "USD" }, + description: "First connection card transaction", + posted_at_date: "2026-02-02", + card_id: "card_account_1" + } + ] + ) + + result = BrexItem::Importer.new(@brex_item, brex_provider: provider, sync_start_date: Date.new(2026, 2, 1)).import + + assert result[:success] + assert_equal [ "first_connection_card_tx" ], first_brex_card_account.reload.raw_transactions_payload.map { |tx| tx["id"] } + assert_equal [ "second_connection_card_tx" ], second_brex_card_account.reload.raw_transactions_payload.map { |tx| tx["id"] } + end + + test "raises and reports snapshot persistence failures" do + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: [ cash_account_payload ]) + @brex_item.expects(:upsert_brex_snapshot!).raises(StandardError.new("snapshot failed")) + + error = assert_raises StandardError do + BrexItem::Importer.new(@brex_item, brex_provider: provider).import + end + + assert_equal "snapshot failed", error.message + end + + test "marks item as requiring update on authorization errors" do + provider = mock("brex_provider") + provider.expects(:get_accounts).raises( + Provider::Brex::BrexError.new("Access forbidden", :access_forbidden, http_status: 403, trace_id: "trace_123") + ) + + result = BrexItem::Importer.new(@brex_item, brex_provider: provider).import + + refute result[:success] + assert @brex_item.reload.requires_update? + end + + test "clears requires update after a clean import" do + @brex_item.update!(status: :requires_update) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns(accounts: [ cash_account_payload ]) + provider.expects(:get_cash_transactions).with("cash_1", start_date: anything).returns(transactions: []) + + result = BrexItem::Importer.new(@brex_item, brex_provider: provider).import + + assert result[:success] + assert @brex_item.reload.good? + end + + test "refreshes already discovered unlinked accounts during import" do + unlinked_account = @brex_item.brex_accounts.create!( + account_id: "cash_unlinked_1", + account_kind: "cash", + name: "Old Unlinked Cash", + currency: "USD", + current_balance: 1 + ) + + provider = mock("brex_provider") + provider.expects(:get_accounts).returns( + accounts: [ + cash_account_payload, + cash_account_payload.merge( + id: "cash_unlinked_1", + name: "Updated Unlinked Cash", + current_balance: { amount: 987_65, currency: "USD" } + ) + ] + ) + provider.expects(:get_cash_transactions).with("cash_1", start_date: anything).returns(transactions: []) + + result = BrexItem::Importer.new(@brex_item, brex_provider: provider).import + + assert result[:success] + assert_equal 2, result[:accounts_updated] + assert_equal "Updated Unlinked Cash", unlinked_account.reload.name + assert_equal BigDecimal("987.65"), unlinked_account.current_balance + end + + private + + def cash_account_payload + { + id: "cash_1", + name: "Operating Cash", + account_kind: "cash", + status: "ACTIVE", + current_balance: { amount: 120_000, currency: "USD" }, + available_balance: { amount: 110_000, currency: "USD" }, + account_number: "account-last4-9012", + routing_number: "routing-last4-0021" + } + end + + def card_account_payload + { + id: BrexAccount.card_account_id, + name: "Brex Card", + account_kind: "card", + status: "ACTIVE", + current_balance: { amount: 1_234, currency: "USD" }, + available_balance: { amount: 100_000, currency: "USD" }, + account_limit: { amount: 150_000, currency: "USD" }, + raw_card_accounts: [ + { + id: "card_account_1", + card_metadata: { + pan: "test-pan-placeholder" + } + } + ] + } + end +end diff --git a/test/models/brex_item/syncer_test.rb b/test/models/brex_item/syncer_test.rb new file mode 100644 index 000000000..ce192c5a5 --- /dev/null +++ b/test/models/brex_item/syncer_test.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require "test_helper" + +class BrexItem::SyncerTest < ActiveSupport::TestCase + setup do + @brex_item = brex_items(:one) + @syncer = BrexItem::Syncer.new(@brex_item) + end + + test "passes sync window start date to importer" do + window_start_date = Date.new(2026, 2, 1) + sync = mock_sync(window_start_date: window_start_date) + + @brex_item.expects(:import_latest_brex_data).with(sync_start_date: window_start_date).once + + @syncer.perform_sync(sync) + end + + test "records localized setup status text and counts" do + window_start_date = Date.new(2026, 2, 1) + sync = recording_sync(window_start_date: window_start_date) + + @brex_item.expects(:import_latest_brex_data).with(sync_start_date: window_start_date).once + + @syncer.perform_sync(sync) + + assert_equal [ + I18n.t("brex_items.syncer.importing_accounts"), + I18n.t("brex_items.syncer.checking_account_configuration"), + I18n.t("brex_items.syncer.accounts_need_setup", count: 1) + ], sync.updates.filter_map { |attrs| attrs[:status_text] } + + assert_equal 1, sync.sync_stats["total_accounts"] + assert_equal 0, sync.sync_stats["linked_accounts"] + assert_equal 1, sync.sync_stats["unlinked_accounts"] + end + + test "records importer failure counts in health stats" do + sync = recording_sync(window_start_date: Date.new(2026, 2, 1)) + @brex_item.expects(:import_latest_brex_data).returns( + success: false, + accounts_failed: 2, + transactions_failed: 1 + ) + + @syncer.perform_sync(sync) + + assert_equal 2, sync.sync_stats["total_errors"] + assert_equal [ + I18n.t("brex_items.syncer.accounts_failed", count: 2), + I18n.t("brex_items.syncer.transactions_failed", count: 1) + ], sync.sync_stats["errors"].map { |error| error["message"] } + end + + test "records account processing and scheduling failures in health stats" do + account = @brex_item.family.accounts.create!( + name: "Linked Brex Checking", + balance: 0, + currency: "USD", + accountable: Depository.new + ) + brex_account = @brex_item.brex_accounts.first + AccountProvider.create!(account: account, provider: brex_account) + + sync = recording_sync(window_start_date: Date.new(2026, 2, 1)) + @brex_item.expects(:import_latest_brex_data).returns( + success: true, + accounts_failed: 0, + transactions_failed: 0 + ) + @brex_item.expects(:process_accounts).returns([ + { brex_account_id: brex_account.id, success: false, error: "processing failure" } + ]) + @brex_item.expects(:schedule_account_syncs).returns([ + { account_id: account.id, success: false, error: "scheduling failure" } + ]) + + @syncer.perform_sync(sync) + + assert_equal 2, sync.sync_stats["total_errors"] + assert_equal [ + I18n.t("brex_items.syncer.account_processing_failed", count: 1), + I18n.t("brex_items.syncer.account_sync_failed", count: 1) + ], sync.sync_stats["errors"].map { |error| error["message"] } + end + + test "raises user safe credential error for Brex auth failures" do + sync = mock_sync(window_start_date: Date.new(2026, 2, 1)) + @brex_item.expects(:import_latest_brex_data) + .raises(Provider::Brex::BrexError.new("raw upstream auth body", :unauthorized, http_status: 401)) + Sentry.expects(:capture_exception) + + error = assert_raises(BrexItem::Syncer::SafeSyncError) do + @syncer.perform_sync(sync) + end + + assert_equal I18n.t("brex_items.syncer.credentials_invalid"), error.message + end + + private + + def mock_sync(window_start_date:) + sync = mock("sync") + sync.stubs(:respond_to?).with(:status_text).returns(true) + sync.stubs(:respond_to?).with(:sync_stats).returns(true) + sync.stubs(:sync_stats).returns({}) + sync.stubs(:window_start_date).returns(window_start_date) + sync.stubs(:window_end_date).returns(nil) + sync.stubs(:update!) + sync + end + + def recording_sync(window_start_date:) + Class.new do + attr_accessor :sync_stats, :status_text + attr_reader :updates + + define_method(:initialize) do |start_date| + @window_start_date = start_date + @window_end_date = nil + @created_at = Time.current + @sync_stats = {} + @updates = [] + end + + attr_reader :window_start_date, :window_end_date, :created_at + + def update!(attributes) + @updates << attributes + self.sync_stats = attributes[:sync_stats] if attributes.key?(:sync_stats) + self.status_text = attributes[:status_text] if attributes.key?(:status_text) + end + end.new(window_start_date) + end +end diff --git a/test/models/brex_item_test.rb b/test/models/brex_item_test.rb new file mode 100644 index 000000000..9d454b2fd --- /dev/null +++ b/test/models/brex_item_test.rb @@ -0,0 +1,198 @@ +require "test_helper" + +class BrexItemTest < ActiveSupport::TestCase + def setup + @brex_item = brex_items(:one) + end + + test "fixture is valid" do + assert @brex_item.valid? + end + + test "belongs to family" do + assert_equal families(:dylan_family), @brex_item.family + end + + test "credentials_configured returns true when token present" do + assert @brex_item.credentials_configured? + end + + test "credentials_configured returns false when token blank" do + @brex_item.token = nil + assert_not @brex_item.credentials_configured? + end + + test "credentials_configured returns false when token is whitespace" do + @brex_item.token = " " + assert_not @brex_item.credentials_configured? + end + + test "effective_base_url returns custom url when set" do + assert_equal "https://api-staging.brex.com", @brex_item.effective_base_url + end + + test "effective_base_url returns default when base_url blank" do + @brex_item.base_url = nil + assert_equal "https://api.brex.com", @brex_item.effective_base_url + end + + test "base_url accepts official Brex API roots" do + assert BrexItem.new(family: families(:empty), name: "Production", token: "token", base_url: "https://api.brex.com").valid? + assert BrexItem.new(family: families(:empty), name: "Staging", token: "token", base_url: "https://api-staging.brex.com").valid? + end + + test "base_url normalizes official URL case and trailing slash" do + item = BrexItem.create!( + family: families(:empty), + name: "Normalized Brex", + token: "token", + base_url: " HTTPS://API.BREX.COM/ " + ) + + assert_equal "https://api.brex.com", item.base_url + end + + test "token is stripped before validation and save" do + item = BrexItem.create!( + family: families(:empty), + name: "Token Normalized Brex", + token: " normalized_token ", + base_url: "https://api.brex.com" + ) + + assert_equal "normalized_token", item.token + end + + test "token cannot be blanked on update" do + original_token = @brex_item.token + + assert_raises(ActiveRecord::RecordInvalid) do + @brex_item.update!(token: " ") + end + + assert_equal original_token, @brex_item.reload.token + assert_includes @brex_item.errors[:token], "can't be blank" + end + + test "base_url rejects non-Brex hosts and endpoint paths" do + [ + "http://api.brex.com", + "https://evil.example.test", + "https://localhost", + "https://127.0.0.1", + "https://10.0.0.1", + "https://api.brex.com.evil.example", + "https://api.brex.com@127.0.0.1", + "https://api.brex.com:444", + "https://api.brex.com/v2", + "https://api.brex.com?debug=true", + "//api.brex.com" + ].each do |base_url| + item = BrexItem.new(family: families(:empty), name: "Invalid Brex", token: "token", base_url: base_url) + + refute item.valid?, "Expected #{base_url.inspect} to be invalid" + assert_includes item.errors[:base_url], I18n.t("activerecord.errors.models.brex_item.attributes.base_url.official_hosts_only") + end + end + + test "brex_provider returns Provider::Brex instance" do + provider = @brex_item.brex_provider + assert_instance_of Provider::Brex, provider + assert_equal @brex_item.token, provider.token + end + + test "declares Brex token and raw payload as encrypted" do + skip "Encryption not configured" unless BrexItem.encryption_ready? + + assert_includes BrexItem.encrypted_attributes.map(&:to_s), "token" + assert_includes BrexItem.encrypted_attributes.map(&:to_s), "raw_payload" + end + + test "resolve for returns explicit credentialed item scoped to family" do + resolved = BrexItem.resolve_for(family: @brex_item.family, brex_item_id: " #{@brex_item.id} ") + + assert_equal @brex_item, resolved + end + + test "resolve for refuses explicit items without usable credentials" do + item = BrexItem.create!( + family: @brex_item.family, + name: "Blank Resolve Brex", + token: "temporary_token", + base_url: "https://api.brex.com" + ) + item.update_column(:token, " ") + + assert_nil BrexItem.resolve_for(family: @brex_item.family, brex_item_id: item.id) + end + + test "resolve for does not select one item when multiple credentialed items exist" do + BrexItem.create!( + family: @brex_item.family, + name: "Second Resolve Brex", + token: "second_resolve_token", + base_url: "https://api.brex.com" + ) + + assert_nil BrexItem.resolve_for(family: @brex_item.family) + end + + test "schema requires name and token" do + columns = BrexItem.columns.index_by(&:name) + + assert_equal false, columns["name"].null + assert_equal false, columns["token"].null + end + + test "brex_provider returns nil when credentials not configured" do + @brex_item.token = nil + assert_nil @brex_item.brex_provider + end + + test "brex_provider returns nil when persisted base_url is not allowed" do + @brex_item.update_column(:base_url, "https://evil.example.test") + + assert_nil @brex_item.reload.brex_provider + end + + test "family credential check ignores blank and scheduled for deletion items" do + family = families(:empty) + blank_item = BrexItem.create!( + family: family, + name: "Blank Brex", + token: "temporary_token", + base_url: "https://api-staging.brex.com" + ) + blank_item.update_column(:token, "") + + whitespace_item = BrexItem.create!( + family: family, + name: "Whitespace Brex", + token: "temporary_token", + base_url: "https://api-staging.brex.com" + ) + whitespace_item.update_column(:token, " ") + + deleted_item = BrexItem.create!( + family: family, + name: "Deleted Brex", + token: "deleted_token", + base_url: "https://api-staging.brex.com", + scheduled_for_deletion: true + ) + + refute family.has_brex_credentials? + + whitespace_item.update_column(:token, "configured_token") + assert family.has_brex_credentials? + + whitespace_item.update_column(:token, " ") + deleted_item.update!(scheduled_for_deletion: false) + assert family.has_brex_credentials? + end + + test "syncer returns BrexItem::Syncer instance" do + syncer = @brex_item.send(:syncer) + assert_instance_of BrexItem::Syncer, syncer + end +end diff --git a/test/models/family/syncer_test.rb b/test/models/family/syncer_test.rb index 48ed9ffb2..be9624109 100644 --- a/test/models/family/syncer_test.rb +++ b/test/models/family/syncer_test.rb @@ -10,6 +10,7 @@ class Family::SyncerTest < ActiveSupport::TestCase manual_accounts_count = @family.accounts.manual.count plaid_items_count = @family.plaid_items.syncable.count + brex_items_count = @family.brex_items.syncable.count binance_items_count = @family.binance_items.syncable.count syncer = Family::Syncer.new(@family) @@ -24,6 +25,11 @@ class Family::SyncerTest < ActiveSupport::TestCase .with(parent_sync: family_sync, window_start_date: nil, window_end_date: nil) .times(plaid_items_count) + BrexItem.any_instance + .expects(:sync_later) + .with(parent_sync: family_sync, window_start_date: nil, window_end_date: nil) + .times(brex_items_count) + BinanceItem.any_instance .expects(:sync_later) .with(parent_sync: family_sync, window_start_date: nil, window_end_date: nil) @@ -67,6 +73,7 @@ class Family::SyncerTest < ActiveSupport::TestCase LunchflowItem.any_instance.stubs(:sync_later) EnableBankingItem.any_instance.stubs(:sync_later) SophtronItem.any_instance.stubs(:sync_later) + BrexItem.any_instance.stubs(:sync_later) BinanceItem.any_instance.stubs(:sync_later) syncer.perform_sync(family_sync) diff --git a/test/models/provider/brex_adapter_test.rb b/test/models/provider/brex_adapter_test.rb new file mode 100644 index 000000000..10fd26aad --- /dev/null +++ b/test/models/provider/brex_adapter_test.rb @@ -0,0 +1,224 @@ +require "uri" + +require "test_helper" + +class Provider::BrexAdapterTest < ActiveSupport::TestCase + test "supports Depository accounts" do + assert_includes Provider::BrexAdapter.supported_account_types, "Depository" + end + + test "supports CreditCard accounts" do + assert_includes Provider::BrexAdapter.supported_account_types, "CreditCard" + end + + test "does not support Investment accounts" do + assert_not_includes Provider::BrexAdapter.supported_account_types, "Investment" + end + + test "returns fallback connection config when no credentials exist yet" do + # Brex is a per-family provider - any family can connect + family = families(:empty) + configs = Provider::BrexAdapter.connection_configs(family: family) + + assert_equal 1, configs.length + assert_equal "brex", configs.first[:key] + assert_equal I18n.t("brex_items.provider_connection.default_name"), configs.first[:name] + assert configs.first[:can_connect] + end + + test "returns one connection config per credentialed brex item" do + family = families(:dylan_family) + first_item = brex_items(:one) + second_item = BrexItem.create!( + family: family, + name: "Business Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + + configs = Provider::BrexAdapter.connection_configs(family: family) + + assert_equal 2, configs.length + assert_equal [ "brex_#{second_item.id}", "brex_#{first_item.id}" ], configs.map { |config| config[:key] } + assert_equal [ + I18n.t("brex_items.provider_connection.name", name: second_item.name), + I18n.t("brex_items.provider_connection.name", name: first_item.name) + ], configs.map { |config| config[:name] } + + new_account_uri = URI.parse(configs.first[:new_account_path].call("Depository", "/accounts")) + assert_equal "/brex_items/select_accounts", new_account_uri.path + assert_includes new_account_uri.query, "brex_item_id=#{second_item.id}" + + existing_account_uri = URI.parse(configs.first[:existing_account_path].call(accounts(:depository).id)) + assert_equal "/brex_items/select_existing_account", existing_account_uri.path + assert_includes existing_account_uri.query, "brex_item_id=#{second_item.id}" + end + + test "connection configs ignore items with whitespace-only tokens" do + family = families(:dylan_family) + BrexItem.create!( + family: family, + name: "Blank Brex", + token: "temporary_token", + base_url: "https://api.brex.com" + ).update_column(:token, " ") + + configs = Provider::BrexAdapter.connection_configs(family: family) + + assert_equal [ "brex_#{brex_items(:one).id}" ], configs.map { |config| config[:key] } + end + + test "build_provider returns nil when family is nil" do + assert_nil Provider::BrexAdapter.build_provider(family: nil) + end + + test "build_provider returns nil when family has no brex items" do + family = families(:empty) + assert_nil Provider::BrexAdapter.build_provider(family: family) + end + + test "build_provider returns Brex provider when credentials configured" do + family = families(:dylan_family) + provider = Provider::BrexAdapter.build_provider(family: family) + + assert_instance_of Provider::Brex, provider + end + + test "build_provider uses explicit brex item credentials" do + family = families(:dylan_family) + second_item = BrexItem.create!( + family: family, + name: "Business Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + + provider = Provider::BrexAdapter.build_provider(family: family, brex_item_id: second_item.id) + + assert_instance_of Provider::Brex, provider + assert_equal "second_brex_token", provider.token + assert_equal "https://api.brex.com", provider.base_url + end + + test "build_provider does not pick the first connection when multiple credentials exist" do + family = families(:dylan_family) + BrexItem.create!( + family: family, + name: "Business Brex", + token: "second_brex_token", + base_url: "https://api.brex.com" + ) + + assert_nil Provider::BrexAdapter.build_provider(family: family) + end + + test "build_provider strips surrounding token whitespace" do + family = families(:dylan_family) + second_item = BrexItem.create!( + family: family, + name: "Business Brex", + token: " second_brex_token \n", + base_url: "https://api.brex.com" + ) + + provider = Provider::BrexAdapter.build_provider(family: family, brex_item_id: second_item.id) + + assert_equal "second_brex_token", provider.token + end + + test "build_provider refuses brex items outside the family" do + family = families(:dylan_family) + other_item = BrexItem.create!( + family: families(:empty), + name: "Other Brex", + token: "other_brex_token", + base_url: "https://api.brex.com" + ) + + assert_nil Provider::BrexAdapter.build_provider(family: family, brex_item_id: other_item.id) + end + + test "build_provider refuses explicit brex item without usable credentials" do + family = families(:dylan_family) + blank_item = BrexItem.create!( + family: family, + name: "Blank Brex", + token: "temporary_token", + base_url: "https://api.brex.com" + ) + blank_item.update_column(:token, " ") + + assert_nil Provider::BrexAdapter.build_provider(family: family, brex_item_id: blank_item.id) + end + + test "build_provider refuses explicit brex item with invalid persisted base_url" do + family = families(:dylan_family) + item = BrexItem.create!( + family: family, + name: "Invalid URL Brex", + token: "token", + base_url: "https://api.brex.com" + ) + item.update_column(:base_url, "https://evil.example.test") + + assert_nil Provider::BrexAdapter.build_provider(family: family, brex_item_id: item.id) + end + + test "reads institution metadata from brex account column" do + brex_account = brex_items(:one).brex_accounts.create!( + account_id: "metadata_cash", + account_kind: "cash", + name: "Metadata Cash", + currency: "USD", + institution_metadata: { + "name" => "Brex", + "domain" => "brex.com", + "url" => "https://brex.com" + } + ) + + adapter = Provider::BrexAdapter.new(brex_account) + + assert_equal "brex.com", brex_account.institution_metadata["domain"] + assert_equal "brex.com", adapter.institution_domain + assert_equal "Brex", adapter.institution_name + assert_equal "https://brex.com", adapter.institution_url + end + + test "falls back to brex item institution metadata" do + brex_item = brex_items(:one) + brex_item.update!( + institution_name: "Brex Item Name", + institution_url: "https://brex.com/item", + institution_color: "#123456" + ) + brex_account = brex_item.brex_accounts.create!( + account_id: "metadata_fallback_cash", + account_kind: "cash", + name: "Metadata Fallback Cash", + currency: "USD" + ) + + adapter = Provider::BrexAdapter.new(brex_account) + + assert_equal "Brex Item Name", adapter.institution_name + assert_equal "https://brex.com/item", adapter.institution_url + assert_equal "#123456", adapter.institution_color + end + + test "logs institution urls without hosts" do + brex_account = brex_items(:one).brex_accounts.create!( + account_id: "metadata_bad_url_cash", + account_kind: "cash", + name: "Metadata Bad URL Cash", + currency: "USD", + institution_metadata: { + "url" => "not-a-url" + } + ) + + Rails.logger.expects(:warn).with(regexp_matches(/institution URL has no host/)) + + assert_nil Provider::BrexAdapter.new(brex_account).institution_domain + end +end diff --git a/test/models/provider/brex_test.rb b/test/models/provider/brex_test.rb new file mode 100644 index 000000000..f84185bff --- /dev/null +++ b/test/models/provider/brex_test.rb @@ -0,0 +1,289 @@ +require "test_helper" + +class Provider::BrexTest < ActiveSupport::TestCase + def setup + @provider = Provider::Brex.new("test_token", base_url: "https://api-staging.brex.com") + end + + test "initializes with token and default base_url" do + provider = Provider::Brex.new("my_token") + assert_equal "my_token", provider.token + assert_equal "https://api.brex.com", provider.base_url + end + + test "initializes with custom base_url" do + assert_equal "test_token", @provider.token + assert_equal "https://api-staging.brex.com", @provider.base_url + end + + test "initializes with stripped token and removes trailing base url slash" do + provider = Provider::Brex.new(" test_token \n", base_url: "https://api.brex.com/") + + assert_equal "test_token", provider.token + assert_equal "https://api.brex.com", provider.base_url + end + + test "initializes with official staging base url" do + provider = Provider::Brex.new("test_token", base_url: "https://api-staging.brex.com/") + + assert_equal "https://api-staging.brex.com", provider.base_url + end + + test "rejects arbitrary base urls" do + [ + "http://api.brex.com", + "https://evil.example.test", + "https://localhost", + "https://127.0.0.1", + "https://10.0.0.1", + "https://api.brex.com.evil.example", + "https://api.brex.com@127.0.0.1", + "https://api.brex.com:444", + "https://api.brex.com/v1", + "https://api.brex.com?host=evil.example.test", + "//api.brex.com" + ].each do |base_url| + assert_raises ArgumentError do + Provider::Brex.new("test_token", base_url: base_url) + end + end + end + + test "BrexError includes error_type" do + error = Provider::Brex::BrexError.new("Test error", :unauthorized) + assert_equal "Test error", error.message + assert_equal :unauthorized, error.error_type + end + + test "BrexError defaults error_type to unknown" do + error = Provider::Brex::BrexError.new("Test error") + assert_equal :unknown, error.error_type + end + + test "fetches cash accounts from the v2 endpoint with bearer auth" do + response = OpenStruct.new( + code: 200, + body: { items: [ { id: "cash_1", name: "Operating" } ] }.to_json, + headers: {} + ) + + Provider::Brex.expects(:get) + .with( + "https://api.brex.com/v2/accounts/cash?limit=1000", + headers: { + "Authorization" => "Bearer test_token", + "Content-Type" => "application/json", + "Accept" => "application/json" + } + ) + .returns(response) + + accounts = Provider::Brex.new(" test_token ").get_cash_accounts + + assert_equal 1, accounts.length + assert_equal "cash_1", accounts.first[:id] + assert_equal "cash", accounts.first[:account_kind] + end + + test "fetches card accounts from the paginated v2 endpoint" do + response = OpenStruct.new( + code: 200, + body: [ { id: "card_account_1", status: "ACTIVE" } ].to_json, + headers: {} + ) + + Provider::Brex.expects(:get) + .with( + "https://api.brex.com/v2/accounts/card?limit=1000", + headers: { + "Authorization" => "Bearer test_token", + "Content-Type" => "application/json", + "Accept" => "application/json" + } + ) + .returns(response) + + accounts = Provider::Brex.new("test_token").get_card_accounts + + assert_equal 1, accounts.length + assert_equal "card_account_1", accounts.first[:id] + assert_equal "card", accounts.first[:account_kind] + end + + test "aggregates card accounts into one provider account" do + cash_response = OpenStruct.new( + code: 200, + body: { items: [] }.to_json, + headers: {} + ) + card_response = OpenStruct.new( + code: 200, + body: { + items: [ + { + id: "card_account_1", + status: "ACTIVE", + current_balance: { amount: 12_345, currency: "USD" }, + available_balance: { amount: 100_000, currency: "USD" }, + account_limit: { amount: 250_000, currency: "USD" } + } + ] + }.to_json, + headers: {} + ) + + Provider::Brex.stubs(:get).returns(cash_response, card_response) + + accounts_data = Provider::Brex.new("test_token").get_accounts + + assert_equal [ "card_primary" ], accounts_data[:accounts].map { |account| account[:id] } + assert_equal "card", accounts_data[:accounts].first[:account_kind] + assert_equal 1, accounts_data[:accounts].first[:card_accounts_count] + end + + test "does not aggregate mixed currency card balances" do + cash_response = OpenStruct.new( + code: 200, + body: { items: [] }.to_json, + headers: {} + ) + card_response = OpenStruct.new( + code: 200, + body: [ + { + id: "card_account_1", + current_balance: { amount: 12_345, currency: "USD" } + }, + { + id: "card_account_2", + current_balance: { amount: 6_789, currency: "EUR" } + } + ].to_json, + headers: {} + ) + + Provider::Brex.stubs(:get).returns(cash_response, card_response) + + accounts_data = Provider::Brex.new("test_token").get_accounts + + assert_nil accounts_data[:accounts].first[:current_balance] + end + + test "guards repeated pagination cursors" do + first_response = OpenStruct.new( + code: 200, + body: { items: [ { id: "tx_1" } ], next_cursor: "cursor_1" }.to_json, + headers: {} + ) + second_response = OpenStruct.new( + code: 200, + body: { items: [ { id: "tx_2" } ], next_cursor: "cursor_1" }.to_json, + headers: {} + ) + + Provider::Brex.stubs(:get).returns(first_response, second_response) + + error = assert_raises Provider::Brex::BrexError do + Provider::Brex.new("test_token").get_primary_card_transactions + end + + assert_equal :pagination_error, error.error_type + end + + test "guards pagination page cap" do + responses = (1..26).map do |page| + OpenStruct.new( + code: 200, + body: { items: [ { id: "tx_#{page}" } ], next_cursor: "cursor_#{page}" }.to_json, + headers: {} + ) + end + + Provider::Brex.stubs(:get).returns(*responses) + + error = assert_raises Provider::Brex::BrexError do + Provider::Brex.new("test_token").get_primary_card_transactions + end + + assert_equal :pagination_error, error.error_type + assert_includes error.message, "exceeded 25 pages" + end + + test "sends posted_at_start as RFC3339 date time" do + response = OpenStruct.new( + code: 200, + body: { items: [] }.to_json, + headers: {} + ) + + Provider::Brex.expects(:get) + .with( + "https://api.brex.com/v2/transactions/card/primary?posted_at_start=2026-01-02T00%3A00%3A00Z&limit=1000", + headers: { + "Authorization" => "Bearer test_token", + "Content-Type" => "application/json", + "Accept" => "application/json" + } + ) + .returns(response) + + Provider::Brex.new("test_token").get_primary_card_transactions(start_date: Date.new(2026, 1, 2)) + end + + test "raises clear error for invalid start date" do + error = assert_raises ArgumentError do + Provider::Brex.new("test_token").get_primary_card_transactions(start_date: "not-a-date") + end + + assert_includes error.message, "Invalid start_date" + end + + test "maps rate limits and exposes trace id without leaking body" do + response = OpenStruct.new( + code: 429, + body: { message: "secret raw provider body" }.to_json, + headers: { "x-brex-trace-id" => "trace_123" } + ) + + Provider::Brex.stubs(:get).returns(response) + + error = assert_raises Provider::Brex::BrexError do + Provider::Brex.new("test_token").get_cash_accounts + end + + assert_equal :rate_limited, error.error_type + assert_equal 429, error.http_status + assert_equal "trace_123", error.trace_id + refute_includes error.message, "secret raw provider body" + end + + test "maps non-success responses without exposing provider body" do + expectations = { + 400 => [ :bad_request, "Bad request to Brex API" ], + 401 => [ :unauthorized, "Invalid Brex API token or account permissions" ], + 403 => [ :access_forbidden, "Access forbidden - check Brex API token scopes" ], + 404 => [ :not_found, "Brex resource not found" ], + 500 => [ :fetch_failed, "Failed to fetch data from Brex API: HTTP 500" ] + } + + expectations.each do |status, (error_type, message)| + response = OpenStruct.new( + code: status, + body: { message: "secret provider body #{status}" }.to_json, + headers: { "X-Brex-Trace-Id" => "trace_#{status}" } + ) + + Provider::Brex.stubs(:get).returns(response) + + error = assert_raises Provider::Brex::BrexError do + Provider::Brex.new("test_token").get_cash_accounts + end + + assert_equal error_type, error.error_type + assert_equal status, error.http_status + assert_equal "trace_#{status}", error.trace_id + assert_equal message, error.message + refute_includes error.message, "secret provider body" + end + end +end From 834686cffd3e35120c151b7f062c2ad3f9343dba Mon Sep 17 00:00:00 2001 From: plind <59729252+plind-junior@users.noreply.github.com> Date: Wed, 13 May 2026 09:17:10 -0700 Subject: [PATCH 31/31] fix(simplefin): treat Vanguard/Fidelity cost_basis as total when needed (#1772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(simplefin): treat Vanguard/Fidelity cost_basis as total when needed PR #1692 normalized SimpleFIN holdings cost_basis under the assumption that the `cost_basis` / `basis` keys carry a per-share value (per the SimpleFIN spec) and only `total_cost` / `value` carry a total position cost. Vanguard and Fidelity violate the spec — they populate `cost_basis` with the *total* (see the payload in #1182). After PR #1692 those holdings get stored with cost_basis = total, and Holding#calculate_trend then computes previous = qty × avg_cost, so the "previous" value is inflated by a factor of qty and an entire investment account renders a phantom return of roughly -(1 − 1/qty), i.e. -97% to -99%. Fix: sanity-check raw cost_basis against the holding's market share price. Let share_price = market_value / qty; the geometric midpoint between "raw is per-share" (raw ≈ share_price) and "raw is total" (raw ≈ qty × share_price) is share_price × √qty. If raw is above the midpoint it is divided by qty; otherwise it is kept as per-share. Falls back to the pre-fix behaviour (trust the spec) when market_value or qty is unavailable, so confidently-correct readings are never made worse. Verified against the reported Vanguard payload (qty=139, cost_basis= 22004.40, market_value=22626.42): normalize_cost_basis now returns $158.31/share, matching 22004.40 / 139, and the phantom -99% return collapses to a realistic ~+2.8%. Per-share readings ($45 cost on a $50 share price) remain untouched. Closes #1718. Refs #1182, #1692. * fixup: replace cost_basis heuristic with institution allowlist Codex and @EdeAbreu23 flagged a real false-positive in the previous geometric-midpoint heuristic: a legitimate per-share `cost_basis` on a holding with a large unrealized loss (e.g. 100 shares with $100/share basis now worth $5/share) trips `share_price × √qty` and gets divided to $1/share — corrupting any standards-compliant brokerage with a big loss. Adopt @EdeAbreu23's safer shape: - total_cost / value: always divide by qty (unchanged from #1692). - cost_basis / basis: keep as-is by default. - Only divide cost_basis / basis when the holding's SimpleFIN account is connected to a known-misbehaving institution. Allowlist starts with `vanguard` and `fidelity`, matched case-insensitively against the account's stored org name and domain. Easy to extend as more brokerages turn up. Trades a small maintenance cost (curated list) for zero risk of corrupting compliant providers. Verified against five scenarios (all expected): Vanguard total in cost_basis (allowlist) → +2.83% Fidelity total in basis (allowlist) → +33.33% Big-loss per-share (Codex case) → -95.0% (preserved) Honest per-share, small loss → +11.11% (unchanged) total_cost on any institution → +11.11% (unchanged) --------- Co-authored-by: plind-junior --- .../investments/holdings_processor.rb | 51 ++++++++++++--- .../investments/holdings_processor_test.rb | 63 +++++++++++++++++++ 2 files changed, 104 insertions(+), 10 deletions(-) diff --git a/app/models/simplefin_account/investments/holdings_processor.rb b/app/models/simplefin_account/investments/holdings_processor.rb index 8cb4882ef..29bf62155 100644 --- a/app/models/simplefin_account/investments/holdings_processor.rb +++ b/app/models/simplefin_account/investments/holdings_processor.rb @@ -48,7 +48,7 @@ class SimplefinAccount::Investments::HoldingsProcessor qty = parse_decimal(any_of(simplefin_holding, %w[shares quantity qty units])) market_value = parse_decimal(any_of(simplefin_holding, %w[market_value current_value])) raw_cost_basis, cost_basis_source_key = cost_basis_from(simplefin_holding) - cost_basis = normalize_cost_basis(raw_cost_basis, qty, cost_basis_source_key) + cost_basis = normalize_cost_basis(raw_cost_basis, qty, cost_basis_source_key, institution_reports_total_basis?) # Derive price from market_value when possible; otherwise fall back to any price field fallback_price = parse_decimal(any_of(simplefin_holding, %w[purchase_price price unit_price average_cost avg_cost])) @@ -124,19 +124,50 @@ class SimplefinAccount::Investments::HoldingsProcessor [ nil, nil ] end - # Sure stores holding cost_basis as per-share average cost. Some SimpleFIN - # providers expose total position basis via total_cost/value, so normalize only - # when the selected provider field is known to represent total position basis. - def normalize_cost_basis(raw_cost_basis, qty, source_key) + # Sure stores holding cost_basis as per-share average cost. SimpleFIN + # brokerages are inconsistent about which field carries which shape: + # + # - total_cost / value: always a total position cost per the SimpleFIN + # spec and observed payloads; divide by qty unconditionally. + # - cost_basis / basis: the spec calls this per-share, and most + # brokerages comply. Keep these values unchanged by default. + # + # Exception: a small allowlist of brokerages (Vanguard, Fidelity) is + # known to populate cost_basis with the total position cost in violation + # of the spec (#1718, #1182). For those connections only, divide by qty. + # + # An earlier revision of this fix used a magnitude heuristic + # (share_price × √qty midpoint). It was withdrawn because a legitimate + # per-share basis on a holding with a large unrealized loss + # (e.g. 100 shares with basis $100 now worth $5) trips the midpoint and + # gets mis-divided to $1/share — corrupting compliant providers. The + # allowlist trades some manual maintenance for that safety. + def normalize_cost_basis(raw_cost_basis, qty, source_key, total_basis_institution = false) return nil if raw_cost_basis.nil? - if %w[total_cost value].include?(source_key) + if %w[total_cost value].include?(source_key) || + (total_basis_institution && %w[cost_basis basis].include?(source_key)) return nil unless qty.to_d.positive? - - raw_cost_basis / qty - else - raw_cost_basis + return raw_cost_basis / qty end + + raw_cost_basis + end + + # Institutions known to populate the SimpleFIN `cost_basis` / `basis` + # field with the total position cost rather than the per-share value the + # spec requires. Matched as case-insensitive substrings against the + # account's stored org name and domain. + TOTAL_BASIS_INSTITUTIONS = %w[vanguard fidelity].freeze + + def institution_reports_total_basis? + org = simplefin_account.respond_to?(:org_data) ? simplefin_account.org_data : nil + return false if org.blank? + + candidates = [ org["name"], org[:name], org["domain"], org[:domain] ].compact.map(&:to_s).map(&:downcase) + return false if candidates.empty? + + TOTAL_BASIS_INSTITUTIONS.any? { |needle| candidates.any? { |c| c.include?(needle) } } end def resolve_security(symbol, description) diff --git a/test/models/simplefin_account/investments/holdings_processor_test.rb b/test/models/simplefin_account/investments/holdings_processor_test.rb index 122d908c0..a3842ac6a 100644 --- a/test/models/simplefin_account/investments/holdings_processor_test.rb +++ b/test/models/simplefin_account/investments/holdings_processor_test.rb @@ -80,6 +80,69 @@ class SimplefinAccount::Investments::HoldingsProcessorTest < ActiveSupport::Test assert_equal "total_cost", source_key end + test "cost_basis from a known total-basis institution is divided by qty" do + # Issue #1718 / #1182: Vanguard populates cost_basis with the total + # position cost. When the institution is on the allowlist we divide. + cost_basis = @processor.send( + :normalize_cost_basis, + BigDecimal("22004.40"), + BigDecimal("139.00"), + "cost_basis", + true # institution_reports_total_basis? + ) + + assert_in_delta 158.30, cost_basis.to_f, 0.01 + end + + test "basis from a known total-basis institution is divided by qty" do + cost_basis = @processor.send( + :normalize_cost_basis, + BigDecimal("9000.00"), + BigDecimal("200"), + "basis", + true + ) + + assert_equal BigDecimal("45.00"), cost_basis + end + + test "cost_basis from a compliant institution is kept untouched (no false divide)" do + # Codex regression: a legitimate per-share basis on a holding with a + # large unrealized loss (e.g. $100/share basis now worth $5/share) must + # NOT be divided by qty. Per the SimpleFIN spec, cost_basis is per-share + # — only the institution allowlist should override that. + cost_basis = @processor.send( + :normalize_cost_basis, + BigDecimal("100.00"), + BigDecimal("100"), + "cost_basis", + false + ) + + assert_equal BigDecimal("100.00"), cost_basis + end + + test "institution_reports_total_basis? matches Vanguard and Fidelity org metadata" do + cases = { + { "name" => "Vanguard" } => true, + { "name" => "VANGUARD BROKERAGE" } => true, + { "name" => "Fidelity Investments" } => true, + { "domain" => "vanguard.com" } => true, + { "domain" => "401k.fidelity.com" } => true, + { "name" => "Charles Schwab", "domain" => "schwab.com" } => false, + { "name" => "Chase" } => false, + {} => false + } + + cases.each do |org, expected| + account = Struct.new(:org_data).new(org) + processor = SimplefinAccount::Investments::HoldingsProcessor.new(account) + assert_equal expected, + processor.send(:institution_reports_total_basis?), + "org_data #{org.inspect} expected #{expected}" + end + end + test "missing cost basis fields return nil" do payload = { "market_value" => "10108.16"