diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 119f883aa..8d62c1525 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION=3.4.7 +ARG RUBY_VERSION=3.4.9 FROM ruby:${RUBY_VERSION}-slim-bookworm ENV DEBIAN_FRONTEND=noninteractive diff --git a/.env.example b/.env.example index 9128d758a..c84d0e414 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,8 @@ OPENAI_URI_BASE= # Optional: LLM token budget (applies to chat, auto-categorize, merchant detection, PDF processing). # Lower these for small-context local models (Ollama, LM Studio, LocalAI). -# Defaults work for modern cloud OpenAI models without configuration. +# For larger local models, raise the context window to match the model you actually run. +# Example: Gemma 3/4, Qwen, and other large-context models often need `LLM_CONTEXT_WINDOW=8192` or higher. # LLM_CONTEXT_WINDOW=2048 # LLM_MAX_RESPONSE_TOKENS=512 # LLM_MAX_HISTORY_TOKENS= @@ -111,6 +112,13 @@ REDIS_URL=redis://localhost:6379/1 # REDIS_SENTINEL_USERNAME=default # REDIS_PASSWORD=your-redis-password # pipelock:ignore +# Sidekiq Web UI (/sidekiq) +# The queue dashboard is reachable in production only by signed-in super admins +# (the first user created on the instance). Optionally set BOTH variables below +# to require basic-auth credentials as a second layer on top of that. +# SIDEKIQ_WEB_USERNAME= +# SIDEKIQ_WEB_PASSWORD= + # App Domain # This is the domain that your Sure instance will be hosted at. It is used to generate links in emails and other places. APP_DOMAIN= @@ -138,6 +146,14 @@ POSTHOG_HOST= # Disable enforcing SSL connections # DISABLE_SSL=true +# Customizations to outbound SSL/TLS connections +# Path to custom CA certificate (PEM format) +# SSL_CA_FILE= +# Enable/disable SSL verification +# SSL_VERIFY=true +# Enable verbose SSL logging +# SSL_DEBUG=false + # Active Record Encryption Keys (Optional) # These keys are used to encrypt sensitive data like API keys in the database. # For managed mode: Set these environment variables to provide encryption keys. diff --git a/.env.local.example b/.env.local.example index fea16335c..383fb71ac 100644 --- a/.env.local.example +++ b/.env.local.example @@ -12,6 +12,11 @@ SIMPLEFIN_DEBUG_RAW=false # SIMPLEFIN_INCLUDE_PENDING: when truthy, forces `pending=1` on SimpleFIN fetches when caller doesn't specify `pending:` SIMPLEFIN_INCLUDE_PENDING=false +# SnapTrade OAuth app credentials (register an OAuth app at https://dashboard.snaptrade.com, +# add https:///snaptrade_items/oauth_callback as a redirect URI) +SNAPTRADE_OAUTH_CLIENT_ID= +SNAPTRADE_OAUTH_CLIENT_SECRET= + # Lunchflow runtime flags (default-off) # LUNCHFLOW_DEBUG_RAW: when truthy, logs the raw payload returned by Lunchflow (debug-only; can be noisy) LUNCHFLOW_DEBUG_RAW=false diff --git a/.env.test.example b/.env.test.example index 02d7fea24..e8f7848de 100644 --- a/.env.test.example +++ b/.env.test.example @@ -11,6 +11,11 @@ SIMPLEFIN_DEBUG_RAW=false # SIMPLEFIN_INCLUDE_PENDING: when truthy, forces `pending=1` on SimpleFIN fetches when caller doesn't specify `pending:` SIMPLEFIN_INCLUDE_PENDING=false +# SnapTrade OAuth app credentials (register an OAuth app at https://dashboard.snaptrade.com, +# add https:///snaptrade_items/oauth_callback as a redirect URI) +SNAPTRADE_OAUTH_CLIENT_ID= +SNAPTRADE_OAUTH_CLIENT_SECRET= + # Lunchflow runtime flags (default-off) # LUNCHFLOW_DEBUG_RAW: when truthy, logs the raw payload returned by Lunchflow (debug-only; can be noisy) LUNCHFLOW_DEBUG_RAW=false diff --git a/.gitattributes b/.gitattributes index b387bb10a..36ff81d2b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,3 +15,4 @@ config/credentials.yml.enc diff=rails_credentials bin/* text eol=lf *.sh text eol=lf *.rb text eol=lf +.devcontainer/.bashrc text eol=lf diff --git a/.github/workflows/chart-ci.yml b/.github/workflows/chart-ci.yml index 8ca9111d4..a80af679f 100644 --- a/.github/workflows/chart-ci.yml +++ b/.github/workflows/chart-ci.yml @@ -1,6 +1,7 @@ name: Chart CI on: + workflow_dispatch: pull_request: paths: - 'charts/**' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9ab5d2c4..76d7c52a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: env: PLAID_CLIENT_ID: foo PLAID_SECRET: bar - DATABASE_URL: postgres://postgres:postgres@localhost:5432 + DATABASE_URL: postgres://postgres:postgres@localhost:5432 # pipelock:ignore REDIS_URL: redis://localhost:6379 RAILS_ENV: test @@ -141,7 +141,7 @@ jobs: env: PLAID_CLIENT_ID: foo PLAID_SECRET: bar - DATABASE_URL: postgres://postgres:postgres@localhost:5432 + DATABASE_URL: postgres://postgres:postgres@localhost:5432 # pipelock:ignore REDIS_URL: redis://localhost:6379 RAILS_ENV: test diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml new file mode 100644 index 000000000..222ecf164 --- /dev/null +++ b/.github/workflows/desktop-release.yml @@ -0,0 +1,82 @@ +name: Desktop Build + +# Build the macOS desktop app as part of the normal Sure release. The caller +# supplies the v* release ref, so desktop versions cannot diverge independently. +on: + workflow_call: + +permissions: + contents: read + +jobs: + build: + name: Build macOS Desktop App + runs-on: macos-latest # Apple Silicon runner; cross-builds the x86_64 slice + timeout-minutes: 30 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - name: Resolve and verify release version + id: ver + shell: bash + run: | + set -euo pipefail + VERSION="$(tr -d '[:space:]' < .sure-version)" + TAG_VERSION="${GITHUB_REF_NAME#v}" + + if [ -z "$VERSION" ]; then + echo "::error::.sure-version is empty or unreadable" + exit 1 + fi + if [ "$VERSION" != "$TAG_VERSION" ]; then + echo "::error::.sure-version ($VERSION) does not match release tag (v$TAG_VERSION)" + exit 1 + fi + echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$' \ + || { echo "::error::invalid version '$VERSION'"; exit 1; } + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Stamp desktop version for the build + env: + VERSION: ${{ steps.ver.outputs.version }} + run: | + set -euo pipefail + npm version "$VERSION" --no-git-tag-version --allow-same-version --prefix desktop + tmp="$(mktemp)" + jq --arg v "$VERSION" '.version = $v' desktop/src-tauri/tauri.conf.json > "$tmp" + mv "$tmp" desktop/src-tauri/tauri.conf.json + perl -0pi -e 's/(^version = ").*?(")/$1$ENV{VERSION}$2/m' desktop/src-tauri/Cargo.toml + perl -0pi -e 's/(name = "sure-desktop"\nversion = ").*?(")/$1$ENV{VERSION}$2/' desktop/src-tauri/Cargo.lock + + # No dependency/build caches on the release workflow: a poisoned Actions + # cache written by another workflow must never flow into a published .dmg. + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + with: + targets: aarch64-apple-darwin,x86_64-apple-darwin + + - name: Install frontend dependencies + working-directory: desktop + run: npm ci + + - name: Build universal unsigned DMG + uses: tauri-apps/tauri-action@84b9d35b5fc46c1e45415bdb6144030364f7ebc5 # v0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + projectPath: desktop + includeUpdaterJson: false + args: --target universal-apple-darwin + + - name: Upload desktop DMG + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: desktop-release-dmg + path: desktop/src-tauri/target/universal-apple-darwin/release/bundle/dmg/*.dmg + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/gittensor-impact.yml b/.github/workflows/gittensor-impact.yml new file mode 100644 index 000000000..64bec0433 --- /dev/null +++ b/.github/workflows/gittensor-impact.yml @@ -0,0 +1,32 @@ +name: Gittensor Impact Report + +on: + schedule: + # Refresh daily at 14:00 UTC. + - cron: "0 14 * * *" + workflow_dispatch: + inputs: + since: + description: "Git date window for the report" + default: "30 days ago" + required: true + +permissions: + contents: write + +jobs: + refresh: + name: Refresh Gittensor impact assets + if: github.repository == 'we-promise/sure' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: matthewevans/gittensor-impact-action@v1 + with: + repo: we-promise/sure + since: ${{ inputs.since || '30 days ago' }} + publish-mode: branch + asset-branch: gittensor-impact-assets + title: "Sure is part of the Gittensor community" + accent-color: "#ff6a00" + neutral-color: "#85898b" \ No newline at end of file diff --git a/.github/workflows/helm-publish.yml b/.github/workflows/helm-publish.yml index 510c64049..0588162c0 100644 --- a/.github/workflows/helm-publish.yml +++ b/.github/workflows/helm-publish.yml @@ -40,6 +40,12 @@ jobs: - name: Resolve chart and app versions id: version shell: bash + # Bind workflow inputs to env so the values arrive as shell variables + # instead of being interpolated verbatim by the `${{ }}` runner pass. + # zizmor flags the direct expansion as a template-injection risk. + env: + CHART_VERSION_INPUT: ${{ inputs.chart_version }} + APP_VERSION_INPUT: ${{ inputs.app_version }} run: | set -euo pipefail @@ -48,18 +54,24 @@ jobs: echo "${raw#v}" } - if [ -n "${{ inputs.chart_version }}" ]; then - CHART_VERSION="$(normalize_version "${{ inputs.chart_version }}")" + if [ -n "$CHART_VERSION_INPUT" ]; then + CHART_VERSION="$(normalize_version "$CHART_VERSION_INPUT")" elif [[ "${GITHUB_REF_TYPE}" == "tag" && "${GITHUB_REF_NAME}" == v* ]]; then CHART_VERSION="$(normalize_version "${GITHUB_REF_NAME}")" else CHART_VERSION="0.0.0-nightly.$(date -u +'%Y%m%d.%H%M%S')" fi - if [ -n "${{ inputs.app_version }}" ]; then - APP_VERSION="${{ inputs.app_version }}" + # Normalize APP_VERSION the same way CHART_VERSION is — appVersion + # must match the OCI image tag in GHCR, which is published without a + # leading `v`. Without this, a release on tag `v0.7.1-rc.1` writes + # `appVersion: "v0.7.1-rc.1"` into Chart.yaml / index.yaml, and Helm + # then fails to pull `ghcr.io/we-promise/sure:v0.7.1-rc.1` (the real + # tag is `0.7.1-rc.1`). See #2050. + if [ -n "$APP_VERSION_INPUT" ]; then + APP_VERSION="$(normalize_version "$APP_VERSION_INPUT")" elif [[ "${GITHUB_REF_TYPE}" == "tag" && "${GITHUB_REF_NAME}" == v* ]]; then - APP_VERSION="${GITHUB_REF_NAME}" + APP_VERSION="$(normalize_version "${GITHUB_REF_NAME}")" else APP_VERSION="${CHART_VERSION}" fi diff --git a/.github/workflows/ios-testflight.yml b/.github/workflows/ios-testflight.yml index 13ae2f52c..718f49bad 100644 --- a/.github/workflows/ios-testflight.yml +++ b/.github/workflows/ios-testflight.yml @@ -153,7 +153,7 @@ jobs: working-directory: mobile if: ${{ steps.check_prereqs.outputs.enabled == 'true' }} env: - APP_BUNDLE_ID: am.sure.mobile + APP_BUNDLE_ID: ${{ vars.IOS_APP_BUNDLE_ID || 'am.sure.mobile' }} IOS_TEAM_ID: ${{ secrets.IOS_TEAM_ID }} PROFILE_NAME: ${{ secrets.IOS_PROVISIONING_PROFILE_NAME }} IOS_DISTRIBUTION_CERT_NAME: ${{ secrets.IOS_DISTRIBUTION_CERT_NAME }} @@ -173,14 +173,16 @@ jobs: path = Path("ios/Runner.xcodeproj/project.pbxproj") text = path.read_text() + app_bundle_id = os.environ["APP_BUNDLE_ID"] team = os.environ["IOS_TEAM_ID"] profile = os.environ["PROFILE_NAME"] identity = os.environ["IOS_DISTRIBUTION_CERT_NAME"] def patch_block(match): block = match.group(0) - if "PRODUCT_BUNDLE_IDENTIFIER = am.sure.mobile;" not in block: + if "PRODUCT_BUNDLE_IDENTIFIER =" not in block: return block + block = re.sub(r'PRODUCT_BUNDLE_IDENTIFIER = .*?;', f'PRODUCT_BUNDLE_IDENTIFIER = {app_bundle_id};', block) if "CODE_SIGN_STYLE = Manual;" not in block: block = block.replace("CURRENT_PROJECT_VERSION = \"$(FLUTTER_BUILD_NUMBER)\";", "CURRENT_PROJECT_VERSION = \"$(FLUTTER_BUILD_NUMBER)\";\n\t\t\t\tCODE_SIGN_STYLE = Manual;") if '"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "Apple Distribution";' not in block: diff --git a/.github/workflows/llm-evals.yml b/.github/workflows/llm-evals.yml index aaa5a13a1..249afa96b 100644 --- a/.github/workflows/llm-evals.yml +++ b/.github/workflows/llm-evals.yml @@ -11,7 +11,7 @@ permissions: env: EVAL_MODELS: gpt-4.1 RAILS_ENV: test - DATABASE_URL: postgres://postgres:postgres@localhost:5432 + DATABASE_URL: postgres://postgres:postgres@localhost:5432 # pipelock:ignore REDIS_URL: redis://localhost:6379 PLAID_CLIENT_ID: foo PLAID_SECRET: bar diff --git a/.github/workflows/pipelock.yml b/.github/workflows/pipelock.yml index f19d3b94f..d0e7c72cb 100644 --- a/.github/workflows/pipelock.yml +++ b/.github/workflows/pipelock.yml @@ -17,8 +17,9 @@ jobs: persist-credentials: false - name: Pipelock Scan - uses: luckyPipewrench/pipelock@cef4f47eb99ffe00e20fa7d1423bff1a44742dbe # v2.4.0 + uses: luckyPipewrench/pipelock@818ca0a7af4dbcd56ada7fa57e2dc32f9e799e34 # v2.8.0 with: + version: '2.8.0' scan-diff: 'true' fail-on-findings: 'true' test-vectors: 'false' @@ -30,3 +31,6 @@ jobs: docs/hosting/ai.md app/models/provider/binance.rb workers/preview/package-lock.json + # Preview Dockerfile uses local dev credentials (rails/rails) that are + # not real secrets; Dockerfile format does not support inline # pipelock:ignore + Dockerfile.preview diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index edd1e2f6e..5c9e4fd50 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,6 +1,7 @@ name: Pull Request on: + workflow_dispatch: pull_request: types: [opened, synchronize, reopened, labeled] paths-ignore: diff --git a/.github/workflows/preview-cleanup.yml b/.github/workflows/preview-cleanup.yml index 60f21c8d9..052708868 100644 --- a/.github/workflows/preview-cleanup.yml +++ b/.github/workflows/preview-cleanup.yml @@ -24,7 +24,12 @@ permissions: jobs: cleanup-on-close: name: Cleanup closed PR preview - if: github.event_name == 'pull_request' && (github.event.action == 'closed' || (github.event.action == 'unlabeled' && github.event.label.name == 'preview-cf')) + if: | + github.event_name == 'pull_request' && + ( + (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'preview-cf')) || + (github.event.action == 'unlabeled' && github.event.label.name == 'preview-cf') + ) runs-on: ubuntu-latest timeout-minutes: 10 @@ -34,27 +39,51 @@ jobs: with: persist-credentials: false - - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 - with: - node-version: "24" - - - name: Install Wrangler - run: npm install -g wrangler - - name: Delete preview Worker env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_PREVIEW_API_TOKEN || secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} run: | + set -euo pipefail + WORKER_NAME="sure-preview-${{ github.event.pull_request.number }}" echo "Deleting Worker: $WORKER_NAME" + if [ -z "${CLOUDFLARE_API_TOKEN:-}" ] || [ -z "${CLOUDFLARE_ACCOUNT_ID:-}" ]; then + echo "Missing Cloudflare credentials; refusing to mark preview cleanup as successful" + exit 1 + fi + # Delete the worker (this also stops any running containers) - wrangler delete --name "$WORKER_NAME" --force || echo "Worker may not exist" + response_file="$(mktemp)" + http_status="$(curl -sS -o "$response_file" -w "%{http_code}" -X DELETE \ + "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/workers/scripts/$WORKER_NAME?force=true" \ + -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \ + -H "Content-Type: application/json")" || { + echo "Failed to delete preview Worker via Cloudflare API" + exit 1 + } + response="$(cat "$response_file")" + + if [ -z "$response" ] && [[ "$http_status" =~ ^2 ]]; then + echo "Deleted Worker: $WORKER_NAME" + elif [ -n "$response" ] && echo "$response" | jq -e '.success == true' >/dev/null; then + echo "Deleted Worker: $WORKER_NAME" + elif [ -n "$response" ] && echo "$response" | jq -e '[.errors[]?.code] | index(10007)' >/dev/null; then + echo "$response" | jq -c '.errors' + echo "Worker may not exist" + else + echo "Cloudflare API failed to delete preview Worker (HTTP $http_status)" + if [ -n "$response" ]; then + echo "$response" | jq -c '.errors // .' || printf '%s\n' "$response" + else + echo "No response body" + fi + exit 1 + fi - name: Delete GitHub Deployment - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const environment = `preview-pr-${{ github.event.pull_request.number }}`; @@ -88,7 +117,7 @@ jobs: cleanup-expired: name: Cleanup expired previews - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + if: github.repository == 'we-promise/sure' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/.github/workflows/preview-deploy.yml b/.github/workflows/preview-deploy.yml index 63ea9590d..3064ab9f6 100644 --- a/.github/workflows/preview-deploy.yml +++ b/.github/workflows/preview-deploy.yml @@ -25,6 +25,7 @@ jobs: head_sha: ${{ steps.preview.outputs.head_sha }} is_fork: ${{ steps.preview.outputs.is_fork }} pr_number: ${{ steps.preview.outputs.pr_number }} + resolution_source: ${{ steps.preview.outputs.resolution_source }} should_deploy: ${{ steps.preview.outputs.should_deploy }} steps: @@ -39,7 +40,7 @@ jobs: - name: Resolve preview request id: preview - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const { resolvePreviewRequest } = require('./trusted-preview-resolver/workers/preview/deploy/resolve_preview_request.cjs'); @@ -65,7 +66,7 @@ jobs: - name: Create GitHub Deployment if: env.IS_FORK == 'false' id: deployment - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const prNumber = process.env.PR_NUMBER; @@ -105,6 +106,7 @@ jobs: HEAD_SHA: ${{ needs.preview-gate.outputs.head_sha }} IS_FORK: ${{ needs.preview-gate.outputs.is_fork }} PR_NUMBER: ${{ needs.preview-gate.outputs.pr_number }} + RESOLUTION_SOURCE: ${{ needs.preview-gate.outputs.resolution_source }} steps: - name: Checkout trusted preview tooling @@ -202,11 +204,16 @@ jobs: cp trusted/workers/preview/tsconfig.json "$preview_dir/tsconfig.json" cp trusted/workers/preview/wrangler.toml "$preview_dir/wrangler.toml" cp -R trusted/workers/preview/src "$preview_dir/src" + mkdir -p "$preview_dir/deploy" + cp trusted/workers/preview/deploy/redact_preview_log.sh "$preview_dir/deploy/redact_preview_log.sh" + cp trusted/workers/preview/deploy/render_preview_config.cjs "$preview_dir/deploy/render_preview_config.cjs" + chmod 0755 "$preview_dir/deploy/redact_preview_log.sh" diagnostics_nonce="$(openssl rand -hex 32)" sed -i "s/\${PR_NUMBER}/${PR_NUMBER}/g" "$preview_dir/wrangler.toml" sed -i "s/\${PR_NUMBER}/${PR_NUMBER}/g" "$preview_dir/src/index.ts" sed -i "s/\${PREVIEW_DIAGNOSTICS_NONCE}/${diagnostics_nonce}/g" "$preview_dir/src/index.ts" + cp "$preview_dir/wrangler.toml" "$preview_dir/wrangler.source.toml" if grep -F "\${PREVIEW_DIAGNOSTICS_NONCE}" "$preview_dir/src/index.ts" >/dev/null; then echo "Preview diagnostics nonce placeholder was not replaced" >&2 @@ -243,34 +250,30 @@ jobs: set -euo pipefail cd "$RUNNER_TEMP/sure-preview-worker" + source_config="$RUNNER_TEMP/sure-preview-worker/wrangler.source.toml" config_path="$RUNNER_TEMP/sure-preview-worker/wrangler.toml" image_tag="sure-preview-pr-${PR_NUMBER}:${HEAD_SHA}" + temporary_image_ref="registry.cloudflare.com/${CLOUDFLARE_ACCOUNT_ID}/${image_tag}" push_log="$RUNNER_TEMP/wrangler-containers-push.log" clean_log="$RUNNER_TEMP/wrangler-containers-push.clean.log" + push_status=0 # wrangler containers push validates wrangler.toml, so point the trusted - # config at the loaded CI image before replacing it with the registry ref. - LOCAL_IMAGE_TAG="$image_tag" node - "$config_path" <<'NODE' - const fs = require('node:fs'); + # config at a registry-shaped ref while it pushes the verified local image. + PREVIEW_IMAGE_REF="$temporary_image_ref" node ./deploy/render_preview_config.cjs render "$source_config" "$config_path" + cp "$config_path" "$RUNNER_TEMP/wrangler-push.toml" - const configPath = process.argv[2]; - const imageTag = process.env.LOCAL_IMAGE_TAG; - - if (!/^sure-preview-pr-[1-9][0-9]*:[a-f0-9]{40}$/.test(imageTag || '')) { - throw new Error('Expected local preview image tag for wrangler containers push'); - } - - const original = fs.readFileSync(configPath, 'utf8'); - const updated = original.replace(/image = "[^"]+"/, `image = ${JSON.stringify(imageTag)}`); - if (updated === original) { - throw new Error('Expected wrangler.toml to contain an image entry to rewrite before push'); - } - fs.writeFileSync(configPath, updated); - NODE - - ./node_modules/.bin/wrangler containers push "$image_tag" 2>&1 | tee "$push_log" + set +e + ./node_modules/.bin/wrangler containers push "$image_tag" 2>&1 | tee "$push_log" | ./deploy/redact_preview_log.sh + push_status=${PIPESTATUS[0]} + set -e perl -pe 's/\e\[[0-9;]*[A-Za-z]//g' "$push_log" > "$clean_log" - image_ref=$(grep -Eo 'registry\.cloudflare\.com/[^[:space:]]+' "$clean_log" | tail -n 1 | tr -d '\r') + + if [ "$push_status" -ne 0 ]; then + exit "$push_status" + fi + + image_ref="$(node ./deploy/render_preview_config.cjs find "$clean_log")" if [ -z "$image_ref" ]; then echo "Could not find Cloudflare registry image reference in wrangler output" >&2 @@ -285,27 +288,17 @@ jobs: run: | set -euo pipefail + source_config="$RUNNER_TEMP/sure-preview-worker/wrangler.source.toml" config_path="$RUNNER_TEMP/sure-preview-worker/wrangler.toml" - # Use Node instead of sed so the replacement preserves TOML string syntax. - node - "$config_path" <<'NODE' - const fs = require('node:fs'); + # Render from the preserved trusted source template so the push-time + # registry ref cannot make the final deploy rewrite stateful. + PREVIEW_IMAGE_REF="$IMAGE_REF" node "$RUNNER_TEMP/sure-preview-worker/deploy/render_preview_config.cjs" render "$source_config" "$config_path" + cp "$config_path" "$RUNNER_TEMP/wrangler-final.toml" - const configPath = process.argv[2]; - const imageRef = process.env.IMAGE_REF; - - if (!imageRef || !imageRef.startsWith('registry.cloudflare.com/')) { - throw new Error('Expected a Cloudflare registry image reference'); - } - - const original = fs.readFileSync(configPath, 'utf8'); - const updated = original.replace(/image = "[^"]+"/, `image = ${JSON.stringify(imageRef)}`); - if (updated === original) { - throw new Error('Expected wrangler.toml to contain an image entry to rewrite'); - } - fs.writeFileSync(configPath, updated); - NODE - - cat "$config_path" + # Print a redacted copy for logs without mutating the config used by deploy. + redacted_config="$RUNNER_TEMP/wrangler-redacted.toml" + "$RUNNER_TEMP/sure-preview-worker/deploy/redact_preview_log.sh" < "$config_path" > "$redacted_config" + cat "$redacted_config" - name: Deploy to Cloudflare Containers id: deploy @@ -321,7 +314,11 @@ jobs: clean_deploy_log="$RUNNER_TEMP/wrangler-deploy.clean.log" deploy_once() { - ./node_modules/.bin/wrangler deploy --config wrangler.toml --var "PR_NUMBER:${PR_NUMBER}" 2>&1 | tee "$deploy_log" + set +e + ./node_modules/.bin/wrangler deploy --config wrangler.toml --var "PR_NUMBER:${PR_NUMBER}" 2>&1 | tee "$deploy_log" | ./deploy/redact_preview_log.sh + local deploy_status=${PIPESTATUS[0]} + set -e + return "$deploy_status" } if ! deploy_once; then @@ -356,27 +353,72 @@ jobs: run: | set -euo pipefail - diagnostics_file="$RUNNER_TEMP/preview-diagnostics.json" + diagnostics_dir="$RUNNER_TEMP/preview-diagnostics" + diagnostics_file="$diagnostics_dir/preview-diagnostics.json" + latest_metrics_file="$diagnostics_dir/latest-metrics.json" + polls_log="$diagnostics_dir/metrics-polls.log" + summary_file="$diagnostics_dir/summary.md" last_error="" + mkdir -p "$diagnostics_dir" - for attempt in $(seq 1 40); do + # ~100 polls x ~3.5s = up to ~6 min. The preview must cold-start the + # container (≈1.3 GB image pull + sandbox init) and generate the full + # demo dataset before previewReady flips. A real Cloudflare standard-1 + # run measured previewReady at ~195s (rails ~46s, demo data ~149s) — + # the old 40-poll (~140s) budget would have failed a working preview. + # Keep generous headroom; the loop still breaks early on ready/failed. + for attempt in $(seq 1 100); do if curl -fsS --connect-timeout 5 --max-time 15 "$PREVIEW_URL/_container_status" -o "$diagnostics_file"; then - if jq -e '.previewReady == true or .previewFailed == true' "$diagnostics_file" >/dev/null; then - break + if jq -e . "$diagnostics_file" >/dev/null 2>&1; then + jq -c --argjson attempt "$attempt" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{attempt: $attempt, at: $at, previewReady: (.previewReady // false), previewFailed: (.previewFailed // false), progress: (.progress // {}), timings: (.timings // {})}' \ + "$diagnostics_file" >> "$polls_log" + jq '{previewReady: (.previewReady // false), previewFailed: (.previewFailed // false), progress: (.progress // {}), timings: (.timings // {})}' "$diagnostics_file" > "$latest_metrics_file" + + if jq -e '.previewReady == true or .previewFailed == true' "$diagnostics_file" >/dev/null; then + break + fi + else + last_error="invalid diagnostics JSON on attempt ${attempt}" + raw_snippet="$(head -c 2048 "$diagnostics_file")" + latest_metrics_snapshot="none" + if [ -f "$latest_metrics_file" ]; then + latest_metrics_snapshot="$(head -c 2048 "$latest_metrics_file")" + fi + jq -nc --argjson attempt "$attempt" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg error "$last_error" --arg latestMetrics "$latest_metrics_snapshot" --arg rawSnippet "$raw_snippet" \ + '{attempt: $attempt, at: $at, error: $error, latestMetrics: $latestMetrics, rawSnippet: $rawSnippet}' >> "$polls_log" fi else last_error="curl failed on attempt ${attempt}" + jq -nc --argjson attempt "$attempt" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg error "$last_error" \ + '{attempt: $attempt, at: $at, error: $error}' >> "$polls_log" fi sleep 3 done - if [ ! -s "$diagnostics_file" ]; then + if [ ! -s "$diagnostics_file" ] || ! jq -e . "$diagnostics_file" >/dev/null 2>&1; then jq -n --arg error "${last_error:-preview diagnostics unavailable}" \ --arg url "$PREVIEW_URL" \ '{previewReady: false, previewFailed: false, error: $error, previewUrl: $url}' > "$diagnostics_file" fi + jq '{previewReady: (.previewReady // false), previewFailed: (.previewFailed // false), progress: (.progress // {}), timings: (.timings // {}), error: (.error // null)}' "$diagnostics_file" > "$latest_metrics_file" + { + echo "# Preview diagnostics" + echo + echo "- PR: ${PR_NUMBER}" + echo "- Commit: ${HEAD_SHA}" + echo "- Preview URL: ${PREVIEW_URL}" + echo "- Preview ready: $(jq -r '.previewReady // false' "$diagnostics_file")" + echo "- Preview failed: $(jq -r '.previewFailed // false' "$diagnostics_file")" + echo "- Phase: $(jq -r '.progress.phase // "unknown"' "$diagnostics_file")" + echo "- Stage: $(jq -r '.progress.stage // "unknown"' "$diagnostics_file")" + echo "- Seconds to Rails ready: $(jq -r '.timings.secondsToRailsReady // "unknown"' "$diagnostics_file")" + echo "- Seconds to demo data ready: $(jq -r '.timings.secondsToDemoDataReady // "unknown"' "$diagnostics_file")" + echo "- Seconds to preview ready: $(jq -r '.timings.secondsToPreviewReady // "unknown"' "$diagnostics_file")" + } > "$summary_file" + jq -c . "$diagnostics_file" if jq -e '.previewFailed == true' "$diagnostics_file" >/dev/null; then @@ -402,16 +444,97 @@ jobs: uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: preview-diagnostics-pr-${{ env.PR_NUMBER }}-${{ env.HEAD_SHA }} - path: ${{ runner.temp }}/preview-diagnostics.json + path: ${{ runner.temp }}/preview-diagnostics if-no-files-found: error retention-days: 3 + - name: Collect preview failure diagnostics + if: failure() + run: | + set -euo pipefail + + diagnostics_dir="$RUNNER_TEMP/preview-failure-diagnostics" + manifest_file="$RUNNER_TEMP/preview-image/sure-preview-image.manifest.json" + redaction_helper="$RUNNER_TEMP/sure-preview-worker/deploy/redact_preview_log.sh" + mkdir -p "$diagnostics_dir" + + jq -n \ + --arg artifactName "$ARTIFACT_NAME" \ + --arg headSha "$HEAD_SHA" \ + --arg isFork "$IS_FORK" \ + --arg prNumber "$PR_NUMBER" \ + --arg resolutionSource "$RESOLUTION_SOURCE" \ + '{ + artifactName: $artifactName, + headSha: $headSha, + isFork: $isFork, + prNumber: $prNumber, + resolutionSource: $resolutionSource + }' > "$diagnostics_dir/preview-request.json" + + sanitize_copy() { + local source="$1" + local destination="$2" + if [ -f "$source" ]; then + if [ -x "$redaction_helper" ]; then + "$redaction_helper" < "$source" > "$destination" + else + cp "$source" "$destination" + fi + fi + } + + if [ -f "$manifest_file" ]; then + jq '{ + artifactVersion, + archivePath, + archiveSha256, + headSha, + imageId, + imageTag, + prNumber + }' "$manifest_file" > "$diagnostics_dir/preview-image-manifest.json" + fi + + sanitize_copy "$RUNNER_TEMP/sure-preview-worker/wrangler.source.toml" "$diagnostics_dir/wrangler-source.toml" + sanitize_copy "$RUNNER_TEMP/wrangler-push.toml" "$diagnostics_dir/wrangler-push.toml" + sanitize_copy "$RUNNER_TEMP/wrangler-final.toml" "$diagnostics_dir/wrangler-final.toml" + sanitize_copy "$RUNNER_TEMP/sure-preview-worker/wrangler.toml" "$diagnostics_dir/wrangler.toml" + sanitize_copy "$RUNNER_TEMP/wrangler-containers-push.clean.log" "$diagnostics_dir/wrangler-containers-push.log" + if [ -f "$RUNNER_TEMP/wrangler-deploy.clean.log" ]; then + sanitize_copy "$RUNNER_TEMP/wrangler-deploy.clean.log" "$diagnostics_dir/wrangler-deploy.log" + else + sanitize_copy "$RUNNER_TEMP/wrangler-deploy.log" "$diagnostics_dir/wrangler-deploy.log" + fi + + find "$diagnostics_dir" -maxdepth 1 -type f -print + + - name: Upload preview failure diagnostics + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: preview-failure-diagnostics-pr-${{ env.PR_NUMBER }}-${{ env.HEAD_SHA }} + path: ${{ runner.temp }}/preview-failure-diagnostics + if-no-files-found: error + retention-days: 3 + + - name: Prepare cleanup metadata + if: success() + run: | + set -euo pipefail + + metadata_dir="$RUNNER_TEMP/preview-cleanup-metadata" + mkdir -p "$metadata_dir" + "$RUNNER_TEMP/sure-preview-worker/deploy/redact_preview_log.sh" \ + < "$RUNNER_TEMP/sure-preview-worker/wrangler.toml" \ + > "$metadata_dir/wrangler.toml" + - name: Store cleanup metadata if: success() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: preview-cleanup-pr-${{ env.PR_NUMBER }} - path: ${{ runner.temp }}/sure-preview-worker/wrangler.toml + path: ${{ runner.temp }}/preview-cleanup-metadata/wrangler.toml retention-days: 2 deployment_status: @@ -434,7 +557,7 @@ jobs: steps: - name: Update Deployment Status - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const state = process.env.DEPLOY_RESULT === 'success' ? 'success' : 'failure'; @@ -464,7 +587,7 @@ jobs: steps: - name: Comment on PR - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const previewUrl = process.env.PREVIEW_URL; diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a30609b0..6a93d58f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -199,6 +199,7 @@ jobs: env: TAGS: ${{ needs.build.outputs.tags }} DIGESTS_DIR: ${{ runner.temp }}/digests + REF_NAME: ${{ github.ref_name }} shell: bash -xeuo pipefail {0} run: | tag_args=() @@ -216,7 +217,7 @@ jobs: "index:org.opencontainers.image.created=$(date -Iseconds)" 'index:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}' 'index:org.opencontainers.image.revision=${{ github.sha }}' - 'index:org.opencontainers.image.ref.name=${{ github.ref_name }}' + "index:org.opencontainers.image.ref.name=${REF_NAME}" 'index:org.opencontainers.image.vendor=we-promise' 'index:org.opencontainers.image.licenses=AGPL-3.0' 'index:org.opencontainers.image.title=Sure' @@ -264,10 +265,16 @@ jobs: uses: ./.github/workflows/flutter-build.yml secrets: inherit + desktop: + name: Build macOS Desktop App + if: startsWith(github.ref, 'refs/tags/v') + uses: ./.github/workflows/desktop-release.yml + secrets: inherit + release: name: Create GitHub Release if: startsWith(github.ref, 'refs/tags/v') - needs: [merge, mobile, helm] + needs: [merge, mobile, desktop, helm] runs-on: ubuntu-latest timeout-minutes: 10 @@ -293,7 +300,15 @@ jobs: name: helm-chart-package path: ${{ runner.temp }}/helm-artifacts + - name: Download desktop DMG artifact + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: desktop-release-dmg + path: ${{ runner.temp }}/desktop-artifacts + - name: Prepare release assets + env: + REF_NAME: ${{ github.ref_name }} run: | mkdir -p ${{ runner.temp }}/release-assets @@ -306,13 +321,13 @@ jobs: # Copy debug APK if it exists if [ -f "${{ runner.temp }}/mobile-artifacts/app-debug.apk" ]; then - cp "${{ runner.temp }}/mobile-artifacts/app-debug.apk" "${{ runner.temp }}/release-assets/sure-${{ github.ref_name }}-debug.apk" + cp "${{ runner.temp }}/mobile-artifacts/app-debug.apk" "${{ runner.temp }}/release-assets/sure-${REF_NAME}-debug.apk" echo "✓ Debug APK prepared" fi # Copy release APK if it exists if [ -f "${{ runner.temp }}/mobile-artifacts/app-release.apk" ]; then - cp "${{ runner.temp }}/mobile-artifacts/app-release.apk" "${{ runner.temp }}/release-assets/sure-${{ github.ref_name }}.apk" + cp "${{ runner.temp }}/mobile-artifacts/app-release.apk" "${{ runner.temp }}/release-assets/sure-${REF_NAME}.apk" echo "✓ Release APK prepared" fi @@ -320,7 +335,7 @@ jobs: # Path preserves directory structure from artifact upload if [ -d "${{ runner.temp }}/ios-build/ios/iphoneos/Runner.app" ]; then cd "${{ runner.temp }}/ios-build/ios/iphoneos" - zip -r "${{ runner.temp }}/release-assets/sure-${{ github.ref_name }}-ios-unsigned.zip" Runner.app + zip -r "${{ runner.temp }}/release-assets/sure-${REF_NAME}-ios-unsigned.zip" Runner.app echo "✓ iOS build archive prepared" fi @@ -335,6 +350,12 @@ jobs: echo "✓ Helm chart package prepared" fi + # Copy the universal macOS desktop build. + if compgen -G "${{ runner.temp }}/desktop-artifacts/*.dmg" > /dev/null; then + cp ${{ runner.temp }}/desktop-artifacts/*.dmg "${{ runner.temp }}/release-assets/" + echo "✓ Desktop DMG prepared" + fi + echo "Release assets:" ls -la "${{ runner.temp }}/release-assets/" @@ -398,7 +419,9 @@ jobs: timeout-minutes: 10 permissions: + actions: write contents: write + pull-requests: write steps: - name: Determine source branch for tag @@ -433,71 +456,66 @@ jobs: - name: Bump pre-release version run: | - VERSION_FILE=".sure-version" - CHART_FILE="charts/sure/Chart.yaml" + set -euo pipefail - # Ensure version file exists - if [ ! -f "$VERSION_FILE" ]; then - echo "ERROR: Version file not found: $VERSION_FILE" - exit 1 - fi + VERSION_FILE=".sure-version" + CHART_FILE="charts/sure/Chart.yaml" - # Ensure chart file exists - if [ ! -f "$CHART_FILE" ]; then - echo "ERROR: Chart file not found: $CHART_FILE" - exit 1 - fi - - # Extract current version - CURRENT_VERSION=$(cat "$VERSION_FILE" | tr -d '[:space:]') - if [ -z "$CURRENT_VERSION" ]; then - echo "ERROR: Could not extract version from $VERSION_FILE" - exit 1 - fi - echo "Current version: $CURRENT_VERSION" + # Ensure version file exists + if [ ! -f "$VERSION_FILE" ]; then + echo "ERROR: Version file not found: $VERSION_FILE" + exit 1 + fi - # Extract the pre-release tag and number, then increment it - PRE_RELEASE_TAG=$(echo "$CURRENT_VERSION" | grep -oP '(alpha|beta|rc)') - if [ -z "$PRE_RELEASE_TAG" ]; then - echo "ERROR: Could not extract pre-release tag from $CURRENT_VERSION" - exit 1 - fi + # Ensure chart file exists + if [ ! -f "$CHART_FILE" ]; then + echo "ERROR: Chart file not found: $CHART_FILE" + exit 1 + fi - PRE_RELEASE_NUM=$(echo "$CURRENT_VERSION" | grep -oP '(alpha|beta|rc)\.\K[0-9]+') - if [ -z "$PRE_RELEASE_NUM" ]; then - echo "ERROR: Could not extract pre-release number from $CURRENT_VERSION" - exit 1 - fi - NEW_PRE_RELEASE_NUM=$((PRE_RELEASE_NUM + 1)) - - # Create new version string - BASE_VERSION=$(echo "$CURRENT_VERSION" | grep -oP '^[0-9]+\.[0-9]+\.[0-9]+') - if [ -z "$BASE_VERSION" ]; then - echo "ERROR: Could not extract base version from $CURRENT_VERSION" - exit 1 - fi - NEW_VERSION="${BASE_VERSION}-${PRE_RELEASE_TAG}.${NEW_PRE_RELEASE_NUM}" - echo "New version: $NEW_VERSION" + # Extract current version + CURRENT_VERSION=$(tr -d '[:space:]' < "$VERSION_FILE") + if [ -z "$CURRENT_VERSION" ]; then + echo "ERROR: Could not extract version from $VERSION_FILE" + exit 1 + fi + echo "Current version: $CURRENT_VERSION" - # Update the version file - echo "$NEW_VERSION" > "$VERSION_FILE" + if [[ ! "$CURRENT_VERSION" =~ ^([0-9]+\.[0-9]+\.[0-9]+)-(alpha|beta|rc)\.([0-9]+)$ ]]; then + echo "ERROR: Expected prerelease version like 1.2.3-alpha.4, got $CURRENT_VERSION" + exit 1 + fi - # Verify the change - echo "Updated .sure-version:" - cat "$VERSION_FILE" + BASE_VERSION="${BASH_REMATCH[1]}" + PRE_RELEASE_TAG="${BASH_REMATCH[2]}" + PRE_RELEASE_NUM="${BASH_REMATCH[3]}" + NEW_PRE_RELEASE_NUM=$((PRE_RELEASE_NUM + 1)) + NEW_VERSION="${BASE_VERSION}-${PRE_RELEASE_TAG}.${NEW_PRE_RELEASE_NUM}" + echo "New version: $NEW_VERSION" - # Update Helm chart version and appVersion - sed -i -E "s/^version: .*/version: ${NEW_VERSION}/" "$CHART_FILE" - sed -i -E "s/^appVersion: .*/appVersion: \"${NEW_VERSION}\"/" "$CHART_FILE" + # Update the version file + echo "$NEW_VERSION" > "$VERSION_FILE" - # Verify the change - echo "Updated Chart.yaml:" - grep -E "^(version|appVersion):" "$CHART_FILE" + # Verify the change + echo "Updated .sure-version:" + cat "$VERSION_FILE" + + # Update Helm chart version and appVersion + sed -i -E "s/^version: .*/version: ${NEW_VERSION}/" "$CHART_FILE" + sed -i -E "s/^appVersion: .*/appVersion: \"${NEW_VERSION}\"/" "$CHART_FILE" + + # Verify the change + echo "Updated Chart.yaml:" + grep -E "^(version|appVersion):" "$CHART_FILE" - name: Commit and push version bump env: SOURCE_BRANCH: ${{ steps.source_branch.outputs.branch }} + GH_TOKEN: ${{ github.token }} + REF_NAME: ${{ github.ref_name }} run: | + set -euo pipefail + git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" @@ -510,20 +528,85 @@ jobs: exit 0 fi - git commit -m "Bump version to next iteration after ${{ github.ref_name }} release" + COMMIT_MESSAGE="Bump version to next iteration after ${REF_NAME} release" + git commit -m "$COMMIT_MESSAGE" echo "Pushing to branch: $SOURCE_BRANCH" - # Push with retry logic - attempts=0 - until git push origin HEAD:$SOURCE_BRANCH; do - attempts=$((attempts + 1)) - if [[ $attempts -ge 4 ]]; then - echo "ERROR: Failed to push after 4 attempts." >&2 - exit 1 + # Push directly when allowed. Main is protected and must be updated + # through a pull request. Other branches get a direct-push attempt, but + # protected-branch rejections fall back to a pull request immediately. + push_succeeded=false + if [[ "$SOURCE_BRANCH" == "main" || "$SOURCE_BRANCH" == "master" ]]; then + echo "$SOURCE_BRANCH is protected; creating a pull request instead of pushing directly." + else + direct_push_output="" + for attempt in 1 2 3 4; do + set +e + direct_push_output=$(git push origin HEAD:"refs/heads/${SOURCE_BRANCH}" 2>&1) + push_status=$? + set -e + + echo "$direct_push_output" + + if [[ $push_status -eq 0 ]]; then + push_succeeded=true + break + fi + + if grep -Eq 'GH006|Protected branch update failed|Changes must be made through a pull request' <<< "$direct_push_output"; then + echo "Direct push is blocked by branch protection; creating a pull request instead." + break + fi + + if [[ $attempt -eq 4 ]]; then + echo "Direct push failed after 4 attempts; creating a pull request instead." + break + fi + + delay=$((2 ** attempt)) + echo "Push failed (attempt $attempt). Retrying in ${delay} seconds..." + sleep "$delay" + git fetch origin "${SOURCE_BRANCH}" + git rebase "origin/${SOURCE_BRANCH}" + done + fi + + if [[ "$push_succeeded" == "true" ]]; then + exit 0 + fi + + SAFE_TAG=$(printf '%s' "${REF_NAME}" | tr -c '[:alnum:]._-' '-') + BUMP_BRANCH="automation/bump-version-after-${SAFE_TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + + git push --force-with-lease origin HEAD:"refs/heads/${BUMP_BRANCH}" + + PR_BODY="This automated PR bumps .sure-version and charts/sure/Chart.yaml after the ${REF_NAME} pre-release. It was opened because the workflow could not push directly to ${SOURCE_BRANCH}." + + set +e + pr_create_output=$(gh pr create \ + --repo "$GITHUB_REPOSITORY" \ + --head "$BUMP_BRANCH" \ + --base "$SOURCE_BRANCH" \ + --title "$COMMIT_MESSAGE" \ + --body "$PR_BODY" 2>&1) + pr_create_status=$? + set -e + + if [[ $pr_create_status -ne 0 ]]; then + echo "::error::Pushed ${BUMP_BRANCH}, but could not create the version bump PR: ${pr_create_output}" + if grep -q "GitHub Actions is not permitted to create or approve pull requests" <<< "$pr_create_output"; then + echo "::notice::Enable the organization setting that allows GitHub Actions to create and approve pull requests." fi - delay=$((2 ** attempts)) - echo "Push failed (attempt $attempts). Retrying in ${delay} seconds..." - sleep ${delay} - git pull --rebase origin $SOURCE_BRANCH - done + exit 1 + fi + + PR_URL="$pr_create_output" + echo "Created version bump PR: $PR_URL" + echo "Dispatching PR checks for $BUMP_BRANCH" + if ! gh workflow run pr.yml --repo "$GITHUB_REPOSITORY" --ref "$BUMP_BRANCH"; then + echo "::warning::Could not dispatch pr.yml for $BUMP_BRANCH" + fi + if ! gh workflow run chart-ci.yml --repo "$GITHUB_REPOSITORY" --ref "$BUMP_BRANCH"; then + echo "::warning::Could not dispatch chart-ci.yml for $BUMP_BRANCH" + fi diff --git a/.gitignore b/.gitignore index 19cc81a3a..dd7707feb 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,9 @@ coverage .cursor/rules/structure.mdc .cursor/rules/agent.mdc +# Ignore Redis dump file +dump.rdb + # Ignore node related files node_modules diff --git a/.rubocop.yml b/.rubocop.yml index 1d2341687..d313b5ab5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,6 +1,6 @@ inherit_gem: rubocop-rails-omakase: rubocop.yml - + Layout/IndentationWidth: Enabled: true @@ -12,4 +12,8 @@ Layout/IndentationConsistency: Enabled: true Layout/SpaceInsidePercentLiteralDelimiters: - Enabled: true \ No newline at end of file + Enabled: true + +Layout/SpaceInsideArrayLiteralBrackets: + Exclude: + - "db/schema.rb" diff --git a/.ruby-version b/.ruby-version index 2aa513199..7bcbb3808 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.4.7 +3.4.9 diff --git a/.sure-version b/.sure-version index 845eb31ba..bac2cef88 100644 --- a/.sure-version +++ b/.sure-version @@ -1 +1 @@ -0.7.2-alpha.3 +0.7.3-alpha.7 diff --git a/AGENTS.md b/AGENTS.md index c5359f025..399375924 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,6 +63,15 @@ Reviewers escalate violations of (2)–(3) to close/rewrite; (1) and (4) are req If you need to add a new securities price provider (Tiingo, EODHD, Binance-style crypto, etc.), see [adding-a-securities-provider.md](./docs/llm-guides/adding-a-securities-provider.md) for the full walkthrough — provider class, registry wiring, MIC handling, settings UI, locales, and tests. +## Debug Logging for Provider Syncs + +When a provider sync/import path hits a recoverable error or suspicious partial response that support may need to inspect later, prefer `DebugLogEntry.capture(...)` over `Rails.logger.*`. + +- Record support-relevant diagnostics in the debug log so they surface in the super-admin-friendly `/settings/debug` UI. +- Include `category`, `level`, `message`, `source`, `provider_key`, and useful structured `metadata`. +- Attach `family` and `account_provider` when available so support can filter and trace the affected connection. +- Reserve raw Rails logging for low-value local noise; anything operators may need should go to the debug log. + ## Providers: Pending Transactions and FX Metadata (SimpleFIN/Plaid/Lunchflow) - Pending detection diff --git a/CLAUDE.md b/CLAUDE.md index 4fc9fcf10..a1da3b7bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,6 +118,7 @@ Two primary data ingestion methods: - Set `SIMPLEFIN_INCLUDE_PENDING=0` to disable pending fetching for SimpleFIN. - Set `PLAID_INCLUDE_PENDING=0` to disable pending fetching for Plaid. - Set `SIMPLEFIN_DEBUG_RAW=1` to enable raw payload debug logging. + - Set `UP_DEBUG_RAW=1` to enable raw Up payload debug logging. DEV-ONLY: the dump contains PII and is gated to local environments, so it never logs in managed/production. Provider support notes: - SimpleFIN: supports pending + FX metadata (stored under `extra["simplefin"]`). @@ -132,6 +133,12 @@ Sidekiq handles asynchronous tasks: - AI chat responses (`AssistantResponseJob`) - Scheduled maintenance via sidekiq-cron +### Debug Logging for Provider Syncs +- Prefer `DebugLogEntry.capture(...)` over `Rails.logger.*` for provider sync/import failures, partial responses, and other support-relevant diagnostics. +- Record support-relevant incidents in the super-admin `/settings/debug` UI rather than leaving them only in raw application logs. +- Include `category`, `level`, `message`, `source`, `provider_key`, and structured `metadata`. +- Attach `family` and `account_provider` whenever possible so support can filter to the affected provider connection. + ### Frontend Architecture - **Hotwire Stack**: Turbo + Stimulus for reactive UI without heavy JavaScript - **ViewComponents**: Reusable UI components in `app/components/` diff --git a/Dockerfile b/Dockerfile index c165b18f7..f691c470b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:1 # Make sure RUBY_VERSION matches the Ruby version in .ruby-version and Gemfile -ARG RUBY_VERSION=3.4.7 +ARG RUBY_VERSION=3.4.9 FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim AS base # Rails app lives here diff --git a/Dockerfile.preview b/Dockerfile.preview index b1661d9b3..06a958a5e 100644 --- a/Dockerfile.preview +++ b/Dockerfile.preview @@ -3,7 +3,7 @@ # Preview Dockerfile for Cloudflare Containers # Includes PostgreSQL and Redis for self-contained development testing -ARG RUBY_VERSION=3.4.7 +ARG RUBY_VERSION=3.4.9 FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim AS base WORKDIR /rails @@ -48,13 +48,41 @@ RUN groupadd --system --gid 1000 rails && \ echo "rails ALL=(ALL) NOPASSWD: /usr/bin/pg_ctlcluster, /usr/bin/redis-server" > /etc/sudoers.d/rails && \ chmod 0440 /etc/sudoers.d/rails -# Configure PostgreSQL to allow local connections -RUN PG_HBA=$(find /etc/postgresql -name pg_hba.conf 2>/dev/null | head -1) && \ - if [ -n "$PG_HBA" ]; then \ - echo "local all all trust" > "$PG_HBA" && \ - echo "host all all 127.0.0.1/32 trust" >> "$PG_HBA" && \ - echo "host all all ::1/128 trust" >> "$PG_HBA"; \ - fi +# Configure PostgreSQL to allow local connections. Target the highest-version +# cluster's pg_hba.conf -- the same one the entrypoint starts (ls +# /etc/postgresql | sort -V | tail -1) -- so the trust rules always land on the +# cluster that actually runs, even if multiple majors are present. +RUN PG_VERSION="$(ls /etc/postgresql 2>/dev/null | sort -V | tail -1)" && \ + PG_HBA="/etc/postgresql/${PG_VERSION}/main/pg_hba.conf" && \ + if [ -z "$PG_VERSION" ] || [ ! -f "$PG_HBA" ]; then \ + echo "ERROR: pg_hba.conf not found for cluster '${PG_VERSION:-none}'; cannot configure local trust" >&2; \ + exit 1; \ + fi && \ + echo "local all all trust" > "$PG_HBA" && \ + echo "host all all 127.0.0.1/32 trust" >> "$PG_HBA" && \ + echo "host all all ::1/128 trust" >> "$PG_HBA" + +# Use file-backed dynamic shared memory instead of POSIX /dev/shm. +# Cloudflare Containers provide only a tiny /dev/shm, and PostgreSQL's default +# dynamic_shared_memory_type = posix FATALs on startup there with +# "could not resize shared memory segment ... No space left on device", which +# kills the container before it can serve a port. mmap keeps DSM in the data +# directory ($PGDATA/pg_dynshmem), removing the /dev/shm dependency. Local +# Docker hides this because its default /dev/shm is 64MB. +# +# Select the highest-version cluster's config -- the same one the entrypoint +# starts (ls /etc/postgresql | sort -V | tail -1) -- so the override always +# lands on the cluster that actually runs, even if multiple majors are present. +RUN PG_VERSION="$(ls /etc/postgresql 2>/dev/null | sort -V | tail -1)" && \ + PG_CONF="/etc/postgresql/${PG_VERSION}/main/postgresql.conf" && \ + if [ -z "$PG_VERSION" ] || [ ! -f "$PG_CONF" ]; then \ + echo "ERROR: postgresql.conf not found for cluster '${PG_VERSION:-none}'; cannot disable /dev/shm DSM dependency" >&2; \ + exit 1; \ + fi && \ + sed -i 's/^[[:space:]]*dynamic_shared_memory_type[[:space:]]*=/# &/' "$PG_CONF" && \ + printf '\n# Preview: avoid /dev/shm dependency (small in Cloudflare Containers)\ndynamic_shared_memory_type = mmap\n' >> "$PG_CONF" && \ + grep -qx 'dynamic_shared_memory_type = mmap' "$PG_CONF" && \ + echo "Configured dynamic_shared_memory_type=mmap in $PG_CONF" # Create database directory with correct permissions RUN mkdir -p /var/run/postgresql && \ @@ -72,19 +100,26 @@ set -e cd /rails -emit_status() { +# Diagnostics posts are best-effort and must NEVER stall boot: the worker's +# Durable Object can be unresponsive while it waits for this container's port, +# so an unbounded curl here deadlocks startup against the port check. +emit_status_sync() { if [ -n "$PREVIEW_ORIGIN" ] && [ -n "$PREVIEW_DIAGNOSTICS_NONCE" ]; then local stage="$1" local detail="$2" local payload payload=$(STAGE="$stage" DETAIL="$detail" ruby -rjson -e 'print JSON.generate({stage: ENV.fetch("STAGE"), detail: ENV.fetch("DETAIL", "")})' 2>/dev/null) || return 0 - curl -fsS -X POST "$PREVIEW_ORIGIN/_container_event" \ + curl -fsS --connect-timeout 2 --max-time 5 -X POST "$PREVIEW_ORIGIN/_container_event" \ -H 'content-type: application/json' \ -H "x-preview-diagnostics-nonce: $PREVIEW_DIAGNOSTICS_NONCE" \ --data "$payload" >/dev/null || true fi } +emit_status() { + emit_status_sync "$1" "$2" & +} + summarize_log_tail() { local file="$1" local label="$2" @@ -101,9 +136,90 @@ summarize_log_tail() { cut -c 1-1600 } -trap 'emit_status failed "preview-entrypoint failed on line ${LINENO}"' ERR +fail_preview() { + local detail="$1" + trap - ERR + # Always log to stderr too: the HTTP diagnostics channel can be unreachable + # while the worker's Durable Object is still waiting for this container's + # port, but stderr is captured by Cloudflare container observability, so the + # real reason survives even when the event POST does not. + echo "PREVIEW FAILED: ${detail}" >&2 + emit_status_sync failed "$detail" + exit 1 +} + +postgres_cluster_snapshot() { + local snapshot="" + local cluster_status + local postgres_log + + if command -v pg_lsclusters >/dev/null 2>&1; then + cluster_status="$(pg_lsclusters 2>&1 | tr '\n' '|' | sed 's/"/'"'"'/g' | cut -c 1-500)" + snapshot="clusters=${cluster_status}" + fi + + postgres_log="/var/log/postgresql/postgresql-${PG_VERSION}-main.log" + if [ -f "$postgres_log" ]; then + snapshot="${snapshot} log=$(summarize_log_tail "$postgres_log" postgres)" + fi + + printf '%s' "$snapshot" +} + +trap 'emit_status_sync failed "preview-entrypoint failed on line ${LINENO}"' ERR emit_status boot "preview-entrypoint started" +# Bind :3000 immediately with a tiny placeholder responder. Cloudflare's +# container supervisor only waits ~20s for the port, while the full stack +# (redis, postgres, migrations) needs 60s+ on a basic instance. The +# placeholder answers 503 with a meta-refresh; the worker still gates +# previewReady on the real Rails /up probe and sample data, so readiness +# semantics are unchanged. It is replaced by the real server below. +ruby -rsocket -e ' + server = TCPServer.new("0.0.0.0", 3000) + body = "Sure preview warmingSure preview is booting (database setup in progress). This page refreshes automatically." + loop do + client = begin + server.accept + rescue StandardError + next + end + begin + # Never read from the client: the response is static, and a blocking read + # would let one silent connection (e.g. a bare TCP port probe) wedge this + # single-threaded loop and starve every later probe. + client.write("HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: #{body.bytesize}\r\nRetry-After: 3\r\nConnection: close\r\n\r\n#{body}") + rescue StandardError + ensure + begin + client.close + rescue StandardError + end + end + end +' >/tmp/port-placeholder.log 2>&1 & +PLACEHOLDER_PID=$! + +# Confirm the placeholder actually bound :3000 before reporting it. A plain +# TCP connect is used because the placeholder intentionally answers 503. +PLACEHOLDER_BOUND=0 +for i in {1..10}; do + if (exec 3<>/dev/tcp/127.0.0.1/3000) 2>/dev/null; then + PLACEHOLDER_BOUND=1 + break + fi + if ! kill -0 "$PLACEHOLDER_PID" 2>/dev/null; then + break + fi + sleep 0.2 +done +if [ "$PLACEHOLDER_BOUND" -eq 1 ]; then + emit_status port-placeholder "bound :3000 placeholder responder (pid ${PLACEHOLDER_PID})" +else + echo "Warning: :3000 placeholder did not come up; continuing boot" >&2 + emit_status port-placeholder-missing "placeholder responder failed to bind :3000: $(summarize_log_tail /tmp/port-placeholder.log port-placeholder)" +fi + REDIS_READY=0 POSTGRES_READY=0 @@ -125,8 +241,7 @@ for i in {1..10}; do done if [ "$REDIS_READY" -ne 1 ]; then - echo "Redis did not become ready in time" - exit 1 + fail_preview "redis did not become ready in time" fi # Start PostgreSQL @@ -134,13 +249,15 @@ echo "Starting PostgreSQL..." emit_status postgres-start "starting postgres" PG_VERSION=$(ls /etc/postgresql/ | sort -V | tail -1) if [ -z "$PG_VERSION" ]; then - echo "Could not determine installed PostgreSQL version" - exit 1 + fail_preview "could not determine installed PostgreSQL version" fi if sudo pg_ctlcluster --skip-systemctl-redirect "$PG_VERSION" main status > /dev/null 2>&1; then emit_status postgres-already-running "postgres cluster already running" else - sudo pg_ctlcluster --skip-systemctl-redirect "$PG_VERSION" main start + POSTGRES_START_LOG=/tmp/postgres-start.log + if ! sudo pg_ctlcluster --skip-systemctl-redirect "$PG_VERSION" main start >"$POSTGRES_START_LOG" 2>&1; then + fail_preview "pg_ctlcluster start failed: $(summarize_log_tail "$POSTGRES_START_LOG" pg_ctlcluster-start) | $(postgres_cluster_snapshot)" + fi fi # Wait for PostgreSQL to be ready @@ -156,8 +273,7 @@ for i in {1..30}; do done if [ "$POSTGRES_READY" -ne 1 ]; then - echo "PostgreSQL did not become ready in time" - exit 1 + fail_preview "postgres did not become ready in time: $(postgres_cluster_snapshot)" fi # Create database user and database if they don't exist @@ -205,6 +321,11 @@ else emit_status demo-data-deferred "deferring demo data creation until after rails boot" fi +# Release :3000 for the real server. The brief listener gap is handled by the +# worker, which catches containerFetch errors and serves its wait page. +kill "$PLACEHOLDER_PID" 2>/dev/null || true +wait "$PLACEHOLDER_PID" 2>/dev/null || true + # Execute the main command with an internal readiness probe echo "Starting Rails server..." emit_status rails-start "starting rails server" diff --git a/Gemfile b/Gemfile index a58e0ccf9..f38a1a3a3 100644 --- a/Gemfile +++ b/Gemfile @@ -3,14 +3,14 @@ source "https://rubygems.org" ruby file: ".ruby-version" # Rails -gem "rails", "~> 7.2.2" +gem "rails", "~> 8.1.0" # Drivers gem "pg", "~> 1.5" gem "redis", "~> 5.4" # Deployment -gem "puma", ">= 5.0" +gem "puma", ">= 7.2.1" gem "bootsnap", require: false # Assets @@ -59,7 +59,6 @@ gem "image_processing", ">= 1.2" gem "ostruct" gem "bcrypt", "~> 3.1" gem "jwt" -gem "ed25519" # For Coinbase CDP API authentication gem "jbuilder" gem "countries" @@ -82,7 +81,6 @@ gem "rchardet" # Character encoding detection gem "redcarpet" gem "stripe" gem "plaid" -gem "snaptrade", "~> 2.0" gem "httparty" gem "rotp", "~> 6.3" gem "rqrcode", "~> 3.0" @@ -93,7 +91,7 @@ gem "pdf-reader", "~> 2.12" # OpenID Connect, OAuth & SAML authentication gem "omniauth", "~> 2.1" -gem "omniauth-rails_csrf_protection" +gem "omniauth-rails_csrf_protection", ">= 2.0" gem "omniauth_openid_connect" gem "omniauth-google-oauth2" gem "omniauth-github" diff --git a/Gemfile.lock b/Gemfile.lock index 6bc0b988d..e929585e1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,83 +4,84 @@ GEM Ascii85 (2.0.1) aasm (5.5.1) concurrent-ruby (~> 1.0) - actioncable (7.2.3.1) - actionpack (= 7.2.3.1) - activesupport (= 7.2.3.1) + action_text-trix (2.1.19) + railties + actioncable (8.1.3.1) + actionpack (= 8.1.3.1) + activesupport (= 8.1.3.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.2.3.1) - actionpack (= 7.2.3.1) - activejob (= 7.2.3.1) - activerecord (= 7.2.3.1) - activestorage (= 7.2.3.1) - activesupport (= 7.2.3.1) + actionmailbox (8.1.3.1) + actionpack (= 8.1.3.1) + activejob (= 8.1.3.1) + activerecord (= 8.1.3.1) + activestorage (= 8.1.3.1) + activesupport (= 8.1.3.1) mail (>= 2.8.0) - actionmailer (7.2.3.1) - actionpack (= 7.2.3.1) - actionview (= 7.2.3.1) - activejob (= 7.2.3.1) - activesupport (= 7.2.3.1) + actionmailer (8.1.3.1) + actionpack (= 8.1.3.1) + actionview (= 8.1.3.1) + activejob (= 8.1.3.1) + activesupport (= 8.1.3.1) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.2.3.1) - actionview (= 7.2.3.1) - activesupport (= 7.2.3.1) - cgi + actionpack (8.1.3.1) + actionview (= 8.1.3.1) + activesupport (= 8.1.3.1) nokogiri (>= 1.8.5) - racc - rack (>= 2.2.4, < 3.3) + rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (7.2.3.1) - actionpack (= 7.2.3.1) - activerecord (= 7.2.3.1) - activestorage (= 7.2.3.1) - activesupport (= 7.2.3.1) + actiontext (8.1.3.1) + action_text-trix (~> 2.1.15) + actionpack (= 8.1.3.1) + activerecord (= 8.1.3.1) + activestorage (= 8.1.3.1) + activesupport (= 8.1.3.1) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.2.3.1) - activesupport (= 7.2.3.1) + actionview (8.1.3.1) + activesupport (= 8.1.3.1) builder (~> 3.1) - cgi erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.2.3.1) - activesupport (= 7.2.3.1) + activejob (8.1.3.1) + activesupport (= 8.1.3.1) globalid (>= 0.3.6) - activemodel (7.2.3.1) - activesupport (= 7.2.3.1) - activerecord (7.2.3.1) - activemodel (= 7.2.3.1) - activesupport (= 7.2.3.1) + activemodel (8.1.3.1) + activesupport (= 8.1.3.1) + activerecord (8.1.3.1) + activemodel (= 8.1.3.1) + activesupport (= 8.1.3.1) timeout (>= 0.4.0) activerecord-import (2.2.0) activerecord (>= 4.2) - activestorage (7.2.3.1) - actionpack (= 7.2.3.1) - activejob (= 7.2.3.1) - activerecord (= 7.2.3.1) - activesupport (= 7.2.3.1) + activestorage (8.1.3.1) + actionpack (= 8.1.3.1) + activejob (= 8.1.3.1) + activerecord (= 8.1.3.1) + activesupport (= 8.1.3.1) marcel (~> 1.0) - activesupport (7.2.3.1) + activesupport (8.1.3.1) base64 - benchmark (>= 0.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + json logger (>= 1.4.2) - minitest (>= 5.1, < 6) + minitest (>= 5.1) securerandom (>= 0.3) tzinfo (~> 2.0, >= 2.0.5) - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) + uri (>= 0.13.1) + addressable (2.9.0) + public_suffix (>= 2.0.2, < 8.0) aes_key_wrap (1.1.0) afm (1.0.0) after_commit_everywhere (1.6.0) @@ -93,6 +94,8 @@ GEM standardwebhooks ast (2.4.3) attr_required (1.0.2) + auth-sanitizer (0.2.3) + version_gem (~> 1.1, >= 1.1.14) aws-eventstream (1.4.0) aws-partitions (1.1196.0) aws-sdk-core (3.240.0) @@ -146,7 +149,7 @@ GEM logger (~> 1.5) chunky_png (1.4.0) climate_control (1.2.0) - concurrent-ruby (1.3.6) + concurrent-ruby (1.3.7) connection_pool (2.5.5) cose (1.3.1) cbor (~> 0.5.9) @@ -156,14 +159,15 @@ GEM crack (1.0.0) bigdecimal rexml - crass (1.0.6) + crass (1.0.7) cronex (0.15.0) tzinfo unicode (>= 0.4.4.5) - css_parser (1.21.1) + css_parser (3.0.0) addressable + ssrf_filter (~> 1.5) csv (3.3.5) - date (3.4.1) + date (3.5.1) debug (1.11.0) irb (~> 1.10) reline (>= 0.3.8) @@ -197,10 +201,9 @@ GEM dotenv (= 3.1.8) railties (>= 6.1) drb (2.2.3) - ed25519 (1.4.0) email_validator (2.2.4) activemodel - erb (5.0.1) + erb (6.0.4) erb_lint (0.9.0) activesupport better_html (>= 2.0.1) @@ -214,7 +217,7 @@ GEM event_stream_parser (1.0.0) faker (3.5.2) i18n (>= 1.8.11, < 2) - faraday (2.14.1) + faraday (2.14.3) faraday-net_http (>= 2.0, < 3.5) json logger @@ -222,7 +225,7 @@ GEM faraday (>= 1, < 3) faraday-multipart (1.1.1) multipart-post (~> 2.0) - faraday-net_http (3.4.2) + faraday-net_http (3.4.4) net-http (~> 0.5) faraday-retry (2.3.2) faraday (~> 2.0) @@ -282,7 +285,8 @@ GEM signet (>= 0.16, < 2.a) hashdiff (1.2.0) hashery (2.1.2) - hashie (5.0.0) + hashie (5.1.0) + logger heapy (0.2.0) thor highline (3.1.2) @@ -325,16 +329,17 @@ GEM inline_svg (1.10.0) activesupport (>= 3.0) nokogiri (>= 1.6) - io-console (0.8.0) - irb (1.15.2) + io-console (0.8.2) + irb (1.18.0) pp (>= 0.6.0) + prism (>= 1.3.0) rdoc (>= 4.0.0) reline (>= 0.4.2) jbuilder (2.13.0) actionview (>= 5.0.0) activesupport (>= 5.0.0) jmespath (1.6.2) - json (2.19.2) + json (2.21.1) json-jwt (1.16.7) activesupport (>= 4.2) aes_key_wrap @@ -345,7 +350,7 @@ GEM json-schema (5.2.2) addressable (~> 2.8) bigdecimal (~> 3.1) - jwt (2.10.2) + jwt (2.10.3) base64 langfuse-ruby (0.1.4) concurrent-ruby (~> 1.0) @@ -375,7 +380,7 @@ GEM logtail (~> 0.1, >= 0.1.14) logtail-rack (~> 0.1) railties (>= 5.0.0) - loofah (2.25.1) + loofah (2.25.2) crass (~> 1.0.2) nokogiri (>= 1.12.0) lookbook (2.3.11) @@ -392,15 +397,15 @@ GEM zeitwerk (~> 2.5) lucide-rails (0.7.3) railties (>= 4.1.0) - mail (2.8.1) + mail (2.9.0) + logger mini_mime (>= 0.1.1) net-imap net-pop net-smtp - marcel (1.1.0) + marcel (1.2.1) matrix (0.4.2) memory_profiler (1.1.0) - method_source (1.1.0) mini_histogram (0.3.1) mini_magick (5.2.0) benchmark @@ -411,13 +416,13 @@ GEM ruby2_keywords (>= 0.0.5) msgpack (1.8.0) multi_json (1.20.1) - multi_xml (0.8.0) + multi_xml (0.9.1) bigdecimal (>= 3.1, < 5) multipart-post (2.4.1) mutex_m (0.3.0) net-http (0.9.1) uri (>= 0.11.1) - net-imap (0.5.8) + net-imap (0.6.4.1) date net-protocol net-pop (0.1.2) @@ -426,31 +431,32 @@ GEM timeout net-smtp (0.5.1) net-protocol - nio4r (2.7.4) - nokogiri (1.19.2-aarch64-linux-gnu) + nio4r (2.7.5) + nokogiri (1.19.4-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.19.2-aarch64-linux-musl) + nokogiri (1.19.4-aarch64-linux-musl) racc (~> 1.4) - nokogiri (1.19.2-arm-linux-gnu) + nokogiri (1.19.4-arm-linux-gnu) racc (~> 1.4) - nokogiri (1.19.2-arm-linux-musl) + nokogiri (1.19.4-arm-linux-musl) racc (~> 1.4) - nokogiri (1.19.2-arm64-darwin) + nokogiri (1.19.4-arm64-darwin) racc (~> 1.4) - nokogiri (1.19.2-x86_64-darwin) + nokogiri (1.19.4-x86_64-darwin) racc (~> 1.4) - nokogiri (1.19.2-x86_64-linux-gnu) + nokogiri (1.19.4-x86_64-linux-gnu) racc (~> 1.4) - nokogiri (1.19.2-x86_64-linux-musl) + nokogiri (1.19.4-x86_64-linux-musl) racc (~> 1.4) - oauth2 (2.0.18) + oauth2 (2.0.22) + auth-sanitizer (~> 0.2, >= 0.2.1) faraday (>= 0.17.3, < 4.0) jwt (>= 1.0, < 4.0) logger (~> 1.2) multi_xml (~> 0.5) rack (>= 1.2, < 4) - snaky_hash (~> 2.0, >= 2.0.3) - version_gem (~> 1.1, >= 1.1.9) + snaky_hash (~> 2.0, >= 2.0.5) + version_gem (~> 1.1, >= 1.1.11) octokit (10.0.0) faraday (>= 1, < 3) sawyer (~> 0.9) @@ -469,7 +475,7 @@ GEM omniauth-oauth2 (1.8.0) oauth2 (>= 1.4, < 3) omniauth (~> 2.0) - omniauth-rails_csrf_protection (1.0.2) + omniauth-rails_csrf_protection (2.0.1) actionpack (>= 4.2) omniauth (~> 2.0) omniauth-saml (2.2.4) @@ -516,7 +522,7 @@ GEM useragent (~> 0.16.3) posthog-ruby (3.3.3) concurrent-ruby (~> 1) - pp (0.6.2) + pp (0.6.3) prettyprint prettyprint (0.2.0) prism (1.4.0) @@ -525,11 +531,11 @@ GEM activesupport (>= 7.0.0) rack railties (>= 7.0.0) - psych (5.2.6) + psych (5.4.0) date stringio - public_suffix (6.0.2) - puma (6.6.0) + public_suffix (7.0.5) + puma (7.2.1) nio4r (~> 2.0) pundit (2.5.2) activesupport (>= 3.0.0) @@ -559,39 +565,38 @@ GEM rack (>= 3.0.0) rack-test (2.2.0) rack (>= 1.3) - rackup (2.2.1) + rackup (2.3.1) rack (>= 3) - rails (7.2.3.1) - actioncable (= 7.2.3.1) - actionmailbox (= 7.2.3.1) - actionmailer (= 7.2.3.1) - actionpack (= 7.2.3.1) - actiontext (= 7.2.3.1) - actionview (= 7.2.3.1) - activejob (= 7.2.3.1) - activemodel (= 7.2.3.1) - activerecord (= 7.2.3.1) - activestorage (= 7.2.3.1) - activesupport (= 7.2.3.1) + rails (8.1.3.1) + actioncable (= 8.1.3.1) + actionmailbox (= 8.1.3.1) + actionmailer (= 8.1.3.1) + actionpack (= 8.1.3.1) + actiontext (= 8.1.3.1) + actionview (= 8.1.3.1) + activejob (= 8.1.3.1) + activemodel (= 8.1.3.1) + activerecord (= 8.1.3.1) + activestorage (= 8.1.3.1) + activesupport (= 8.1.3.1) bundler (>= 1.15.0) - railties (= 7.2.3.1) + railties (= 8.1.3.1) rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.7.0) - loofah (~> 2.25) + rails-html-sanitizer (1.7.1) + loofah (~> 2.25, >= 2.25.2) nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) - rails-i18n (7.0.10) + rails-i18n (8.1.0) i18n (>= 0.7, < 2) - railties (>= 6.0.0, < 8) + railties (>= 8.0.0, < 9) rails-settings-cached (2.9.6) activerecord (>= 5.0.0) railties (>= 5.0.0) - railties (7.2.3.1) - actionpack (= 7.2.3.1) - activesupport (= 7.2.3.1) - cgi + railties (8.1.3.1) + actionpack (= 8.1.3.1) + activesupport (= 8.1.3.1) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) @@ -599,23 +604,24 @@ GEM tsort (>= 0.2) zeitwerk (~> 2.6) rainbow (3.1.1) - rake (13.3.0) + rake (13.4.2) rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) rbs (3.9.4) logger rchardet (1.10.0) - rdoc (6.14.2) + rdoc (7.2.0) erb psych (>= 4.0.0) + tsort redcarpet (3.6.1) redis (5.4.0) redis-client (>= 0.22.0) redis-client (0.25.0) connection_pool regexp_parser (2.10.0) - reline (0.6.1) + reline (0.6.3) io-console (~> 0.5) representable (3.2.0) declarative (< 0.1.0) @@ -646,17 +652,17 @@ GEM rspec-mocks (~> 3.13) rspec-support (~> 3.13) rspec-support (3.13.6) - rswag-api (2.16.0) - activesupport (>= 5.2, < 8.1) - railties (>= 5.2, < 8.1) - rswag-specs (2.16.0) - activesupport (>= 5.2, < 8.1) - json-schema (>= 2.2, < 6.0) - railties (>= 5.2, < 8.1) + rswag-api (2.17.0) + activesupport (>= 5.2, < 8.2) + railties (>= 5.2, < 8.2) + rswag-specs (2.17.0) + activesupport (>= 5.2, < 8.2) + json-schema (>= 2.2, < 7.0) + railties (>= 5.2, < 8.2) rspec-core (>= 2.14) - rswag-ui (2.16.0) - actionpack (>= 5.2, < 8.1) - railties (>= 5.2, < 8.1) + rswag-ui (2.17.0) + actionpack (>= 5.2, < 8.2) + railties (>= 5.2, < 8.2) rubocop (1.76.1) json (~> 2.3) language_server-protocol (~> 3.17.0.2) @@ -756,17 +762,15 @@ GEM skylight (6.0.4) activesupport (>= 5.2.0) smart_properties (1.17.0) - snaky_hash (2.0.3) + snaky_hash (2.0.7) hashie (>= 0.1.0, < 6) - version_gem (>= 1.1.8, < 3) - snaptrade (2.0.156) - faraday (>= 1.0.1, < 3.0) - faraday-multipart (~> 1.0, >= 1.0.4) + version_gem (~> 1.1, >= 1.1.14) + ssrf_filter (1.5.0) stackprof (0.2.27) standardwebhooks (1.1.0) stimulus-rails (1.3.4) railties (>= 6.0.0) - stringio (3.1.7) + stringio (3.2.0) stripe (15.3.0) swd (2.0.3) activesupport (>= 3) @@ -785,7 +789,7 @@ GEM tailwindcss-ruby (4.1.8-x86_64-linux-musl) terminal-table (4.0.0) unicode-display_width (>= 1.1.1, < 4) - thor (1.4.0) + thor (1.5.0) timeout (0.6.1) tpm-key_attestation (0.14.1) bindata (~> 2.4) @@ -814,11 +818,11 @@ GEM vcr (6.3.1) base64 vernier (1.8.0) - version_gem (1.1.9) - view_component (3.23.2) - activesupport (>= 5.2.0, < 8.1) + version_gem (1.1.14) + view_component (4.12.0) + actionview (>= 7.1.0) + activesupport (>= 7.1.0) concurrent-ruby (~> 1) - method_source (~> 1.0) web-console (4.2.1) actionview (>= 6.0.0) activemodel (>= 6.0.0) @@ -841,14 +845,14 @@ GEM crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) websocket (1.2.11) - websocket-driver (0.8.0) + websocket-driver (0.8.2) base64 websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) xpath (3.2.0) nokogiri (~> 1.8) - yard (0.9.37) - zeitwerk (2.7.3) + yard (0.9.44) + zeitwerk (2.8.2) PLATFORMS aarch64-linux-gnu @@ -879,7 +883,6 @@ DEPENDENCIES derailed_benchmarks doorkeeper dotenv-rails - ed25519 erb_lint faker faraday @@ -906,7 +909,7 @@ DEPENDENCIES omniauth (~> 2.1) omniauth-github omniauth-google-oauth2 - omniauth-rails_csrf_protection + omniauth-rails_csrf_protection (>= 2.0) omniauth-saml (~> 2.1) omniauth_openid_connect ostruct @@ -916,12 +919,12 @@ DEPENDENCIES plaid posthog-ruby propshaft - puma (>= 5.0) + puma (>= 7.2.1) pundit rack-attack (~> 6.6) rack-cors rack-mini-profiler - rails (~> 7.2.2) + rails (~> 8.1.0) rails-i18n rails-settings-cached rchardet @@ -946,7 +949,6 @@ DEPENDENCIES sidekiq-unique-jobs simplecov skylight - snaptrade (~> 2.0) stackprof stimulus-rails stripe @@ -961,7 +963,7 @@ DEPENDENCIES webmock RUBY VERSION - ruby 3.4.7p58 + ruby 3.4.9p82 BUNDLED WITH 2.6.7 diff --git a/README.md b/README.md index 41c93b4c6..c2deadee1 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ For further instructions, see guides below. [![Run on PikaPods](https://www.pikapods.com/static/run-button.svg)](https://www.pikapods.com/pods?run=sure) -[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/T_draF?referralCode=CW_fPQ) +[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/sure?referralCode=CW_fPQ) ### Managed OpenClaw for Sure Finances @@ -120,3 +120,13 @@ an [AGPLv3 license](https://github.com/we-promise/sure/blob/main/LICENSE). - "Sure" is not, and refers to this community fork. ![Alt](https://repobeats.axiom.co/api/embed/3a9753cff07501fba8a6749d0ebd567ff63848c8.svg "Repobeats analytics image") + +

+ + + + + Gittensor contributor impact for Sure repo + + +

diff --git a/SECURITY.md b/SECURITY.md index 034e84803..390fb7b2e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,20 +2,15 @@ ## Supported Versions -Use this section to tell people about which versions of your project are -currently being supported with security updates. +We maintain `vX.Y.Z-release-branch` [active areas](https://github.com/we-promise/sure/branches/all?query=release-branch&lastTab=overview) for high impact fixes that need to go out as "hotfix" releases ASAP. | Version | Supported | | ------- | ------------------ | -| 5.1.x | :white_check_mark: | -| 5.0.x | :x: | -| 4.0.x | :white_check_mark: | -| < 4.0 | :x: | +| 0.7.1 | :white_check_mark: | +| 0.7.0 | :white_check_mark: | +| 0.6.x | :x: | ## Reporting a Vulnerability -Use this section to tell people how to report a vulnerability. - -Tell them where to go, how often they can expect to get an update on a -reported vulnerability, what to expect if the vulnerability is accepted or -declined, etc. +Join our Discord and DM @Juanjo there so he can address the vulnerability before +disclosing it. diff --git a/app/assets/images/google-icon.svg b/app/assets/images/google-icon.svg new file mode 100644 index 000000000..31f21e61d --- /dev/null +++ b/app/assets/images/google-icon.svg @@ -0,0 +1,6 @@ + diff --git a/app/assets/images/ynab-logo.png b/app/assets/images/ynab-logo.png new file mode 100644 index 000000000..21ce33ac5 Binary files /dev/null and b/app/assets/images/ynab-logo.png differ diff --git a/app/assets/tailwind/application.css b/app/assets/tailwind/application.css index ddbda9b04..035b80946 100644 --- a/app/assets/tailwind/application.css +++ b/app/assets/tailwind/application.css @@ -10,7 +10,6 @@ @import "./simonweb_pickr.css"; -@import "./google-sign-in.css"; @import "./date-picker-dark-mode.css"; @import "./print-report.css"; @import "./privacy-mode.css"; diff --git a/app/assets/tailwind/google-sign-in.css b/app/assets/tailwind/google-sign-in.css deleted file mode 100644 index 151b14fe3..000000000 --- a/app/assets/tailwind/google-sign-in.css +++ /dev/null @@ -1,106 +0,0 @@ -@layer components { - .gsi-material-button { - -moz-user-select: none; - -webkit-user-select: none; - -ms-user-select: none; - -webkit-appearance: none; - background-color: WHITE; - background-image: none; - border: 1px solid #747775; - -webkit-border-radius: 4px; - border-radius: 4px; - -webkit-box-sizing: border-box; - box-sizing: border-box; - color: #1f1f1f; - cursor: pointer; - font-family: 'Roboto', arial, sans-serif; - font-size: 14px; - height: 40px; - letter-spacing: 0.25px; - outline: none; - overflow: hidden; - padding: 0 12px; - position: relative; - text-align: center; - -webkit-transition: background-color .218s, border-color .218s, box-shadow .218s; - transition: background-color .218s, border-color .218s, box-shadow .218s; - vertical-align: middle; - white-space: nowrap; - width: auto; - max-width: 400px; - min-width: min-content; - display: inline-flex; - } - - .gsi-material-button .gsi-material-button-icon { - height: 20px; - margin-right: 12px; - min-width: 20px; - width: 20px; - } - - .gsi-material-button .gsi-material-button-content-wrapper { - -webkit-align-items: center; - align-items: center; - display: flex; - -webkit-flex-direction: row; - flex-direction: row; - -webkit-flex-wrap: nowrap; - flex-wrap: nowrap; - height: 100%; - justify-content: space-between; - position: relative; - width: 100%; - } - - .gsi-material-button .gsi-material-button-contents { - -webkit-flex-grow: 1; - flex-grow: 1; - font-family: 'Roboto', arial, sans-serif; - font-weight: 500; - overflow: hidden; - text-overflow: ellipsis; - vertical-align: top; - } - - .gsi-material-button .gsi-material-button-state { - -webkit-transition: opacity .218s; - transition: opacity .218s; - bottom: 0; - left: 0; - opacity: 0; - position: absolute; - right: 0; - top: 0; - } - - .gsi-material-button:disabled { - cursor: default; - background-color: #ffffff61; - border-color: #1f1f1f1f; - } - - .gsi-material-button:disabled .gsi-material-button-contents { - opacity: 38%; - } - - .gsi-material-button:disabled .gsi-material-button-icon { - opacity: 38%; - } - - .gsi-material-button:not(:disabled):active .gsi-material-button-state, - .gsi-material-button:not(:disabled):focus .gsi-material-button-state { - background-color: #303030; - opacity: 12%; - } - - .gsi-material-button:not(:disabled):hover { - -webkit-box-shadow: 0 1px 2px 0 rgba(60, 64, 67, .30), 0 1px 3px 1px rgba(60, 64, 67, .15); - box-shadow: 0 1px 2px 0 rgba(60, 64, 67, .30), 0 1px 3px 1px rgba(60, 64, 67, .15); - } - - .gsi-material-button:not(:disabled):hover .gsi-material-button-state { - background-color: #303030; - opacity: 8%; - } -} diff --git a/app/assets/tailwind/sure-design-system/_generated.css b/app/assets/tailwind/sure-design-system/_generated.css index 79ae54d43..27a345930 100644 --- a/app/assets/tailwind/sure-design-system/_generated.css +++ b/app/assets/tailwind/sure-design-system/_generated.css @@ -7,14 +7,17 @@ @theme { --font-sans: 'Geist', system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; --font-mono: 'Geist Mono', ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + --font-weight-medium: 500; + --font-weight-semibold: 600; --color-white: #ffffff; --color-black: #0B0B0B; --color-success: var(--color-green-700); --color-warning: var(--color-yellow-600); - --color-destructive: var(--color-red-600); + --color-destructive: var(--color-red-500); --color-info: var(--color-blue-600); --color-shadow: --alpha(var(--color-black) / 6%); --color-link: var(--color-blue-600); + --color-focus-ring: var(--color-alpha-black-700); --color-tertiary: var(--color-alpha-black-100); --color-surface: var(--color-gray-50); --color-surface-hover: var(--color-gray-100); @@ -25,7 +28,7 @@ --color-container-inset: var(--color-gray-50); --color-container-inset-hover: var(--color-gray-100); --color-nav-indicator: var(--color-black); - --color-toggle-track: var(--color-gray-100); + --color-toggle-track: var(--color-gray-300); --color-destructive-subtle: var(--color-red-200); --color-gray-25: #FAFAFA; --color-gray-50: #F7F7F7; @@ -216,6 +219,7 @@ --color-info: var(--color-blue-500); --color-shadow: --alpha(var(--color-white) / 8%); --color-link: var(--color-blue-500); + --color-focus-ring: var(--color-alpha-white-700); --color-tertiary: var(--color-alpha-white-200); --color-surface: var(--color-black); --color-surface-hover: var(--color-gray-800); @@ -294,7 +298,7 @@ @apply text-gray-400; @variant theme-dark { - @apply text-gray-500; + @apply text-gray-400; } } @@ -342,7 +346,7 @@ @apply border-alpha-black-300; @variant theme-dark { - @apply border-alpha-white-400; + @apply border-alpha-white-500; } } @@ -350,7 +354,7 @@ @apply border-alpha-black-200; @variant theme-dark { - @apply border-alpha-white-300; + @apply border-alpha-white-400; } } @@ -362,7 +366,7 @@ @apply border-alpha-black-50; @variant theme-dark { - @apply border-alpha-white-100; + @apply border-alpha-white-200; } } @@ -447,7 +451,7 @@ } @utility button-bg-destructive { - @apply bg-red-500; + @apply bg-red-600; @variant theme-dark { @apply bg-red-400; @@ -455,7 +459,7 @@ } @utility button-bg-destructive-hover { - @apply bg-red-600; + @apply bg-red-700; @variant theme-dark { @apply bg-red-500; diff --git a/app/assets/tailwind/sure-design-system/base.css b/app/assets/tailwind/sure-design-system/base.css index b23d9fb9e..9bf606a10 100644 --- a/app/assets/tailwind/sure-design-system/base.css +++ b/app/assets/tailwind/sure-design-system/base.css @@ -11,10 +11,21 @@ } button { - @apply cursor-pointer focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-alpha-black-300; + @apply cursor-pointer; - @variant theme-dark { - @apply focus-visible:ring-alpha-white-300; + /* Canonical keyboard-focus indicator (#2136) — the same recipe as + `.focus-ring` in components.css, applied here so every -