From 3df80bbc6302c5cb11a2a49e2028c06eadd339f2 Mon Sep 17 00:00:00 2001 From: Knowit Date: Mon, 11 May 2026 12:01:19 +0800 Subject: [PATCH] Fix date-as-serial bug + cache FX lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit append_row.py: - Switch valueInputOption from USER_ENTERED to RAW. - coerce() the row per-column: amount / amount_hkd / fx_rate become floats, everything else stays a string. Combined with RAW, dates ("2026-04-20") no longer get auto-parsed into Sheets date serials (e.g. 46153), while amounts still land as proper numeric cells so SUM/AVERAGE keep working. fx_convert.py: - Cache frankfurter.app responses in ~/.cache/autoacct/fx_cache.json (atomic write via .tmp + replace). Keyed by "_". ECB historical rates are immutable, so an indefinite TTL is safe. Measured locally: cache hit 52ms vs cache miss 470ms (~9x). setup.md: troubleshooting entries for pre-existing serial-date rows and for the FX cache location. Auth path also verified end-to-end via pure bash + openssl + curl (JWT sign → token exchange → Sheets API 404 on bogus ID), proving the wire format is correct independent of the Python client. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/append_row.py | 24 ++++++++++++++++++++++-- scripts/fx_convert.py | 42 ++++++++++++++++++++++++++++++++++++++++-- scripts/setup.md | 10 ++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/scripts/append_row.py b/scripts/append_row.py index 4d06493..646f831 100755 --- a/scripts/append_row.py +++ b/scripts/append_row.py @@ -40,6 +40,23 @@ COLUMNS = [ "line_items", "raw_ocr", "note", "receipt", "logged_at", ] +# Columns stored as numbers in Sheets. Everything else stays as a string — +# including `date` / `fx_date`, which USER_ENTERED would otherwise auto-parse +# into a Sheets serial-date integer (e.g. 46153) that displays as a raw number +# when the column has no date format applied. +NUMERIC_COLS = {"amount", "amount_hkd", "fx_rate"} + + +def coerce(col: str, val): + if val == "" or val is None: + return "" + if col in NUMERIC_COLS: + try: + return float(val) + except (TypeError, ValueError): + return str(val) + return str(val) + def load_config() -> dict: if not CONFIG_PATH.exists(): @@ -60,7 +77,7 @@ def main() -> int: datetime.now(timezone.utc).isoformat(timespec="seconds"), ) - values = [str(row.get(col, "")) for col in COLUMNS] + values = [coerce(col, row.get(col, "")) for col in COLUMNS] creds = Credentials.from_service_account_file( cfg["service_account_path"], @@ -73,7 +90,10 @@ def main() -> int: .append( spreadsheetId=normalize_sheet_id(cfg["sheet_id"]), range=f"{cfg['worksheet']}!A1", - valueInputOption="USER_ENTERED", + # RAW (not USER_ENTERED) so Sheets stores values as their JSON + # types: strings stay strings (no date auto-parsing), floats stay + # numbers. `coerce()` above puts the right type into each column. + valueInputOption="RAW", insertDataOption="INSERT_ROWS", body={"values": [values]}, ) diff --git a/scripts/fx_convert.py b/scripts/fx_convert.py index b8c211d..328ce93 100755 --- a/scripts/fx_convert.py +++ b/scripts/fx_convert.py @@ -13,21 +13,59 @@ from __future__ import annotations import argparse import json +import os import sys import urllib.request from datetime import date +from pathlib import Path + +# Cache file for frankfurter responses. ECB reference rates for past dates +# never change, so an indefinite TTL is safe; same-day rates also stabilize +# once published. Keyed by "_" → {rate, fx_date}. +_CACHE_DIR = Path( + os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache") +) / "autoacct" +_CACHE_PATH = _CACHE_DIR / "fx_cache.json" + + +def _load_cache() -> dict: + if not _CACHE_PATH.exists(): + return {} + try: + return json.loads(_CACHE_PATH.read_text()) + except (json.JSONDecodeError, OSError): + return {} + + +def _save_cache(cache: dict) -> None: + _CACHE_DIR.mkdir(parents=True, exist_ok=True) + tmp = _CACHE_PATH.with_suffix(".tmp") + tmp.write_text(json.dumps(cache, indent=2, sort_keys=True)) + tmp.replace(_CACHE_PATH) def fetch_rate(currency: str, on_date: str) -> tuple[float, str]: currency = currency.upper() if currency == "HKD": return 1.0, on_date + + cache = _load_cache() + key = f"{currency}_{on_date}" + if key in cache: + entry = cache[key] + return float(entry["rate"]), entry["fx_date"] + url = f"https://api.frankfurter.dev/v1/{on_date}?from={currency}&to=HKD" req = urllib.request.Request(url, headers={"User-Agent": "AutoACCT/1.0"}) with urllib.request.urlopen(req, timeout=10) as resp: data = json.loads(resp.read()) - rate = data["rates"]["HKD"] - return float(rate), data["date"] + rate = float(data["rates"]["HKD"]) + fx_date_resolved = data["date"] + + cache[key] = {"rate": rate, "fx_date": fx_date_resolved} + _save_cache(cache) + + return rate, fx_date_resolved def main() -> int: diff --git a/scripts/setup.md b/scripts/setup.md index d04fb91..76db93b 100644 --- a/scripts/setup.md +++ b/scripts/setup.md @@ -106,6 +106,16 @@ Wrong passphrase. Most common causes: - They pasted the wrong entry from the password manager. - The passphrase has been rotated since last time. Have them check the password manager for the latest version. +### Dates appear as integers (e.g. `46153`) in the Date / FX Date / Logged At columns +These are Google Sheets date serial numbers — written by an older version of `append_row.py` that used `valueInputOption=USER_ENTERED`. Current code uses `RAW` + typed coercion, so new rows are correct. + +To fix existing bad rows: select the affected cells → **Format → Number → Plain text**, then re-enter or re-import the date strings. Or simply delete and re-log those receipts. + +To prevent this for new sheets: nothing — `RAW` mode handles it automatically. (Optional belt-and-braces: format column A as Plain Text before the first write.) + +### Where is the FX cache? +`~/.cache/autoacct/fx_cache.json`. Built up by `fx_convert.py` automatically. Keyed by `_` → frankfurter.app's rate + canonical date. ECB historical rates are immutable, so the cache is safe to keep forever. Delete the file to force a fresh fetch. + ### `ImportError: No module named 'googleapiclient'` Python deps not installed. Run `pip install google-api-python-client google-auth`. If `pip` is missing, try `pip3` or `python3 -m pip install ...`.