From 12b6b46e2e53267b5ce3e59d8e106dd17fdf3993 Mon Sep 17 00:00:00 2001 From: gsinghpal Date: Sun, 19 Apr 2026 11:45:36 -0400 Subject: [PATCH] feat(fusion_accounting_bank_rec): pre-aggregated MV for OWL widget perf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CREATE MATERIALIZED VIEW fusion_unreconciled_bank_line_mv pre-computes the data the kanban widget needs (top suggestion, confidence band, attachment count, partner reconcile hint) so that listing 50-100 lines is one indexed query instead of N+1. Refresh strategy: - Triggered on fusion.reconcile.suggestion create/write (best-effort, never poisons the originating transaction) - Cron (every 5 min) — added in Task 25 The MV is created in the model's init() (Odoo calls this on install/upgrade). The SQL DDL is idempotent (CREATE MATERIALIZED VIEW IF NOT EXISTS / CREATE INDEX IF NOT EXISTS) and includes a UNIQUE(id) index so REFRESH MATERIALIZED VIEW CONCURRENTLY is supported. _refresh() falls back to a blocking refresh on the first call after creation. Made-with: Cursor --- fusion_accounting_bank_rec/__manifest__.py | 2 +- .../sql/create_mv_unreconciled_bank_line.sql | 53 ++++++++ fusion_accounting_bank_rec/models/__init__.py | 1 + .../models/fusion_reconcile_suggestion.py | 31 +++++ .../fusion_unreconciled_bank_line_mv.py | 86 +++++++++++++ .../security/ir.model.access.csv | 2 + fusion_accounting_bank_rec/tests/__init__.py | 1 + .../tests/test_mv_unreconciled.py | 115 ++++++++++++++++++ 8 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 fusion_accounting_bank_rec/data/sql/create_mv_unreconciled_bank_line.sql create mode 100644 fusion_accounting_bank_rec/models/fusion_unreconciled_bank_line_mv.py create mode 100644 fusion_accounting_bank_rec/tests/test_mv_unreconciled.py diff --git a/fusion_accounting_bank_rec/__manifest__.py b/fusion_accounting_bank_rec/__manifest__.py index 23c7cd98..1ccba25a 100644 --- a/fusion_accounting_bank_rec/__manifest__.py +++ b/fusion_accounting_bank_rec/__manifest__.py @@ -1,6 +1,6 @@ { 'name': 'Fusion Accounting — Bank Reconciliation', - 'version': '19.0.1.0.5', + 'version': '19.0.1.0.6', 'category': 'Accounting/Accounting', 'sequence': 28, 'summary': 'Native V19 bank reconciliation widget with AI confidence scoring + behavioural learning.', diff --git a/fusion_accounting_bank_rec/data/sql/create_mv_unreconciled_bank_line.sql b/fusion_accounting_bank_rec/data/sql/create_mv_unreconciled_bank_line.sql new file mode 100644 index 00000000..1483fee5 --- /dev/null +++ b/fusion_accounting_bank_rec/data/sql/create_mv_unreconciled_bank_line.sql @@ -0,0 +1,53 @@ +-- Materialized view: pre-aggregated data for the OWL bank reconciliation widget. +-- Refreshed on cron (Task 25) and on suggestion writes. +-- Indexed on (company_id, journal_id, date) for fast UI queries. + +CREATE MATERIALIZED VIEW IF NOT EXISTS fusion_unreconciled_bank_line_mv AS +SELECT + bsl.id AS id, + bsl.company_id AS company_id, + bsl.journal_id AS journal_id, + bsl.date AS date, + bsl.amount AS amount, + bsl.payment_ref AS payment_ref, + bsl.currency_id AS currency_id, + bsl.partner_id AS partner_id, + bsl.create_date AS create_date, + -- Top suggestion (highest confidence pending one) + (SELECT s.id FROM fusion_reconcile_suggestion s + WHERE s.statement_line_id = bsl.id AND s.state = 'pending' + ORDER BY s.confidence DESC, s.rank ASC LIMIT 1) AS top_suggestion_id, + (SELECT s.confidence FROM fusion_reconcile_suggestion s + WHERE s.statement_line_id = bsl.id AND s.state = 'pending' + ORDER BY s.confidence DESC, s.rank ASC LIMIT 1) AS top_confidence, + CASE + WHEN (SELECT MAX(s.confidence) FROM fusion_reconcile_suggestion s + WHERE s.statement_line_id = bsl.id AND s.state = 'pending') >= 0.85 + THEN 'high' + WHEN (SELECT MAX(s.confidence) FROM fusion_reconcile_suggestion s + WHERE s.statement_line_id = bsl.id AND s.state = 'pending') >= 0.60 + THEN 'medium' + WHEN (SELECT MAX(s.confidence) FROM fusion_reconcile_suggestion s + WHERE s.statement_line_id = bsl.id AND s.state = 'pending') > 0 + THEN 'low' + ELSE 'none' + END AS confidence_band, + -- Attachment count (assumes ir_attachment.res_model='account.bank.statement.line') + (SELECT COUNT(*) FROM ir_attachment att + WHERE att.res_model = 'account.bank.statement.line' AND att.res_id = bsl.id) + AS attachment_count, + -- Partner reconcile pattern hint + COALESCE((SELECT p.reconcile_count FROM fusion_reconcile_pattern p + WHERE p.partner_id = bsl.partner_id AND p.company_id = bsl.company_id LIMIT 1), 0) + AS partner_reconcile_count +FROM account_bank_statement_line bsl +WHERE bsl.is_reconciled = FALSE; + +-- Indexes for the common UI queries: filter by company + journal, sort by date desc. +CREATE INDEX IF NOT EXISTS fusion_mv_unrec_company_journal_date_idx + ON fusion_unreconciled_bank_line_mv (company_id, journal_id, date DESC); +CREATE INDEX IF NOT EXISTS fusion_mv_unrec_partner_idx + ON fusion_unreconciled_bank_line_mv (partner_id) WHERE partner_id IS NOT NULL; +-- UNIQUE index required for CONCURRENTLY refresh +CREATE UNIQUE INDEX IF NOT EXISTS fusion_mv_unrec_id_idx + ON fusion_unreconciled_bank_line_mv (id); diff --git a/fusion_accounting_bank_rec/models/__init__.py b/fusion_accounting_bank_rec/models/__init__.py index af5c63a2..645b184b 100644 --- a/fusion_accounting_bank_rec/models/__init__.py +++ b/fusion_accounting_bank_rec/models/__init__.py @@ -5,3 +5,4 @@ from . import fusion_bank_rec_widget from . import account_bank_statement_line from . import account_reconcile_model from . import fusion_reconcile_engine +from . import fusion_unreconciled_bank_line_mv diff --git a/fusion_accounting_bank_rec/models/fusion_reconcile_suggestion.py b/fusion_accounting_bank_rec/models/fusion_reconcile_suggestion.py index 29d64478..72df71d8 100644 --- a/fusion_accounting_bank_rec/models/fusion_reconcile_suggestion.py +++ b/fusion_accounting_bank_rec/models/fusion_reconcile_suggestion.py @@ -9,8 +9,12 @@ suggestions here, and the user (or batch-accept action) approves them through the engine's accept_suggestion() method. """ +import logging + from odoo import api, fields, models +_logger = logging.getLogger(__name__) + class FusionReconcileSuggestion(models.Model): _name = "fusion.reconcile.suggestion" @@ -96,3 +100,30 @@ class FusionReconcileSuggestion(models.Model): sug.confidence_band = 'low' else: sug.confidence_band = 'none' + + # ------------------------------------------------------------------ + # CRUD overrides — trigger MV refresh so the OWL widget sees fresh + # confidence bands / top suggestion ids without waiting for cron. + # ------------------------------------------------------------------ + + @api.model_create_multi + def create(self, vals_list): + records = super().create(vals_list) + self._trigger_mv_refresh() + return records + + def write(self, vals): + res = super().write(vals) + # Only refresh on changes that affect the MV's projected columns. + if 'state' in vals or 'confidence' in vals or 'rank' in vals: + self._trigger_mv_refresh() + return res + + def _trigger_mv_refresh(self): + """Best-effort MV refresh; never poison the originating transaction.""" + try: + self.env['fusion.unreconciled.bank.line.mv']._refresh( + concurrently=True) + except Exception as e: # noqa: BLE001 + _logger.warning( + "MV refresh after suggestion write failed: %s", e) diff --git a/fusion_accounting_bank_rec/models/fusion_unreconciled_bank_line_mv.py b/fusion_accounting_bank_rec/models/fusion_unreconciled_bank_line_mv.py new file mode 100644 index 00000000..0d831543 --- /dev/null +++ b/fusion_accounting_bank_rec/models/fusion_unreconciled_bank_line_mv.py @@ -0,0 +1,86 @@ +"""Materialized view exposing pre-aggregated unreconciled-bank-line data. + +The MV is created in the model's init() (called by Odoo on install/upgrade). +Refresh strategy: +- Cron (every 5 min) — see fusion_accounting_bank_rec/data/cron.xml (Task 25) +- Triggered refresh after suggestion writes (handled in fusion_reconcile_suggestion.py) +""" + +import logging +import os + +from odoo import api, fields, models + +_logger = logging.getLogger(__name__) + + +class FusionUnreconciledBankLineMV(models.Model): + _name = "fusion.unreconciled.bank.line.mv" + _description = "Materialized view of unreconciled bank lines for OWL widget" + _auto = False # we manage the table ourselves + _table = "fusion_unreconciled_bank_line_mv" + _order = "date desc, id desc" + + # Fields mirror the columns in the SQL view; required so Odoo can read them. + company_id = fields.Many2one('res.company', readonly=True) + journal_id = fields.Many2one('account.journal', readonly=True) + date = fields.Date(readonly=True) + amount = fields.Float(readonly=True) + payment_ref = fields.Char(readonly=True) + currency_id = fields.Many2one('res.currency', readonly=True) + partner_id = fields.Many2one('res.partner', readonly=True) + create_date = fields.Datetime(readonly=True) + top_suggestion_id = fields.Many2one('fusion.reconcile.suggestion', readonly=True) + top_confidence = fields.Float(readonly=True) + confidence_band = fields.Selection([ + ('high', 'High'), + ('medium', 'Medium'), + ('low', 'Low'), + ('none', 'None'), + ], readonly=True) + attachment_count = fields.Integer(readonly=True) + partner_reconcile_count = fields.Integer(readonly=True) + + def init(self): + """Create the MV if missing. + + Reads create_mv_unreconciled_bank_line.sql and executes it. Idempotent + because the SQL uses CREATE MATERIALIZED VIEW IF NOT EXISTS.""" + sql_path = os.path.join( + os.path.dirname(__file__), '..', 'data', 'sql', + 'create_mv_unreconciled_bank_line.sql') + with open(sql_path, 'r') as f: + sql = f.read() + self.env.cr.execute(sql) + _logger.info( + "fusion_unreconciled_bank_line_mv: created/verified MV + indexes") + + @api.model + def _refresh(self, *, concurrently=True): + """Refresh the MV. + + If ``concurrently=True`` (default), uses + REFRESH MATERIALIZED VIEW CONCURRENTLY (requires the unique index). + Falls back to a blocking refresh on the first refresh after creation + (when CONCURRENTLY is not yet allowed because the MV has never been + populated).""" + keyword = "CONCURRENTLY" if concurrently else "" + try: + self.env.cr.execute( + f"REFRESH MATERIALIZED VIEW {keyword} fusion_unreconciled_bank_line_mv" + ) + _logger.debug( + "fusion_unreconciled_bank_line_mv refreshed (%s)", + 'concurrent' if concurrently else 'blocking') + except Exception as e: # noqa: BLE001 + # CONCURRENTLY fails on first refresh after creation if the MV is + # empty / has never been populated; fall back to non-concurrent. + if concurrently: + _logger.warning( + "Concurrent MV refresh failed (%s); falling back to " + "blocking refresh", e) + self.env.cr.execute( + "REFRESH MATERIALIZED VIEW fusion_unreconciled_bank_line_mv" + ) + else: + raise diff --git a/fusion_accounting_bank_rec/security/ir.model.access.csv b/fusion_accounting_bank_rec/security/ir.model.access.csv index de092300..e105c296 100644 --- a/fusion_accounting_bank_rec/security/ir.model.access.csv +++ b/fusion_accounting_bank_rec/security/ir.model.access.csv @@ -6,3 +6,5 @@ access_fusion_reconcile_precedent_admin,precedent admin,model_fusion_reconcile_p access_fusion_reconcile_suggestion_user,suggestion user,model_fusion_reconcile_suggestion,fusion_accounting_core.group_fusion_accounting_user,1,0,0,0 access_fusion_reconcile_suggestion_admin,suggestion admin,model_fusion_reconcile_suggestion,fusion_accounting_core.group_fusion_accounting_admin,1,1,1,1 access_fusion_bank_rec_widget_user,bank rec widget user,model_fusion_bank_rec_widget,fusion_accounting_core.group_fusion_accounting_user,1,1,1,1 +access_fusion_unreconciled_bank_line_mv_user,unreconciled bank line mv user,model_fusion_unreconciled_bank_line_mv,fusion_accounting_core.group_fusion_accounting_user,1,0,0,0 +access_fusion_unreconciled_bank_line_mv_admin,unreconciled bank line mv admin,model_fusion_unreconciled_bank_line_mv,fusion_accounting_core.group_fusion_accounting_admin,1,0,0,0 diff --git a/fusion_accounting_bank_rec/tests/__init__.py b/fusion_accounting_bank_rec/tests/__init__.py index 96de5be4..1e299e5f 100644 --- a/fusion_accounting_bank_rec/tests/__init__.py +++ b/fusion_accounting_bank_rec/tests/__init__.py @@ -13,3 +13,4 @@ from . import test_bank_rec_prompt from . import test_bank_rec_adapter from . import test_bank_rec_tools from . import test_legacy_tools_refactor +from . import test_mv_unreconciled diff --git a/fusion_accounting_bank_rec/tests/test_mv_unreconciled.py b/fusion_accounting_bank_rec/tests/test_mv_unreconciled.py new file mode 100644 index 00000000..8281b9f2 --- /dev/null +++ b/fusion_accounting_bank_rec/tests/test_mv_unreconciled.py @@ -0,0 +1,115 @@ +"""Smoke tests for the fusion_unreconciled_bank_line_mv materialized view. + +NOTE on the explicit ``self.env.cr.commit()`` calls below: + PostgreSQL's ``REFRESH MATERIALIZED VIEW`` only sees data from + *committed* transactions. ``TransactionCase`` rolls back at the end + of each test, so without an explicit commit the freshly-inserted + bank line would not be visible to the refresh and the assertions + would fail. The trade-off is that the records we create *are* + persisted; we therefore unlink them in a ``finally`` block to keep + test isolation. + + If this proves too brittle later we can convert this case to extend + ``BaseCase`` (no rollback) and clean up explicitly. For Phase 1 the + commit-+-finally pattern is the simpler choice. +""" + +from odoo.tests.common import TransactionCase, tagged +from . import _factories as f + + +@tagged('post_install', '-at_install') +class TestUnreconciledBankLineMV(TransactionCase): + + def setUp(self): + super().setUp() + self.partner = self.env['res.partner'].create({ + 'name': 'MV Test Partner', + }) + # Force a refresh so we see freshly-inserted lines from prior tests. + # First refresh after creation may need to be blocking; the + # _refresh helper handles fallback automatically. + self.env['fusion.unreconciled.bank.line.mv']._refresh( + concurrently=False) + + def test_mv_exists_and_is_queryable(self): + # Smoke: the model can be searched without error. + rows = self.env['fusion.unreconciled.bank.line.mv'].search( + [], limit=10) + self.assertIsNotNone(rows) + + def test_mv_includes_unreconciled_line(self): + bank_line = f.make_bank_line( + self.env, amount=999.99, partner=self.partner) + # MV refresh sees committed data only; commit, refresh, assert, + # then unlink to keep test isolation. + self.env.cr.commit() + try: + self.env['fusion.unreconciled.bank.line.mv']._refresh( + concurrently=False) + mv_row = self.env['fusion.unreconciled.bank.line.mv'].search([ + ('id', '=', bank_line.id), + ]) + self.assertTrue( + mv_row, + "MV should contain freshly-inserted unreconciled line") + self.assertAlmostEqual(mv_row.amount, 999.99, places=2) + # No suggestion yet -> band 'none', confidence 0. + self.assertEqual(mv_row.confidence_band, 'none') + self.assertEqual(mv_row.attachment_count, 0) + finally: + bank_line.unlink() + self.env.cr.commit() + + def test_mv_excludes_reconciled_line(self): + bank_line, recv_lines = f.make_reconcileable_pair( + self.env, amount=100.00, partner=self.partner) + self.env['fusion.reconcile.engine'].reconcile_one( + bank_line, against_lines=recv_lines) + self.env.cr.commit() + try: + self.env['fusion.unreconciled.bank.line.mv']._refresh( + concurrently=False) + mv_row = self.env['fusion.unreconciled.bank.line.mv'].search([ + ('id', '=', bank_line.id), + ]) + self.assertFalse( + mv_row, "Reconciled line should be excluded from MV") + finally: + # Best-effort unwind for test isolation. Use the engine's + # standard undo path since reconcile_one rewrites the bank + # move's line_ids. + try: + bank_line.action_undo_reconciliation() + except Exception: + pass + try: + bank_line.unlink() + except Exception: + pass + self.env.cr.commit() + + def test_mv_confidence_band_high_for_high_conf_suggestion(self): + bank_line = f.make_bank_line( + self.env, amount=500.00, partner=self.partner) + f.make_suggestion( + self.env, statement_line=bank_line, confidence=0.92) + self.env.cr.commit() + try: + self.env['fusion.unreconciled.bank.line.mv']._refresh( + concurrently=False) + mv_row = self.env['fusion.unreconciled.bank.line.mv'].search([ + ('id', '=', bank_line.id), + ]) + self.assertTrue(mv_row, "MV row should exist for suggestion line") + # 0.92 falls in the 'high' band per the SQL CASE (>= 0.85). + self.assertEqual(mv_row.confidence_band, 'high') + self.assertAlmostEqual(mv_row.top_confidence, 0.92, places=2) + finally: + # Unlink suggestion first (cascade would handle it but explicit + # is safer if the test order reuses partner records). + self.env['fusion.reconcile.suggestion'].search([ + ('statement_line_id', '=', bank_line.id), + ]).unlink() + bank_line.unlink() + self.env.cr.commit()