Add generated SQL columns for graph properties

AGE keeps all properties in one agtype column and its Cypher writes fire
no triggers, so the bridge periodically calls a stored procedure that adds
a stored generated text column for every new property key. Only label
tables with inserts or updates since the previous run are scanned.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

bridge/peo4c/__main__.py

@@ -7,6 +7,9 @@ import functools
 import logging
 import os
 
+from psycopg.conninfo import make_conninfo
+
+from . import property_columns
 from .backend import Config
 from .bolt import handle_client
 from .web import HttpServer
@@ -43,7 +46,19 @@ async def main():
         servers.append(await asyncio.start_server(http.handle, "0.0.0.0", http_port))
         log.info("neo4j-browser on http://localhost:%d/browser/", http_port)
 
-    await asyncio.gather(*(s.serve_forever() for s in servers))
+    tasks = [s.serve_forever() for s in servers]
+    if env("PEO4C_PROPERTY_COLUMNS", "false") == "true":
+        conninfo = make_conninfo(
+            host=env("PEO4C_PROPERTY_COLUMNS_PG_HOST", "/var/run/postgresql"),
+            port=config.pg_port,
+            dbname=config.pg_database,
+            user=env("PEO4C_PROPERTY_COLUMNS_PG_USER", env("POSTGRES_USER", "postgres")),
+            application_name="peo4c property columns",
+        )
+        tasks.append(property_columns.sync_forever(
+            conninfo, float(env("PEO4C_PROPERTY_COLUMNS_INTERVAL", "5"))))
+
+    await asyncio.gather(*tasks)
 
 
 if __name__ == "__main__":

bridge/peo4c/backend.py

@@ -337,6 +337,9 @@ class Session:
             raise BoltFailure("Neo.ClientError.Statement.ArgumentError",
                               f"Invalid graph name '{name}': AGE graph names must be 3-63 characters "
                               "of letters, digits, '_', '.' or '-', starting with a letter or '_'.")
+        if name == "peo4c":
+            raise BoltFailure("Neo.ClientError.Statement.ArgumentError",
+                              "The name 'peo4c' is reserved: it is the schema of peo4c's own SQL helpers.")
         if name in await self.graphs():
             if if_not_exists:
                 return Result([], query_type="s")

bridge/peo4c/property_columns.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2026 Bart Broere
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""Keeps generated property columns on AGE label tables up to date.
+
+Installs the procedures in ``property_columns.sql`` and calls
+``peo4c.sync_property_columns()`` every ``interval`` seconds. This needs a
+connection that may alter every graph's tables, so it uses its own
+(superuser) connection rather than the credentials of Bolt users.
+"""
+
+import asyncio
+import logging
+from pathlib import Path
+
+import psycopg
+
+log = logging.getLogger("peo4c.property_columns")
+
+INSTALL_SQL = (Path(__file__).parent / "property_columns.sql").read_text()
+
+
+def log_notice(diag):
+    level = logging.WARNING if diag.severity_nonlocalized == "WARNING" else logging.INFO
+    log.log(level, "%s", diag.message_primary)
+
+
+async def sync_forever(conninfo, interval):
+    error = None
+    while True:
+        try:
+            async with await psycopg.AsyncConnection.connect(conninfo, autocommit=True) as conn:
+                conn.add_notice_handler(log_notice)
+                await conn.execute(INSTALL_SQL)
+                log.info("syncing property columns every %gs", interval)
+                error = None
+                while True:
+                    await conn.execute("CALL peo4c.sync_property_columns()")
+                    await asyncio.sleep(interval)
+        except psycopg.Error as e:
+            # PostgreSQL is still starting or restarted, or AGE is not
+            # installed: report each new problem once and keep retrying.
+            message = str(e).strip()
+            if message != error:
+                level = logging.DEBUG if isinstance(e, psycopg.OperationalError) else logging.WARNING
+                log.log(level, "property column sync unavailable, retrying: %s", message)
+                error = message
+            await asyncio.sleep(interval)

bridge/peo4c/property_columns.sql

@@ -0,0 +1,104 @@
+-- SPDX-FileCopyrightText: 2026 Bart Broere
+-- SPDX-License-Identifier: GPL-3.0-or-later
+--
+-- Generated columns for node and relationship properties.
+--
+-- AGE stores all properties of a node or relationship in one agtype column,
+-- "properties". For every property key found in a label table, this adds a
+-- stored generated column with the same name:
+--
+--     name text GENERATED ALWAYS AS (properties ->> 'name'::text) STORED
+--
+-- so the graph can be queried from SQL as  SELECT name FROM graph."Person".
+--
+-- AGE's Cypher writes bypass the executor and fire no triggers, row or
+-- statement level, so new keys cannot be caught by a trigger. Instead
+-- CALL peo4c.sync_property_columns() looks for them; the peo4c bridge calls
+-- it every few seconds. It only scans label tables that saw inserts or updates
+-- since its previous run, according to the statistics collector, and commits
+-- after each table so a table rewrite never holds locks on other tables.
+-- Generated columns are kept up to date by AGE's CREATE, MERGE, SET and REMOVE.
+--
+-- Columns are text, like ->> on json, because a property can hold values of
+-- different types in different rows and a failing cast would make writes fail.
+
+CREATE SCHEMA IF NOT EXISTS peo4c;
+
+CREATE TABLE IF NOT EXISTS peo4c.property_column_state (
+    relid oid PRIMARY KEY,
+    changes bigint NOT NULL  -- inserts + updates seen when the table was last scanned
+);
+
+-- Adds a generated column for every property key in tbl that has no column of
+-- that name yet, and returns the keys it added. Keys that are empty or longer
+-- than PostgreSQL's 63-byte identifier limit are skipped, as are keys that
+-- clash with an existing column such as id, start_id or end_id.
+CREATE OR REPLACE FUNCTION peo4c.add_property_columns(tbl regclass)
+RETURNS SETOF text LANGUAGE plpgsql AS $$
+DECLARE
+    new_keys text[];
+BEGIN
+    EXECUTE format($q$
+        SELECT array_agg(key ORDER BY key) FROM (
+            SELECT DISTINCT ag_catalog.agtype_to_text(k) AS key
+            FROM %s, ag_catalog.age_unnest(ag_catalog.age_keys(properties)) AS k
+        ) keys
+        WHERE key <> '' AND octet_length(key) < 64
+          AND NOT EXISTS (SELECT FROM pg_catalog.pg_attribute
+                          WHERE attrelid = $1 AND attname = key AND NOT attisdropped)
+    $q$, tbl) INTO new_keys USING tbl;
+
+    IF new_keys IS NULL THEN
+        RETURN;
+    END IF;
+
+    -- One ALTER TABLE for all keys, so the table is rewritten only once.
+    EXECUTE format('ALTER TABLE %s %s', tbl, (
+        SELECT string_agg(format(
+            'ADD COLUMN %I text GENERATED ALWAYS AS (properties OPERATOR(ag_catalog.->>) %L::text) STORED',
+            key, key), ', ')
+        FROM unnest(new_keys) AS key));
+
+    RETURN QUERY SELECT unnest(new_keys);
+END
+$$;
+
+CREATE OR REPLACE PROCEDURE peo4c.sync_property_columns()
+LANGUAGE plpgsql AS $$
+DECLARE
+    t record;
+    added text[];
+BEGIN
+    DELETE FROM peo4c.property_column_state s
+    WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_class WHERE oid = s.relid);
+
+    FOR t IN
+        SELECT l.relation AS tbl, c.changes
+        FROM ag_catalog.ag_label AS l
+        CROSS JOIN LATERAL (SELECT pg_stat_get_tuples_inserted(l.relation)
+                                   + pg_stat_get_tuples_updated(l.relation) AS changes) AS c
+        LEFT JOIN peo4c.property_column_state AS s ON s.relid = l.relation
+        WHERE l.name NOT IN ('_ag_label_vertex', '_ag_label_edge')
+          AND s.changes IS DISTINCT FROM c.changes
+    LOOP
+        BEGIN
+            -- Adding a column rewrites the table under an exclusive lock;
+            -- rather than queue behind a long transaction, retry next run.
+            PERFORM set_config('lock_timeout', '2s', true);
+            SELECT array_agg(key) INTO added FROM peo4c.add_property_columns(t.tbl) AS key;
+            -- The rewrite counts as inserts in this transaction; include them
+            -- so the table is not scanned again for its own rewrite.
+            INSERT INTO peo4c.property_column_state AS s
+            VALUES (t.tbl, t.changes + pg_stat_get_xact_tuples_inserted(t.tbl)
+                                     + pg_stat_get_xact_tuples_updated(t.tbl))
+            ON CONFLICT (relid) DO UPDATE SET changes = excluded.changes;
+            IF added IS NOT NULL THEN
+                RAISE NOTICE 'added property columns to %: %', t.tbl, array_to_string(added, ', ');
+            END IF;
+        EXCEPTION WHEN OTHERS THEN
+            RAISE WARNING 'could not add property columns to %: %', t.tbl, SQLERRM;
+        END;
+        COMMIT;
+    END LOOP;
+END
+$$;