bartbroere/boltage
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove automatic generated property columns

Browse Source 
The columns could only be text, since a typed column makes Cypher writes
fail on values of another type. Leave typed columns and views to the
database administrator instead, and document how to set them up.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bart BroereandClaude Opus 5.5 committed 2026-10-14 15:33:29 +02:00
1 parent fd0dda6470
commit 9a25aa11c9
7 files changed
+21 -216
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

bridge/peo/__main__.py
+1 -16
View File
@@ -7,9 +7,6 @@ import functools
import logging
import os
from psycopg.conninfo import make_conninfo
from . import property_columns
from .backend import Config
from .bolt import handle_client
from .web import HttpServer
@@ -46,19 +43,7 @@ async def main():
servers.append(await asyncio.start_server(http.handle, "0.0.0.0", http_port))
log.info("neo4j-browser on http://localhost:%d/browser/", http_port)
tasks = [s.serve_forever() for s in servers]
if env("PEO_PROPERTY_COLUMNS", "false") == "true":
conninfo = make_conninfo(
host=env("PEO_PROPERTY_COLUMNS_PG_HOST", "/var/run/postgresql"),
port=config.pg_port,
dbname=config.pg_database,
user=env("PEO_PROPERTY_COLUMNS_PG_USER", env("POSTGRES_USER", "postgres")),
application_name="peo property columns",
)
tasks.append(property_columns.sync_forever(
conninfo, float(env("PEO_PROPERTY_COLUMNS_INTERVAL", "5"))))
await asyncio.gather(*tasks)
await asyncio.gather(*(s.serve_forever() for s in servers))
if __name__ == "__main__":
bridge/peo/backend.py
-3
View File
@@ -337,9 +337,6 @@ class Session:
raise BoltFailure("Neo.ClientError.Statement.ArgumentError",
f"Invalid graph name '{name}': AGE graph names must be 3-63 characters "
"of letters, digits, '_', '.' or '-', starting with a letter or '_'.")
if name == "peo":
raise BoltFailure("Neo.ClientError.Statement.ArgumentError",
"The name 'peo' is reserved: it is the schema of peo's own SQL helpers.")
if name in await self.graphs():
if if_not_exists:
return Result([], query_type="s")
bridge/peo/property_columns.py
-47
View File
@@ -1,47 +0,0 @@
# SPDX-FileCopyrightText: 2026 Bart Broere
# SPDX-License-Identifier: GPL-3.0-or-later
"""Keeps generated property columns on AGE label tables up to date.
Installs the procedures in ``property_columns.sql`` and calls
``peo.sync_property_columns()`` every ``interval`` seconds. This needs a
connection that may alter every graph's tables, so it uses its own
(superuser) connection rather than the credentials of Bolt users.
"""
import asyncio
import logging
from pathlib import Path
import psycopg
log = logging.getLogger("peo.property_columns")
INSTALL_SQL = (Path(__file__).parent / "property_columns.sql").read_text()
def log_notice(diag):
level = logging.WARNING if diag.severity_nonlocalized == "WARNING" else logging.INFO
log.log(level, "%s", diag.message_primary)
async def sync_forever(conninfo, interval):
error = None
while True:
try:
async with await psycopg.AsyncConnection.connect(conninfo, autocommit=True) as conn:
conn.add_notice_handler(log_notice)
await conn.execute(INSTALL_SQL)
log.info("syncing property columns every %gs", interval)
error = None
while True:
await conn.execute("CALL peo.sync_property_columns()")
await asyncio.sleep(interval)
except psycopg.Error as e:
# PostgreSQL is still starting or restarted, or AGE is not
# installed: report each new problem once and keep retrying.
message = str(e).strip()
if message != error:
level = logging.DEBUG if isinstance(e, psycopg.OperationalError) else logging.WARNING
log.log(level, "property column sync unavailable, retrying: %s", message)
error = message
await asyncio.sleep(interval)
bridge/peo/property_columns.sql
-104
View File
@@ -1,104 +0,0 @@
-- SPDX-FileCopyrightText: 2026 Bart Broere
-- SPDX-License-Identifier: GPL-3.0-or-later
--
-- Generated columns for node and relationship properties.
--
-- AGE stores all properties of a node or relationship in one agtype column,
-- "properties". For every property key found in a label table, this adds a
-- stored generated column with the same name:
--
-- name text GENERATED ALWAYS AS (properties ->> 'name'::text) STORED
--
-- so the graph can be queried from SQL as SELECT name FROM graph."Person".
--
-- AGE's Cypher writes bypass the executor and fire no triggers, row or
-- statement level, so new keys cannot be caught by a trigger. Instead
-- CALL peo.sync_property_columns() looks for them; the peo bridge calls
-- it every few seconds. It only scans label tables that saw inserts or updates
-- since its previous run, according to the statistics collector, and commits
-- after each table so a table rewrite never holds locks on other tables.
-- Generated columns are kept up to date by AGE's CREATE, MERGE, SET and REMOVE.
--
-- Columns are text, like ->> on json, because a property can hold values of
-- different types in different rows and a failing cast would make writes fail.
CREATE SCHEMA IF NOT EXISTS peo;
CREATE TABLE IF NOT EXISTS peo.property_column_state (
relid oid PRIMARY KEY,
changes bigint NOT NULL -- inserts + updates seen when the table was last scanned
);
-- Adds a generated column for every property key in tbl that has no column of
-- that name yet, and returns the keys it added. Keys that are empty or longer
-- than PostgreSQL's 63-byte identifier limit are skipped, as are keys that
-- clash with an existing column such as id, start_id or end_id.
CREATE OR REPLACE FUNCTION peo.add_property_columns(tbl regclass)
RETURNS SETOF text LANGUAGE plpgsql AS $$
DECLARE
new_keys text[];
BEGIN
EXECUTE format($q$
SELECT array_agg(key ORDER BY key) FROM (
SELECT DISTINCT ag_catalog.agtype_to_text(k) AS key
FROM %s, ag_catalog.age_unnest(ag_catalog.age_keys(properties)) AS k
) keys
WHERE key <> '' AND octet_length(key) < 64
AND NOT EXISTS (SELECT FROM pg_catalog.pg_attribute
WHERE attrelid = $1 AND attname = key AND NOT attisdropped)
$q$, tbl) INTO new_keys USING tbl;
IF new_keys IS NULL THEN
RETURN;
END IF;
-- One ALTER TABLE for all keys, so the table is rewritten only once.
EXECUTE format('ALTER TABLE %s %s', tbl, (
SELECT string_agg(format(
'ADD COLUMN %I text GENERATED ALWAYS AS (properties OPERATOR(ag_catalog.->>) %L::text) STORED',
key, key), ', ')
FROM unnest(new_keys) AS key));
RETURN QUERY SELECT unnest(new_keys);
END
$$;
CREATE OR REPLACE PROCEDURE peo.sync_property_columns()
LANGUAGE plpgsql AS $$
DECLARE
t record;
added text[];
BEGIN
DELETE FROM peo.property_column_state s
WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_class WHERE oid = s.relid);
FOR t IN
SELECT l.relation AS tbl, c.changes
FROM ag_catalog.ag_label AS l
CROSS JOIN LATERAL (SELECT pg_stat_get_tuples_inserted(l.relation)
+ pg_stat_get_tuples_updated(l.relation) AS changes) AS c
LEFT JOIN peo.property_column_state AS s ON s.relid = l.relation
WHERE l.name NOT IN ('_ag_label_vertex', '_ag_label_edge')
AND s.changes IS DISTINCT FROM c.changes
LOOP
BEGIN
-- Adding a column rewrites the table under an exclusive lock;
-- rather than queue behind a long transaction, retry next run.
PERFORM set_config('lock_timeout', '2s', true);
SELECT array_agg(key) INTO added FROM peo.add_property_columns(t.tbl) AS key;
-- The rewrite counts as inserts in this transaction; include them
-- so the table is not scanned again for its own rewrite.
INSERT INTO peo.property_column_state AS s
VALUES (t.tbl, t.changes + pg_stat_get_xact_tuples_inserted(t.tbl)
+ pg_stat_get_xact_tuples_updated(t.tbl))
ON CONFLICT (relid) DO UPDATE SET changes = excluded.changes;
IF added IS NOT NULL THEN
RAISE NOTICE 'added property columns to %: %', t.tbl, array_to_string(added, ', ');
END IF;
EXCEPTION WHEN OTHERS THEN
RAISE WARNING 'could not add property columns to %: %', t.tbl, SQLERRM;
END;
COMMIT;
END LOOP;
END
$$;