diff --git a/.containerignore b/.containerignore index 897af85..a5835d0 100644 --- a/.containerignore +++ b/.containerignore @@ -7,3 +7,4 @@ tests **/dist **/*.egg-info .idea +.env diff --git a/Containerfile b/Containerfile index 9c0288c..e61513b 100644 --- a/Containerfile +++ b/Containerfile @@ -3,12 +3,19 @@ # peo: neo4j-browser on top of PostgreSQL + Apache AGE. # -# Stage 1 builds the neo4j-browser static frontend from source. -# Stage 2 is the Apache AGE image (PostgreSQL with the AGE extension) plus the -# peo Python package, which speaks the Bolt protocol to the browser and -# translates Cypher into AGE `cypher()` calls, and pgAdmin as a SQL interface. +# Two images can be built from this file: +# +# full (the default): the Apache AGE image (PostgreSQL with the AGE extension) +# plus peo, neo4j-browser and pgAdmin, all in one container. +# slim (--target slim): only peo and neo4j-browser, for connecting to a +# separate PostgreSQL server with AGE; see compose.yaml. +# +# peo is the Python package that speaks the Bolt protocol to the browser and +# translates Cypher into AGE `cypher()` calls. The browser stage builds the +# neo4j-browser static frontend from source for both images. ARG AGE_IMAGE=docker.io/apache/age:release_PG18_1.8.0 +ARG SLIM_BASE_IMAGE=docker.io/library/debian:trixie-slim # Commit of apache/age's release/PG18/1.8.0 branch, for AGE's LICENSE and NOTICE. ARG AGE_SOURCE_REF=b570cf7c1486863f77c14e9c0e07b0e9bfd01bf4 ARG PGADMIN_VERSION=9.18 @@ -28,7 +35,42 @@ RUN yarn install --frozen-lockfile --network-timeout 600000 RUN yarn build && test -f dist/index.html -FROM ${AGE_IMAGE} +FROM ${SLIM_BASE_IMAGE} AS slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-psycopg python3-venv \ + && rm -rf /var/lib/apt/lists/* + +# peo itself, using Debian's psycopg and the other dependencies from PyPI. License +# notices: see THIRD_PARTY_NOTICES.md. +COPY container/collect-python-licenses.py /tmp/ +COPY container/licenses/ /tmp/licenses/ +COPY pyproject.toml README.md LICENSE /tmp/peo/ +COPY peo /tmp/peo/peo +RUN python3 -m venv --system-site-packages /opt/peo/venv \ + && /opt/peo/venv/bin/pip install --no-cache-dir "/tmp/peo[browser]" \ + && /opt/peo/venv/bin/python /tmp/collect-python-licenses.py \ + /usr/share/doc/peo/peo-python-licenses /tmp/licenses \ + && rm -r /tmp/peo /tmp/collect-python-licenses.py /tmp/licenses +COPY LICENSE THIRD_PARTY_NOTICES.md /usr/share/doc/peo/ +COPY --from=browser /src/LICENSE /src/NOTICE.txt /src/LICENSES.txt /neo4j-browser-source.tar.gz \ + /usr/share/doc/peo/neo4j-browser/ +COPY --from=browser /src/dist /opt/peo/browser + +RUN useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin peo +USER peo + +ENV PEO_BROWSER_DIR=/opt/peo/browser \ + PEO_HTTP_PORT=7474 \ + PEO_BOLT_PORT=7687 + +# 7474: neo4j-browser (HTTP), 7687: Bolt +EXPOSE 7474 7687 + +CMD ["/opt/peo/venv/bin/peo"] + + +FROM ${AGE_IMAGE} AS full ARG AGE_SOURCE_REF ARG PGADMIN_VERSION diff --git a/README.md b/README.md index eab8540..125a972 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ graph engine. peo is a small Python server that speaks Neo4j's Bolt protocol and translates Cypher into AGE `cypher()` calls. It can also serve neo4j-browser. A [container -image](#container-image) bundles everything, together with +image](#container-images) bundles everything, together with [pgAdmin](https://www.pgadmin.org/) for working with the same data in SQL. ``` @@ -37,7 +37,8 @@ and password. peo does not include neo4j-browser itself. To serve it, install the `browser` extra ([Starlette](https://www.starlette.io/) and [uvicorn](https://www.uvicorn.org/)), build neo4j-browser and point `PEO_BROWSER_DIR` at its `dist` directory; it is then -served on . The container image does all of this for you. +served on . The [container images](#container-images) do all of +this for you. ```sh pip install 'peo[binary,browser]' @@ -65,7 +66,7 @@ peo is configured through environment variables: - **Databases are AGE graphs.** Every AGE graph shows up as a Neo4j database, and `:use ` switches between them. Clients that do not pick a database get `PEO_DEFAULT_GRAPH` - (`neo4j`); the container image creates that graph on first start. `CREATE DATABASE x` and `DROP DATABASE x` on the `system` + (`neo4j`); the full container image and the compose setup create that graph on first start. `CREATE DATABASE x` and `DROP DATABASE x` on the `system` database call `create_graph` / `drop_graph`; AGE requires names of at least 3 characters. - **Queries** are sent to AGE as `SELECT * FROM cypher('', $$ $$) AS (...)`. peo works out the result columns from the final `RETURN` clause and inlines @@ -108,38 +109,21 @@ and `REMOVE`, and they can be indexed. Some things to keep in mind: - AGE has no date or time types, so store those as ISO 8601 strings and cast them in SQL. -## Querying the graph from SQL +## Container images -AGE stores each node label and relationship type as a table in the graph's schema, -with all properties in one `agtype` column, `properties`; `properties ->> 'key'::text` -returns a property as text. For typed columns, add them yourself, either as a view or -as a generated column: +The [`Containerfile`](Containerfile) builds two images: -```sql -CREATE VIEW neo4j.people AS - SELECT id, properties ->> 'name'::text AS name, (properties ->> 'age'::text)::bigint AS age - FROM neo4j."Person"; +- **full** (the default): PostgreSQL with AGE, peo, neo4j-browser and pgAdmin in one + container. +- **slim** (`--target slim`): only peo and neo4j-browser, for a PostgreSQL server + with AGE that runs elsewhere. [`compose.yaml`](compose.yaml) runs it next to + separate PostgreSQL and pgAdmin containers. -ALTER TABLE neo4j."Person" - ADD COLUMN age bigint GENERATED ALWAYS AS ((properties ->> 'age'::text)::bigint) STORED; -``` +Any OCI-compatible build tool works. Docker does not pick up a `Containerfile` by +itself, so pass it explicitly: `docker build -f Containerfile -t peo .`. `docker` +takes the same options as `podman` in the commands below. -PostgreSQL keeps generated columns up to date on every Cypher `CREATE`, `MERGE`, `SET` -and `REMOVE`, and they can be indexed. Some things to keep in mind: - -- A value that does not fit the column's type makes the Cypher write fail: with the - column above, `CREATE (:Person {age: 'old'})` is rejected. A view only fails when it - is read. -- AGE does not allow dropping columns from its label tables, so a generated column - cannot be removed again. -- Adding a stored generated column rewrites the table and locks it while doing so. -- AGE has no date or time types, so store those as ISO 8601 strings and cast them in - SQL. - -## Container image - -The container image runs PostgreSQL with AGE, peo, neo4j-browser and pgAdmin in one -container: +### Everything in one container ```sh podman build -t peo . @@ -159,10 +143,6 @@ pgAdmin comes with the bundled PostgreSQL server already registered as "peo"; it asks for the PostgreSQL password the first time you open it. AGE stores each graph as a schema of the same name, with one table per label. -Any OCI-compatible build tool works. With Docker, pass the file name explicitly: -`docker build -f Containerfile -t peo .`; `docker run` takes the same options as -`podman run`. - Besides peo's own variables above, the image uses these: | Variable | Default | Purpose | @@ -174,12 +154,52 @@ Besides peo's own variables above, the image uses these: | `PEO_PGADMIN` | `true` | Set to `false` to not start pgAdmin. | | `PEO_PGADMIN_PORT` | `5050` | Port pgAdmin listens on inside the container. | -Build arguments: `AGE_IMAGE` (base image, default `apache/age:release_PG18_1.8.0`), -`NEO4J_BROWSER_REF` (neo4j-browser commit to build) and `PGADMIN_VERSION` (default `9.18`). - pgAdmin keeps its users and settings in `/var/lib/pgadmin`; changing `PGADMIN_DEFAULT_*` after the first start has no effect unless that directory is empty. +### peo with a separate PostgreSQL server + +The slim image runs peo and serves neo4j-browser, and is configured with the +variables in [Configuration](#configuration). Point it at any PostgreSQL server +that has AGE installed and loaded (`shared_preload_libraries=age`): + +```sh +podman build --target slim -t peo:slim . +podman run -d --name peo -p 7474:7474 -p 7687:7687 \ + -e PEO_PG_HOST=db.example.com -e PEO_PG_DATABASE=graphs peo:slim +``` + +peo creates no graphs of its own. If the database has none yet, run +`CREATE DATABASE neo4j` on the `system` database (`:use system` in neo4j-browser), +or `SELECT ag_catalog.create_graph('neo4j');` in SQL. + +### Compose: peo, PostgreSQL and pgAdmin as separate containers + +[`compose.yaml`](compose.yaml) builds the slim image and runs it with +[`apache/age`](https://hub.docker.com/r/apache/age) for PostgreSQL and the official +[`dpage/pgadmin4`](https://hub.docker.com/r/dpage/pgadmin4) image: + +```sh +echo POSTGRES_PASSWORD=secret > .env +podman compose up -d +``` + +It works with `podman compose` and `docker compose`, and publishes the same ports and +logins as the full image: neo4j-browser on 7474, Bolt on 7687, pgAdmin on 5050 and +PostgreSQL on 5432. Set `PEO_POSTGRES_PORT` to publish PostgreSQL on another host +port. The default graph (`PEO_DEFAULT_GRAPH`, `neo4j`) is created when the database +is initialised, and pgAdmin comes with the server already registered as "peo". +`PGADMIN_DEFAULT_EMAIL`, `PGADMIN_DEFAULT_PASSWORD` and `PEO_LOG_LEVEL` can be set +in `.env` too. Data is kept in the `postgres-data` and `pgadmin-data` volumes; +`podman compose down -v` removes them. + +### Build arguments + +`AGE_IMAGE` (base of the full image, default `apache/age:release_PG18_1.8.0`), +`SLIM_BASE_IMAGE` (base of the slim image, default `debian:trixie-slim`), +`NEO4J_BROWSER_REF` (neo4j-browser commit to build) and `PGADMIN_VERSION` (pgAdmin in +the full image, default `9.18`). + ## Limitations This is a first version. Things that do not work (yet): @@ -215,8 +235,8 @@ python tests/smoke_test.py neo4j://localhost:7687 postgres secret ## License peo is free software, licensed under the GNU General Public License v3.0 or -later; see [LICENSE](LICENSE). The container image bundles third-party software -under its own licenses, including neo4j-browser (GPL-3.0), Apache AGE +later; see [LICENSE](LICENSE). The container images bundle third-party software +under their own licenses, including neo4j-browser (GPL-3.0), Apache AGE (Apache-2.0), PostgreSQL and pgAdmin (PostgreSQL License). See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for the full list and where each license text and source archive is in the image. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index b229ae3..a598f53 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,14 +1,23 @@ # Third-party notices peo's own code (the `peo/`, `container/` and `tests/` directories, the -`Containerfile` and `pyproject.toml`) is licensed under the GNU General Public +`Containerfile`, `compose.yaml` and `pyproject.toml`) is licensed under the GNU General Public License v3.0 or later; see [LICENSE](LICENSE). -The container image built from this repository also contains the third-party +The container images built from this repository also contain the third-party software below. None of it is stored in this repository: it is downloaded when -the image is built. Inside the image, license texts are collected under +an image is built. Inside the images, license texts are collected under `/usr/share/doc/peo/`. +The full image (the default build target) contains everything in the table. The +slim image (`--target slim`) contains only neo4j-browser and the JavaScript +libraries bundled into it, the Debian base system with Python 3 and +`python3-psycopg`, and the Python packages peo depends on. + +`compose.yaml` does not bundle PostgreSQL, AGE or pgAdmin: it runs the +`apache/age` and `dpage/pgadmin4` images from Docker Hub, which come with their +own license information. + | Component | License | Copyright | License text in the image | | --- | --- | --- | --- | | [neo4j-browser](https://github.com/neo4j/neo4j-browser), built unmodified from the commit pinned in the `Containerfile` | GPL-3.0-or-later | Neo4j Sweden AB | `/usr/share/doc/peo/neo4j-browser/` (`LICENSE`, `NOTICE.txt`) | diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..b216280 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2026 Bart Broere +# SPDX-License-Identifier: GPL-3.0-or-later +# +# peo, PostgreSQL with Apache AGE and pgAdmin as separate containers. peo runs +# from the slim image (see Containerfile). Start with: +# +# POSTGRES_PASSWORD=secret podman compose up -d +# +# or put POSTGRES_PASSWORD in a .env file next to this one. + +name: peo + +services: + postgres: + image: docker.io/apache/age:release_PG18_1.8.0 + command: ["postgres", "-c", "shared_preload_libraries=age"] + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD, for example in a .env file} + PEO_DEFAULT_GRAPH: ${PEO_DEFAULT_GRAPH:-neo4j} + volumes: + - postgres-data:/var/lib/postgresql + # Creates the default graph when the database is initialised. + - ./container/initdb/10-peo.sh:/docker-entrypoint-initdb.d/10-peo.sh:ro + ports: + - "${PEO_POSTGRES_PORT:-5432}:5432" + healthcheck: + # Over TCP, so the temporary server that runs the init scripts does not count. + test: ["CMD", "pg_isready", "--host", "127.0.0.1", "--username", "postgres"] + interval: 5s + timeout: 5s + retries: 20 + + peo: + build: + context: . + dockerfile: Containerfile + target: slim + image: localhost/peo:slim + environment: + PEO_PG_HOST: postgres + PEO_PG_DATABASE: postgres + PEO_DEFAULT_GRAPH: ${PEO_DEFAULT_GRAPH:-neo4j} + PEO_LOG_LEVEL: ${PEO_LOG_LEVEL:-INFO} + ports: + - "7474:7474" + - "7687:7687" + depends_on: + postgres: + condition: service_healthy + + pgadmin: + image: docker.io/dpage/pgadmin4:9.18 + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@example.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-${POSTGRES_PASSWORD}} + PGADMIN_CONFIG_UPGRADE_CHECK_ENABLED: "False" + volumes: + - pgadmin-data:/var/lib/pgadmin + # Registers the postgres service on first start. + - ./container/pgadmin/compose-servers.json:/pgadmin4/servers.json:ro + ports: + - "5050:80" + depends_on: + postgres: + condition: service_healthy + +volumes: + postgres-data: + pgadmin-data: diff --git a/container/pgadmin/compose-servers.json b/container/pgadmin/compose-servers.json new file mode 100644 index 0000000..142e10a --- /dev/null +++ b/container/pgadmin/compose-servers.json @@ -0,0 +1,13 @@ +{ + "Servers": { + "1": { + "Name": "peo", + "Group": "Servers", + "Host": "postgres", + "Port": 5432, + "MaintenanceDB": "postgres", + "Username": "postgres", + "SSLMode": "prefer" + } + } +}