From 17640a3293a59997f46b72271e441a32a4bcab7b Mon Sep 17 00:00:00 2001 From: nik-localstack Date: Tue, 12 May 2026 12:27:43 +0300 Subject: [PATCH 1/2] feat(tests): add local integration test suite for proxy and plugins Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 51 ++++++- README.md | 8 ++ plugins/tableau_hll/test.py | 34 ----- tests/README.md | 62 ++++++++ tests/conftest.py | 33 +++++ tests/test_plugin.py | 12 -- tests/test_plugins.py | 86 +++++++++++ tests/test_proxy.py | 276 ++++++++++++++++++++++++++++++++++++ 8 files changed, 515 insertions(+), 47 deletions(-) delete mode 100644 plugins/tableau_hll/test.py create mode 100644 tests/README.md create mode 100644 tests/conftest.py delete mode 100644 tests/test_plugin.py create mode 100644 tests/test_plugins.py create mode 100644 tests/test_proxy.py diff --git a/Makefile b/Makefile index 0b3b2ca..3f569bc 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,17 @@ +SHELL := /bin/bash VENV_DIR ?= .venv VENV_RUN = . $(VENV_DIR)/bin/activate PIP_CMD ?= pip +PYTHON_CMD ?= python +TEST_DEPS ?= pytest pytest-timeout +LINT_DEPS ?= ruff + +PG_TEST_CONTAINER ?= pg-proxy-local-tests +PG_TEST_IMAGE ?= postgres:16 +PG_TEST_PORT ?= 55432 +PG_TEST_USER ?= postgres +PG_TEST_PASSWORD ?= postgres +PG_TEST_DB ?= postgres usage: ## Show this help @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' @@ -14,4 +25,42 @@ install: ## Install dependencies in local virtualenv folder publish: ## Publish the library to the central PyPi repository ($(VENV_RUN); pip install twine; python ./setup.py sdist && twine upload dist/*) -.PHONY: usage install clean publish test lint +install-test: install ## Install test dependencies in local virtualenv + ($(VENV_RUN); $(PIP_CMD) install $(TEST_DEPS)) + +install-lint: install ## Install lint dependencies in local virtualenv + ($(VENV_RUN); $(PIP_CMD) install $(LINT_DEPS)) + +lint: install-lint ## Format code with ruff + $(VENV_DIR)/bin/ruff format postgresql_proxy tests plugins + +test: ## Start local PostgreSQL container and run all tests + @set -euo pipefail; \ + cleanup() { docker rm -f $(PG_TEST_CONTAINER) >/dev/null 2>&1 || true; }; \ + trap cleanup EXIT INT TERM; \ + docker rm -f $(PG_TEST_CONTAINER) >/dev/null 2>&1 || true; \ + docker run --name $(PG_TEST_CONTAINER) \ + -e POSTGRES_USER=$(PG_TEST_USER) \ + -e POSTGRES_PASSWORD=$(PG_TEST_PASSWORD) \ + -e POSTGRES_DB=$(PG_TEST_DB) \ + -p $(PG_TEST_PORT):5432 \ + -d $(PG_TEST_IMAGE) >/dev/null; \ + for i in $$(seq 1 45); do \ + if docker exec $(PG_TEST_CONTAINER) pg_isready -U $(PG_TEST_USER) >/dev/null 2>&1; then \ + echo "PostgreSQL ready on 127.0.0.1:$(PG_TEST_PORT)"; \ + break; \ + fi; \ + sleep 1; \ + done; \ + if ! docker exec $(PG_TEST_CONTAINER) pg_isready -U $(PG_TEST_USER) >/dev/null 2>&1; then \ + echo "PostgreSQL did not become ready in time"; \ + exit 1; \ + fi; \ + E2E_PG_HOST=127.0.0.1 \ + E2E_PG_PORT=$(PG_TEST_PORT) \ + E2E_PG_USER=$(PG_TEST_USER) \ + E2E_PG_PASSWORD=$(PG_TEST_PASSWORD) \ + E2E_PG_DB=$(PG_TEST_DB) \ + $(VENV_DIR)/bin/$(PYTHON_CMD) -m pytest -vv + +.PHONY: usage install install-test install-lint clean publish test lint diff --git a/README.md b/README.md index a507a92..b95a315 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,11 @@ If you want to test it, do this. Otherwise scroll down for instructions on how t - add stop() method to proxy; refactor logging - v0.0.2 - fix socket file descriptors under Linux + +## Testing + +Run the full local test suite (starts a disposable PostgreSQL container automatically): + +```bash +make test +``` diff --git a/plugins/tableau_hll/test.py b/plugins/tableau_hll/test.py deleted file mode 100644 index 46240aa..0000000 --- a/plugins/tableau_hll/test.py +++ /dev/null @@ -1,34 +0,0 @@ -import collections -import os -import plugins.tableau_hll as hll -import yaml - -def test_context(): - with open(os.path.dirname(__file__) + '/config.yml', 'r') as fp: - config = yaml.load(fp) - InstanceConfig = collections.namedtuple('InstanceConfig', 'redirect') - Redirect = collections.namedtuple('Redirect', 'name host port') - return { - 'instance_config': InstanceConfig(redirect=Redirect(**config['redirect'])), - 'connect_params': config['connect_params'] - } - - -def run(): - queries = [ - ( - 'SELECT COUNT(DISTINCT "crm_data_source"."Set of Customers") AS "ctd:Set of Customers:ok"\nFROM "crm_dim"."crm_data_source" "crm_data_source"\nHAVING (COUNT(1) > 0);', - 'SELECT hll_cardinality(hll_union_agg("crm_data_source"."Set of Customers")) :: BIGINT AS "ctd:Set of Customers:ok"\nFROM "crm_dim"."crm_data_source" "crm_data_source"\nHAVING (COUNT(1) > 0);' - ), - ( - 'BEGIN;declare "SQL_CUR0x7fb46c01e3b0" cursor with hold for SELECT CAST("crm_data_source"."Campaign Name" AS TEXT) AS "Campaign Name",\n COUNT(DISTINCT "crm_data_source"."Set of Unique Clicks") AS "usr:# Unique Customers (copy):ok"\nFROM "crm_dim"."crm_data_source" "crm_data_source"\nGROUP BY 1;fetch 2048 in "SQL_CUR0x7fb46c01e3b0"', - 'BEGIN;declare "SQL_CUR0x7fb46c01e3b0" cursor with hold for SELECT CAST("crm_data_source"."Campaign Name" AS TEXT) AS "Campaign Name",\n hll_cardinality(hll_union_agg("crm_data_source"."Set of Unique Clicks")) :: BIGINT AS "usr:# Unique Customers (copy):ok"\nFROM "crm_dim"."crm_data_source" "crm_data_source"\nGROUP BY 1;fetch 2048 in "SQL_CUR0x7fb46c01e3b0"' - ) - ] - context = test_context() - for src, dst in queries: - res = hll.rewrite_query(src, context) - try: - assert res == dst - except AssertionError: - print(f"Rewriting query:\n\n{src}\n\nExpecting:\n\n{dst}\n\nGot:\n\n{res}") diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..bbeb53c --- /dev/null +++ b/tests/README.md @@ -0,0 +1,62 @@ +# Testing Guide + +All tests in this repo require a real PostgreSQL server and are organized at the top level: + +- `test_proxy.py`: proxy behavior tests (connection, SSL, hang regressions) +- `test_plugins.py`: plugin integration tests (HLL rewrite behavior) + +## Prerequisites + +- Python `3.13` (same version as CI) +- Docker (for local disposable PostgreSQL) +- `psql` (`postgresql-client`) +- `openssl` (SSL tests generate a temporary self-signed cert/key at runtime) + +Install Python deps in the project virtualenv: + +```bash +make install-test +``` + +## Which command should I use? + +- Fastest full local run with disposable Postgres: `make test` +- Run only proxy tests (using your own Postgres): `python -m pytest tests/test_proxy.py -vv` +- Run only plugin tests: `python -m pytest tests/test_plugins.py -vv` + +## 1) Full local suite (recommended) + +`make test` starts a temporary PostgreSQL container, waits for readiness, sets DB env vars, then runs: + +```bash +python -m pytest -vv +``` + +Use it when you want one command that matches normal contributor workflow. + +```bash +make test +``` + +## 2) DB-backed proxy tests against an existing PostgreSQL + +If you already have PostgreSQL running, set connection env vars and run only proxy tests: + +```bash +export E2E_PG_HOST=127.0.0.1 +export E2E_PG_PORT=5432 +export E2E_PG_USER=postgres +export E2E_PG_PASSWORD=postgres +export E2E_PG_DB=postgres +python -m pytest tests/test_proxy.py -vv +``` + +If PostgreSQL is not reachable, tests fail fast at startup. + +## 3) Plugin integration tests + +```bash +python -m pytest tests/test_plugins.py -vv +``` + +Requires PostgreSQL to be running with the `E2E_PG_*` env vars set (see section 2). diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..ba7a866 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,33 @@ +import os + +import psycopg2 +import pytest + + +@pytest.fixture(scope="session") +def postgres_settings(): + """PostgreSQL connection settings from environment or defaults.""" + return { + "host": os.environ.get("E2E_PG_HOST", "127.0.0.1"), + "port": int(os.environ.get("E2E_PG_PORT", "5432")), + "user": os.environ.get("E2E_PG_USER", "postgres"), + "password": os.environ.get("E2E_PG_PASSWORD", "postgres"), + "dbname": os.environ.get("E2E_PG_DB", "postgres"), + } + + +@pytest.fixture(scope="session", autouse=True) +def ensure_postgres_available(postgres_settings): + """Ensure PostgreSQL backend is available before running any tests.""" + try: + with psycopg2.connect( + connect_timeout=3, sslmode="disable", **postgres_settings + ) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) + except Exception as err: # pragma: no cover - environment dependent + pytest.fail( + f"PostgreSQL backend is required for tests but is not reachable: {err}" + ) + diff --git a/tests/test_plugin.py b/tests/test_plugin.py deleted file mode 100644 index c05c00b..0000000 --- a/tests/test_plugin.py +++ /dev/null @@ -1,12 +0,0 @@ -import importlib -import sys - -""" Rudimentary test runner for plugins -Pass in the plugin name as an argument, and make sure that there is a test.py file with a run() function in the plugin -directory. -""" - -plugin = sys.argv[1] -test = importlib.import_module('plugins.' + plugin + '.test') - -test.run() diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 0000000..cfe6c6e --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,86 @@ +"""Plugin integration tests. + +These tests verify we can load plugins and that plugin behavior works against a real Postgres backend. +""" + +import collections +import importlib + +import psycopg2 +import pytest + +import plugins.tableau_hll as hll + + +@pytest.fixture() +def plugin_context(postgres_settings, monkeypatch): + # plugin's internal psycopg2 connection does not pass password, so provide it via libpq env var + monkeypatch.setenv("PGPASSWORD", postgres_settings["password"]) + + with psycopg2.connect(sslmode="disable", **postgres_settings) as conn: + conn.autocommit = True + with conn.cursor() as cur: + cur.execute('CREATE SCHEMA IF NOT EXISTS "crm_dim";') + cur.execute('DROP TABLE IF EXISTS "crm_dim"."crm_data_source";') + cur.execute( + """ + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'hll' AND typtype = 'd') THEN + DROP DOMAIN hll; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'hll') THEN + CREATE TYPE hll AS (v text); + END IF; + END $$; + """ + ) + cur.execute( + 'CREATE TABLE "crm_dim"."crm_data_source" (' + '"Set of Customers" hll, ' + '"Campaign Name" text);' + ) + + InstanceConfig = collections.namedtuple("InstanceConfig", "redirect") + Redirect = collections.namedtuple("Redirect", "name host port") + return { + "instance_config": InstanceConfig( + redirect=Redirect( + name="postgres", + host=postgres_settings["host"], + port=postgres_settings["port"], + ) + ), + "connect_params": { + "user": postgres_settings["user"], + "database": postgres_settings["dbname"], + }, + } + + +def test_rewrite_query_for_hll_column(plugin_context): + src = ( + 'SELECT COUNT(DISTINCT "crm_data_source"."Set of Customers") AS "ctd:Set of Customers:ok"\n' + 'FROM "crm_dim"."crm_data_source" "crm_data_source"\n' + "HAVING (COUNT(1) > 0);" + ) + + res = hll.rewrite_query(src, plugin_context) + assert "hll_cardinality(hll_union_agg" in res + + +def test_plugin_module_loads_and_exposes_rewriter(): + module = importlib.import_module("plugins.tableau_hll") + assert hasattr(module, "rewrite_query") + assert callable(module.rewrite_query) + + +def test_does_not_rewrite_non_hll_column(plugin_context): + src = ( + 'SELECT COUNT(DISTINCT "crm_data_source"."Campaign Name") AS "ctd:Campaign Name:ok"\n' + 'FROM "crm_dim"."crm_data_source" "crm_data_source"\n' + "HAVING (COUNT(1) > 0);" + ) + + res = hll.rewrite_query(src, plugin_context) + assert "hll_cardinality(hll_union_agg" not in res diff --git a/tests/test_proxy.py b/tests/test_proxy.py new file mode 100644 index 0000000..105eb6c --- /dev/null +++ b/tests/test_proxy.py @@ -0,0 +1,276 @@ +import contextlib +import os +import shutil +import socket +import ssl +import subprocess +import tempfile +import threading +import time + +import psycopg2 +import pytest + +from postgresql_proxy import config_schema as cfg +from postgresql_proxy.proxy import Proxy + + + + +def _get_free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _wait_for_listen_port(host: str, port: int, timeout: float = 5.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(0.2) + if sock.connect_ex((host, port)) == 0: + return + time.sleep(0.05) + raise TimeoutError(f"Proxy did not start listening on {host}:{port} in {timeout}s") + + +def _build_dump_like_sql(table_count: int = 12, rows_per_table: int = 100) -> str: + chunks = ["BEGIN;"] + for table_idx in range(table_count): + table_name = f"e2e_batch_{table_idx}" + chunks.append(f"DROP TABLE IF EXISTS {table_name};") + chunks.append(f"CREATE TABLE {table_name} (id INTEGER, payload TEXT);") + chunks.append(f"COPY {table_name} (id, payload) FROM STDIN;") + for row_idx in range(rows_per_table): + chunks.append(f"{row_idx}\trow_{table_idx}_{row_idx}") + chunks.append("\\.") + chunks.append(f"SELECT COUNT(*) FROM {table_name};") + + chunks.append("SELECT 'BATCH_OK';") + chunks.append("COMMIT;") + return "\n".join(chunks) + "\n" + + +def _run_psql_file( + postgres_settings, port: int, sql_file_path: str, timeout_sec: int = 60 +): + cmd = [ + "psql", + "-X", + "-q", + "-tA", + "-v", + "ON_ERROR_STOP=1", + "-h", + "127.0.0.1", + "-p", + str(port), + "-U", + postgres_settings["user"], + "-d", + postgres_settings["dbname"], + "-f", + sql_file_path, + ] + env = { + **os.environ, + "PGPASSWORD": postgres_settings["password"], + "PGSSLMODE": "require", + } + return subprocess.run( + cmd, + env=env, + capture_output=True, + text=True, + timeout=timeout_sec, + check=False, + ) + + +@contextlib.contextmanager +def _temporary_server_cert_pair(): + if shutil.which("openssl") is None: + pytest.fail("openssl is required for SSL E2E tests but was not found in PATH") + + with tempfile.TemporaryDirectory(prefix="proxy-e2e-cert-") as tmp_dir: + cert_path = os.path.join(tmp_dir, "server.crt") + key_path = os.path.join(tmp_dir, "server.key") + result = subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-sha256", + "-days", + "1", + "-nodes", + "-subj", + "/CN=localhost", + "-keyout", + key_path, + "-out", + cert_path, + ], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + err_tail = "\n".join((result.stderr or "").splitlines()[-20:]) + pytest.fail( + f"Failed to generate temporary TLS cert/key for E2E tests (rc={result.returncode}): {err_tail}" + ) + + yield cert_path, key_path + + +@contextlib.contextmanager +def _run_proxy(postgres_settings, ssl_context: ssl.SSLContext | None = None): + proxy_port = _get_free_tcp_port() + instance = cfg.InstanceSettings( + { + "listen": {"name": "proxy", "host": "127.0.0.1", "port": proxy_port}, + "redirect": { + "name": "postgres", + "host": postgres_settings["host"], + "port": postgres_settings["port"], + }, + # Keep interceptors active with default no-op behavior. + "intercept": {"commands": {}, "responses": {}}, + } + ) + if not hasattr(instance.intercept.responses, "parameter_status"): + instance.intercept.responses.parameter_status = [] + + proxy = Proxy(instance, plugins={}, debug=True, ssl_context=ssl_context) + thread = threading.Thread( + target=proxy.listen, kwargs={"max_connections": 32}, daemon=True + ) + thread.start() + + _wait_for_listen_port("127.0.0.1", proxy_port) + + try: + yield proxy_port + finally: + proxy.stop() + # Wake selector.select(timeout=1) so shutdown is immediate. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as wake_sock: + wake_sock.settimeout(0.2) + wake_sock.connect_ex(("127.0.0.1", proxy_port)) + thread.join(timeout=4) + assert not thread.is_alive(), "Proxy thread did not stop cleanly" + + +@pytest.fixture() +def plain_proxy_port(postgres_settings): + with _run_proxy(postgres_settings) as proxy_port: + yield proxy_port + + +@pytest.fixture() +def ssl_proxy_port(postgres_settings): + with _temporary_server_cert_pair() as (cert_path, key_path): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_context.load_cert_chain(certfile=cert_path, keyfile=key_path) + with _run_proxy(postgres_settings, ssl_context=ssl_context) as proxy_port: + yield proxy_port + + +@pytest.mark.timeout(20) +def test_connect_query_without_ssl(postgres_settings, plain_proxy_port): + with psycopg2.connect( + host="127.0.0.1", + port=plain_proxy_port, + user=postgres_settings["user"], + password=postgres_settings["password"], + dbname=postgres_settings["dbname"], + sslmode="disable", + connect_timeout=3, + ) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) + + +@pytest.mark.timeout(20) +def test_connect_query_with_ssl(postgres_settings, ssl_proxy_port): + with psycopg2.connect( + host="127.0.0.1", + port=ssl_proxy_port, + user=postgres_settings["user"], + password=postgres_settings["password"], + dbname=postgres_settings["dbname"], + sslmode="require", + connect_timeout=3, + ) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) + + +@pytest.mark.timeout(60) +def test_repeated_connect_query_smoke_no_hang(postgres_settings, plain_proxy_port): + for i in range(20): + with psycopg2.connect( + host="127.0.0.1", + port=plain_proxy_port, + user=postgres_settings["user"], + password=postgres_settings["password"], + dbname=postgres_settings["dbname"], + sslmode="disable", + connect_timeout=3, + ) as conn: + with conn.cursor() as cur: + cur.execute("SELECT %s", (i,)) + assert cur.fetchone() == (i,) + + +@pytest.mark.timeout(60) +def test_psql_ssl_file_batch_stress_no_hang(postgres_settings, ssl_proxy_port): + if shutil.which("psql") is None: + pytest.fail("psql is required for this test but was not found in PATH") + + sql_file_path = None + try: + sql_content = _build_dump_like_sql(table_count=24, rows_per_table=300) + with tempfile.NamedTemporaryFile("w", suffix=".sql", delete=False) as tmp_file: + tmp_file.write(sql_content) + sql_file_path = tmp_file.name + + for run_idx in range(3): + started = time.time() + try: + result = _run_psql_file( + postgres_settings, + port=ssl_proxy_port, + sql_file_path=sql_file_path, + timeout_sec=60, + ) + except subprocess.TimeoutExpired as err: + pytest.fail( + "psql -f batch timed out over SSL via proxy " + f"(run={run_idx + 1}, timeout={err.timeout}s)" + ) + + elapsed = time.time() - started + if result.returncode != 0: + out_tail = "\n".join((result.stdout or "").splitlines()[-20:]) + err_tail = "\n".join((result.stderr or "").splitlines()[-20:]) + pytest.fail( + "psql -f batch failed over SSL via proxy " + f"(run={run_idx + 1}, rc={result.returncode}, elapsed={elapsed:.2f}s) " + f"stdout_tail={out_tail} stderr_tail={err_tail}" + ) + + if "BATCH_OK" not in (result.stdout or ""): + out_tail = "\n".join((result.stdout or "").splitlines()[-20:]) + pytest.fail( + "psql -f batch succeeded but expected marker missing " + f"(run={run_idx + 1}, elapsed={elapsed:.2f}s) stdout_tail={out_tail}" + ) + finally: + if sql_file_path and os.path.exists(sql_file_path): + os.unlink(sql_file_path) From 6125cffbf130c303501a3f2b739c7e390d9001cd Mon Sep 17 00:00:00 2001 From: nik-localstack Date: Wed, 13 May 2026 11:29:42 +0300 Subject: [PATCH 2/2] Address PR comments & change python version support --- Makefile | 8 +-- README.md | 113 ++++++++++++++++++++++++++--------- postgresql_proxy/__main__.py | 3 + postgresql_proxy/proxy.py | 6 +- requirements-lint.txt | 1 + requirements-test.txt | 2 + setup.py | 11 +--- tests/README.md | 62 ------------------- tests/conftest.py | 1 - tests/test_proxy.py | 18 ++---- 10 files changed, 108 insertions(+), 117 deletions(-) create mode 100644 postgresql_proxy/__main__.py create mode 100644 requirements-lint.txt create mode 100644 requirements-test.txt delete mode 100644 tests/README.md diff --git a/Makefile b/Makefile index 3f569bc..c70b861 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ VENV_DIR ?= .venv VENV_RUN = . $(VENV_DIR)/bin/activate PIP_CMD ?= pip PYTHON_CMD ?= python -TEST_DEPS ?= pytest pytest-timeout -LINT_DEPS ?= ruff +TEST_REQS ?= requirements-test.txt +LINT_REQS ?= requirements-lint.txt PG_TEST_CONTAINER ?= pg-proxy-local-tests PG_TEST_IMAGE ?= postgres:16 @@ -26,10 +26,10 @@ publish: ## Publish the library to the central PyPi repository ($(VENV_RUN); pip install twine; python ./setup.py sdist && twine upload dist/*) install-test: install ## Install test dependencies in local virtualenv - ($(VENV_RUN); $(PIP_CMD) install $(TEST_DEPS)) + ($(VENV_RUN); $(PIP_CMD) install -r $(TEST_REQS)) install-lint: install ## Install lint dependencies in local virtualenv - ($(VENV_RUN); $(PIP_CMD) install $(LINT_DEPS)) + ($(VENV_RUN); $(PIP_CMD) install -r $(LINT_REQS)) lint: install-lint ## Format code with ruff $(VENV_DIR)/bin/ruff format postgresql_proxy tests plugins diff --git a/README.md b/README.md index b95a315..6283025 100644 --- a/README.md +++ b/README.md @@ -5,18 +5,22 @@ Serves as a proper server that Postgresql clients can connect to. Can modify pac Currently used for rewriting queries to force proper use of postgres-hll module by external proprietary software that doesn't know about that functionality ## Installing -### Linux -1. Make sure you have [python3 and pip3 installed on your system](https://stackoverflow.com/questions/6587507/how-to-install-pip-with-python-3#6587528). It has been tested with Python3.6 but should also run on Python3.5. -2. Clone it locally and cd to that directory - ``` - git clone git@github.com:kfzteile24/postgresql-proxy.git - cd postgresql-proxy - ``` -3. Run [setup.sh](setup.sh) - ``` - ./setup.sh - ``` -4. Make a copy of [config.yml.example](config.yml.example) called `config.yml` and configure your proxy instances. Create the log directories if they're not there. + +Requires Python `3.13+`. + +1. Clone the repository: + ```bash + git clone git@github.com:localstack/postgresql-proxy.git + cd postgresql-proxy + ``` +2. Install dependencies into a local virtualenv: + ```bash + make install + ``` +3. Copy the example config and edit it for your environment: + ```bash + cp config.yml.example config.yml + ``` ## Configuring In the `config.yml` file you can define the following things @@ -24,7 +28,7 @@ In the `config.yml` file you can define the following things A list of dynamically loaded modules that reside in the [plugins](plugins) directory. These plugins can be used in later configuration, to intercept queries, commands, or responses. View plugin documentation for example plugins for more details on how to do that. ### Settings General application settings. Currently the following settings are used -* `log-level` - the log level for the general log. See [python logging](https://docs.python.org/3.6/library/logging.html) for more details about the logging functionality +* `log-level` - the log level for the general log. See [python logging](https://docs.python.org/3/library/logging.html) for more details about the logging functionality * `general-log` - the location for the general log. All general messages go in there. * `intercept-log` - the location for the intercept log. Intercepted messages and return values from various enabled plugins will be written there. This log can be quite verbose as it contains the full binary messages being circulated. @@ -42,19 +46,22 @@ Make sure to manage the logs yourself, as they accumulate and take up disk space Each interceptor definition must have a `plugin`, which should also be present in the [plugins](#Plugins) configuration, and a `function`, that is found directly in that module, that will be called each time with the intercepted message as a byte string, and a context variable that is an instance of the `Proxy` class, that contains connection information and other useful stuff. -## Running in testing mode -If you want to test it, do this. Otherwise scroll down for instructions on how to install it as a service -### Linux -1. Activate the virtual environment - ``` - source .venv/bin/activate - ``` -2. Run it - ``` - python proxy.py - ``` - -### Changelog +## Running + +Activate the virtualenv and run the proxy directly: + +```bash +source .venv/bin/activate +python -m postgresql_proxy +``` + +Or run it without activating the venv: + +```bash +.venv/bin/python -m postgresql_proxy +``` + +## Changelog - v0.3.1 - Fix SSL COPY stalls by draining pending SSL buffer after recv [#11](https://github.com/localstack/postgresql-proxy/pull/11) - Fix intermittent `BlockingIOError` on macOS during SSL negotiation @@ -84,8 +91,60 @@ If you want to test it, do this. Otherwise scroll down for instructions on how t ## Testing -Run the full local test suite (starts a disposable PostgreSQL container automatically): +All tests require a real PostgreSQL server and are organized at the top level: + +- `test_proxy.py`: proxy behavior tests (connection, SSL, hang regressions) +- `test_plugins.py`: plugin integration tests (HLL rewrite behavior) + +### Prerequisites + +- Python `3.13` (same version as CI) +- Docker (for local disposable PostgreSQL) +- `psql` (`postgresql-client`) +- `openssl` (SSL tests generate a temporary self-signed cert/key at runtime) + +Install Python deps in the project virtualenv: + +```bash +make install-test +``` + +### Which command should I use? + +- Fastest full local run with disposable Postgres: `make test` +- Run only proxy tests (using your own Postgres): `python -m pytest tests/test_proxy.py -vv` +- Run only plugin tests: `python -m pytest tests/test_plugins.py -vv` + +#### 1) Full local suite (recommended) + +`make test` starts a temporary PostgreSQL container, waits for readiness, sets DB env vars, then runs: + +```bash +python -m pytest -vv +``` + +Use it when you want one command that matches normal contributor workflow. ```bash make test ``` + +#### 2) Run against an existing PostgreSQL + +If you already have PostgreSQL running, set connection env vars and run the tests you need: + +```bash +export E2E_PG_HOST=127.0.0.1 +export E2E_PG_PORT=5432 +export E2E_PG_USER=postgres +export E2E_PG_PASSWORD=postgres +export E2E_PG_DB=postgres + +# Proxy tests only +python -m pytest tests/test_proxy.py -vv + +# Plugin tests only +python -m pytest tests/test_plugins.py -vv +``` + +If PostgreSQL is not reachable, tests fail fast at startup. diff --git a/postgresql_proxy/__main__.py b/postgresql_proxy/__main__.py new file mode 100644 index 0000000..57562a3 --- /dev/null +++ b/postgresql_proxy/__main__.py @@ -0,0 +1,3 @@ +from postgresql_proxy.proxy import main + +main() diff --git a/postgresql_proxy/proxy.py b/postgresql_proxy/proxy.py index 4ca3ea0..ba8b31e 100644 --- a/postgresql_proxy/proxy.py +++ b/postgresql_proxy/proxy.py @@ -329,7 +329,7 @@ def stop(self): self.running = False -if __name__ == '__main__': +def main(): import importlib import yaml import os @@ -370,3 +370,7 @@ def stop(self): logging.info("Starting proxy instance") proxy = Proxy(instance, plugins) proxy.listen() + + +if __name__ == "__main__": + main() diff --git a/requirements-lint.txt b/requirements-lint.txt new file mode 100644 index 0000000..ace94c5 --- /dev/null +++ b/requirements-lint.txt @@ -0,0 +1 @@ +ruff==0.15.12 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..a0e00db --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==9.0.3 +pytest-timeout==2.4.0 diff --git a/setup.py b/setup.py index 9d5e052..bbf4c72 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,3 @@ -import os -import re -import sys from setuptools import find_packages, setup install_requires = [] @@ -16,14 +13,8 @@ install_requires=install_requires, zip_safe=False, classifiers=[ - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.13', 'License :: OSI Approved :: Apache Software License', 'Topic :: Software Development :: Testing', ] diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index bbeb53c..0000000 --- a/tests/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Testing Guide - -All tests in this repo require a real PostgreSQL server and are organized at the top level: - -- `test_proxy.py`: proxy behavior tests (connection, SSL, hang regressions) -- `test_plugins.py`: plugin integration tests (HLL rewrite behavior) - -## Prerequisites - -- Python `3.13` (same version as CI) -- Docker (for local disposable PostgreSQL) -- `psql` (`postgresql-client`) -- `openssl` (SSL tests generate a temporary self-signed cert/key at runtime) - -Install Python deps in the project virtualenv: - -```bash -make install-test -``` - -## Which command should I use? - -- Fastest full local run with disposable Postgres: `make test` -- Run only proxy tests (using your own Postgres): `python -m pytest tests/test_proxy.py -vv` -- Run only plugin tests: `python -m pytest tests/test_plugins.py -vv` - -## 1) Full local suite (recommended) - -`make test` starts a temporary PostgreSQL container, waits for readiness, sets DB env vars, then runs: - -```bash -python -m pytest -vv -``` - -Use it when you want one command that matches normal contributor workflow. - -```bash -make test -``` - -## 2) DB-backed proxy tests against an existing PostgreSQL - -If you already have PostgreSQL running, set connection env vars and run only proxy tests: - -```bash -export E2E_PG_HOST=127.0.0.1 -export E2E_PG_PORT=5432 -export E2E_PG_USER=postgres -export E2E_PG_PASSWORD=postgres -export E2E_PG_DB=postgres -python -m pytest tests/test_proxy.py -vv -``` - -If PostgreSQL is not reachable, tests fail fast at startup. - -## 3) Plugin integration tests - -```bash -python -m pytest tests/test_plugins.py -vv -``` - -Requires PostgreSQL to be running with the `E2E_PG_*` env vars set (see section 2). diff --git a/tests/conftest.py b/tests/conftest.py index ba7a866..b793c5f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,4 +30,3 @@ def ensure_postgres_available(postgres_settings): pytest.fail( f"PostgreSQL backend is required for tests but is not reachable: {err}" ) - diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 105eb6c..56a2993 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -15,8 +15,6 @@ from postgresql_proxy.proxy import Proxy - - def _get_free_tcp_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) @@ -233,12 +231,11 @@ def test_psql_ssl_file_batch_stress_no_hang(postgres_settings, ssl_proxy_port): if shutil.which("psql") is None: pytest.fail("psql is required for this test but was not found in PATH") - sql_file_path = None - try: + with tempfile.NamedTemporaryFile("w", suffix=".sql", delete=True) as tmp_file: sql_content = _build_dump_like_sql(table_count=24, rows_per_table=300) - with tempfile.NamedTemporaryFile("w", suffix=".sql", delete=False) as tmp_file: - tmp_file.write(sql_content) - sql_file_path = tmp_file.name + tmp_file.write(sql_content) + tmp_file.flush() + sql_file_path = tmp_file.name for run_idx in range(3): started = time.time() @@ -261,7 +258,7 @@ def test_psql_ssl_file_batch_stress_no_hang(postgres_settings, ssl_proxy_port): err_tail = "\n".join((result.stderr or "").splitlines()[-20:]) pytest.fail( "psql -f batch failed over SSL via proxy " - f"(run={run_idx + 1}, rc={result.returncode}, elapsed={elapsed:.2f}s) " + f"(run={run_idx + 1}, rc={result.returncode}, {elapsed=:.2f}s) " f"stdout_tail={out_tail} stderr_tail={err_tail}" ) @@ -269,8 +266,5 @@ def test_psql_ssl_file_batch_stress_no_hang(postgres_settings, ssl_proxy_port): out_tail = "\n".join((result.stdout or "").splitlines()[-20:]) pytest.fail( "psql -f batch succeeded but expected marker missing " - f"(run={run_idx + 1}, elapsed={elapsed:.2f}s) stdout_tail={out_tail}" + f"(run={run_idx + 1}, {elapsed=:.2f}s) stdout_tail={out_tail}" ) - finally: - if sql_file_path and os.path.exists(sql_file_path): - os.unlink(sql_file_path)