From 00a8723347db10d9f314237f37614953fd8913ab Mon Sep 17 00:00:00 2001 From: Jeppe Bundgaard Date: Tue, 19 May 2026 13:17:07 +0200 Subject: [PATCH] Integrate Coolify API client and module for managing Coolify services, enhancing automation and deployment processes. --- .github/workflows/tests.yml | 24 + docker-compose.prod.standalone.yml | 16 +- docker-compose.prod.yml | 2 +- docker-compose.yml | 24 +- nginx-example.conf | 6 +- nginx.conf | 6 +- openapi.yaml | 332 ++ services/edge-broker/test/config.test.mjs | 2 + services/nginx/app/classes/coolify.php | 39 + .../nginx/app/classes/coolify_api_client.php | 215 + .../nginx/app/classes/coolify_manager.php | 2769 ++++++++++++ .../app/classes/coolify_schema_bootstrap.php | 236 + .../classes/error_report_schema_bootstrap.php | 76 + .../app/classes/error_report_service.php | 567 +++ .../nginx/app/classes/error_report_store.php | 47 + services/nginx/app/classes/failover.php | 17 + .../app/classes/hetzner_cloud_client.php | 123 + .../nginx/app/classes/release_manager.php | 3963 +++++++++++++++++ .../release_manager_schema_bootstrap.php | 411 ++ services/nginx/app/classes/releasemanager.php | 19 + .../app/classes/replica_failover_manager.php | 521 +++ .../nginx/app/classes/replication_manager.php | 2910 +++++++++++- services/nginx/app/classes/response.php | 97 +- .../superuser_system_status_service.php | 78 + services/nginx/app/config.php | 11 +- services/nginx/app/cron/Cron.php | 85 + services/nginx/app/index.php | 25 +- .../coolify/config/coolify_enabled_c.php | 25 + .../coolify_hetzner_cloud_api_token_c.php | 38 + .../coolify_hetzner_load_balancer_id_c.php | 25 + .../coolify_lb_automation_enabled_c.php | 25 + .../config/coolify_lb_automation_mode_c.php | 25 + .../config/coolify_public_gateway_host_c.php | 25 + .../nginx/app/modules/coolify/coolify_c.php | 63 + .../classes/edge_gateway_manager.php | 1 + .../edgegateway_public_broker_url_c.php | 4 +- .../config/failover_database_enabled_c.php | 29 + .../failover/config/failover_enabled_c.php | 29 + .../failover_max_status_age_seconds_c.php | 29 + .../config/failover_minio_enabled_c.php | 29 + .../config/failover_redis_enabled_c.php | 29 + .../nginx/app/modules/failover/failover_c.php | 45 + .../app/modules/washcertificates/index.php | 2 +- .../nginx/app/objects/order_bookings_o.php | 4 +- services/nginx/app/openapi.yaml | 1281 ++++++ .../resources/edge-gateway-agent/agent.php | 29 +- services/nginx/app/routes/authRoute.php | 2 + .../nginx/app/routes/errorReportRoute.php | 95 + .../nginx/app/routes/moduleConfigRoute.php | 76 + .../nginx/app/routes/moduleScannerRoute.php | 8 +- .../nginx/app/routes/releaseManagerRoute.php | 377 ++ .../app/routes/superuserCoolifyRoute.php | 258 ++ .../app/routes/superuserReplicationRoute.php | 21 +- .../app/storage/replication-bootstrap.json | 222 + .../app/tests/Api/EdgeGatewayAgentApiTest.php | 5 +- .../tests/Unit/Coolify/CoolifyManagerTest.php | 323 ++ .../Unit/ErrorReports/ErrorReportTest.php | 79 + .../Http/ResponseRequestParametersTest.php | 71 + .../Infrastructure/CorsReleaseHeadersTest.php | 23 + .../OrderBookingsCompletionDedupTest.php | 20 + .../ReleaseManager/ReleaseManagerTest.php | 243 + .../ReplicaFailoverManagerTest.php | 236 + .../ReplicationManagerStatusTest.php | 395 +- .../SuperuserReplicationRouteWiringTest.php | 3 + .../Unit/Scanner/ModuleScannerRouteTest.php | 10 + .../Selfserve/EdgeGatewayRouteWiringTest.php | 5 + services/nginx/nginx.conf | 10 +- services/nginx/nginx.dev.conf | 4 +- services/traefik/dynamic.yml | 3 + 69 files changed, 16632 insertions(+), 215 deletions(-) create mode 100644 services/nginx/app/classes/coolify.php create mode 100644 services/nginx/app/classes/coolify_api_client.php create mode 100644 services/nginx/app/classes/coolify_manager.php create mode 100644 services/nginx/app/classes/coolify_schema_bootstrap.php create mode 100644 services/nginx/app/classes/error_report_schema_bootstrap.php create mode 100644 services/nginx/app/classes/error_report_service.php create mode 100644 services/nginx/app/classes/error_report_store.php create mode 100644 services/nginx/app/classes/failover.php create mode 100644 services/nginx/app/classes/hetzner_cloud_client.php create mode 100644 services/nginx/app/classes/release_manager.php create mode 100644 services/nginx/app/classes/release_manager_schema_bootstrap.php create mode 100644 services/nginx/app/classes/releasemanager.php create mode 100644 services/nginx/app/classes/replica_failover_manager.php create mode 100644 services/nginx/app/modules/coolify/config/coolify_enabled_c.php create mode 100644 services/nginx/app/modules/coolify/config/coolify_hetzner_cloud_api_token_c.php create mode 100644 services/nginx/app/modules/coolify/config/coolify_hetzner_load_balancer_id_c.php create mode 100644 services/nginx/app/modules/coolify/config/coolify_lb_automation_enabled_c.php create mode 100644 services/nginx/app/modules/coolify/config/coolify_lb_automation_mode_c.php create mode 100644 services/nginx/app/modules/coolify/config/coolify_public_gateway_host_c.php create mode 100644 services/nginx/app/modules/coolify/coolify_c.php create mode 100644 services/nginx/app/modules/failover/config/failover_database_enabled_c.php create mode 100644 services/nginx/app/modules/failover/config/failover_enabled_c.php create mode 100644 services/nginx/app/modules/failover/config/failover_max_status_age_seconds_c.php create mode 100644 services/nginx/app/modules/failover/config/failover_minio_enabled_c.php create mode 100644 services/nginx/app/modules/failover/config/failover_redis_enabled_c.php create mode 100644 services/nginx/app/modules/failover/failover_c.php create mode 100644 services/nginx/app/routes/errorReportRoute.php create mode 100644 services/nginx/app/routes/releaseManagerRoute.php create mode 100644 services/nginx/app/routes/superuserCoolifyRoute.php create mode 100644 services/nginx/app/storage/replication-bootstrap.json create mode 100644 services/nginx/app/tests/Unit/Coolify/CoolifyManagerTest.php create mode 100644 services/nginx/app/tests/Unit/ErrorReports/ErrorReportTest.php create mode 100644 services/nginx/app/tests/Unit/Http/ResponseRequestParametersTest.php create mode 100644 services/nginx/app/tests/Unit/Infrastructure/CorsReleaseHeadersTest.php create mode 100644 services/nginx/app/tests/Unit/ReleaseManager/ReleaseManagerTest.php create mode 100644 services/nginx/app/tests/Unit/Replication/ReplicaFailoverManagerTest.php create mode 100644 services/nginx/app/tests/Unit/Scanner/ModuleScannerRouteTest.php diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3dbc83a7..b06abc52 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,6 +57,30 @@ jobs: cache: npm cache-dependency-path: services/edge-agent/package-lock.json + - name: Install native build tools + run: | + set -euo pipefail + if command -v make >/dev/null 2>&1 && command -v g++ >/dev/null 2>&1; then + exit 0 + fi + + if ! command -v apt-get >/dev/null 2>&1; then + echo "make and g++ are required to install node-pty, but apt-get is not available on this runner." >&2 + exit 1 + fi + + apt_cmd=(apt-get) + if [ "$(id -u)" -ne 0 ]; then + if ! command -v sudo >/dev/null 2>&1; then + echo "make and g++ are missing, and sudo is not available to install them." >&2 + exit 1 + fi + apt_cmd=(sudo apt-get) + fi + + "${apt_cmd[@]}" update + "${apt_cmd[@]}" install -y --no-install-recommends build-essential python3 + - name: Install dependencies working-directory: services/edge-agent run: npm ci diff --git a/docker-compose.prod.standalone.yml b/docker-compose.prod.standalone.yml index e4c40811..33a89c6f 100644 --- a/docker-compose.prod.standalone.yml +++ b/docker-compose.prod.standalone.yml @@ -96,6 +96,13 @@ services: - "traefik.http.routers.edge-broker-api-io.middlewares=secure-headers@file,edge-broker-strip" - "traefik.http.routers.edge-broker-api-io.priority=200" - "traefik.http.routers.edge-broker-api-io.service=edge-broker" + - "traefik.http.routers.edge-broker-api-v2.rule=Host(`api-v2.truckwash.io`) && PathPrefix(`/edge-broker`)" + - "traefik.http.routers.edge-broker-api-v2.entrypoints=websecure" + - "traefik.http.routers.edge-broker-api-v2.tls=true" + - "traefik.http.routers.edge-broker-api-v2.tls.certresolver=le_io" + - "traefik.http.routers.edge-broker-api-v2.middlewares=secure-headers@file,edge-broker-strip" + - "traefik.http.routers.edge-broker-api-v2.priority=200" + - "traefik.http.routers.edge-broker-api-v2.service=edge-broker" - "traefik.http.routers.edge-broker-api-staging.rule=Host(`api.truckwash.io`) && PathPrefix(`/edge-broker`)" - "traefik.http.routers.edge-broker-api-staging.entrypoints=websecure-staging" - "traefik.http.routers.edge-broker-api-staging.tls=true" @@ -153,7 +160,14 @@ services: - "traefik.http.routers.api-io.tls.certresolver=le_io" - "traefik.http.routers.api-io.service=caddy" - "traefik.http.routers.api-io.middlewares=secure-headers@file,api-ratelimit@file" - - "traefik.http.routers.api-http.rule=Host(`api.truckwash.dk`) || Host(`api.truckwash.io`)" + - "traefik.http.routers.api-v2.rule=Host(`api-v2.truckwash.io`)" + - "traefik.http.routers.api-v2.entrypoints=websecure" + - "traefik.http.routers.api-v2.tls=true" + - "traefik.http.routers.api-v2.tls.domains[0].main=api-v2.truckwash.io" + - "traefik.http.routers.api-v2.tls.certresolver=le_io" + - "traefik.http.routers.api-v2.service=caddy" + - "traefik.http.routers.api-v2.middlewares=secure-headers@file,api-ratelimit@file" + - "traefik.http.routers.api-http.rule=Host(`api.truckwash.dk`) || Host(`api.truckwash.io`) || Host(`api-v2.truckwash.io`)" - "traefik.http.routers.api-http.entrypoints=web" - "traefik.http.routers.api-http.middlewares=redirect-to-https@file" - "traefik.http.routers.api-http.service=caddy" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index aafc0bd2..e7d257d4 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -27,5 +27,5 @@ services: ## docker compose up -d traefik caddy php1 php2 php3 php4 php5 db redis ## ## Notes: -## - Traefik uses Let’s Encrypt production. Ensure DNS A/AAAA records for api.truckwash.dk, api.truckwash.io and traefik.truckwash.dk point to this host and ports 80/443 are reachable. +## - Traefik uses Let’s Encrypt production. Ensure DNS A/AAAA records for api.truckwash.dk, api.truckwash.io, api-v2.truckwash.io and traefik.truckwash.dk point to the expected ingress and ports 80/443 are reachable. ## - The dashboard is protected by basic auth and an IP allowlist (defined in dynamic.yml). Replace the bcrypt hash before enabling in production. diff --git a/docker-compose.yml b/docker-compose.yml index d503bf2e..1c91555b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,13 @@ services: - "traefik.http.routers.edge-broker-io.middlewares=secure-headers@file,edge-broker-strip" - "traefik.http.routers.edge-broker-io.priority=200" - "traefik.http.routers.edge-broker-io.service=edge-broker" + - "traefik.http.routers.edge-broker-v2.rule=Host(`api-v2.truckwash.io`) && PathPrefix(`/edge-broker`)" + - "traefik.http.routers.edge-broker-v2.entrypoints=websecure" + - "traefik.http.routers.edge-broker-v2.tls=true" + - "traefik.http.routers.edge-broker-v2.tls.certresolver=le_io" + - "traefik.http.routers.edge-broker-v2.middlewares=secure-headers@file,edge-broker-strip" + - "traefik.http.routers.edge-broker-v2.priority=200" + - "traefik.http.routers.edge-broker-v2.service=edge-broker" - "traefik.http.routers.edge-broker-staging.rule=Host(`api.truckwash.io`) && PathPrefix(`/edge-broker`)" - "traefik.http.routers.edge-broker-staging.entrypoints=websecure-staging" - "traefik.http.routers.edge-broker-staging.tls=true" @@ -133,6 +140,13 @@ services: - "traefik.http.routers.edge-broker-api-io.middlewares=secure-headers@file,edge-broker-strip" - "traefik.http.routers.edge-broker-api-io.priority=200" - "traefik.http.routers.edge-broker-api-io.service=edge-broker" + - "traefik.http.routers.edge-broker-api-v2.rule=Host(`api-v2.truckwash.io`) && PathPrefix(`/edge-broker`)" + - "traefik.http.routers.edge-broker-api-v2.entrypoints=websecure" + - "traefik.http.routers.edge-broker-api-v2.tls=true" + - "traefik.http.routers.edge-broker-api-v2.tls.certresolver=le_io" + - "traefik.http.routers.edge-broker-api-v2.middlewares=secure-headers@file,edge-broker-strip" + - "traefik.http.routers.edge-broker-api-v2.priority=200" + - "traefik.http.routers.edge-broker-api-v2.service=edge-broker" - "traefik.http.routers.edge-broker-api-staging.rule=Host(`api.truckwash.io`) && PathPrefix(`/edge-broker`)" - "traefik.http.routers.edge-broker-api-staging.entrypoints=websecure-staging" - "traefik.http.routers.edge-broker-api-staging.tls=true" @@ -192,8 +206,16 @@ services: - "traefik.http.routers.api-io.tls.certresolver=le_io" - "traefik.http.routers.api-io.service=caddy" - "traefik.http.routers.api-io.middlewares=secure-headers@file,api-ratelimit@file" + # Public API (.io load-balanced gateway) + - "traefik.http.routers.api-v2.rule=Host(`api-v2.truckwash.io`)" + - "traefik.http.routers.api-v2.entrypoints=websecure" + - "traefik.http.routers.api-v2.tls=true" + - "traefik.http.routers.api-v2.tls.domains[0].main=api-v2.truckwash.io" + - "traefik.http.routers.api-v2.tls.certresolver=le_io" + - "traefik.http.routers.api-v2.service=caddy" + - "traefik.http.routers.api-v2.middlewares=secure-headers@file,api-ratelimit@file" # HTTP to HTTPS redirect for both API domains - - "traefik.http.routers.api-http.rule=Host(`api.truckwash.dk`) || Host(`api.truckwash.io`)" + - "traefik.http.routers.api-http.rule=Host(`api.truckwash.dk`) || Host(`api.truckwash.io`) || Host(`api-v2.truckwash.io`)" - "traefik.http.routers.api-http.entrypoints=web" - "traefik.http.routers.api-http.middlewares=redirect-to-https@file" - "traefik.http.routers.api-http.service=caddy" diff --git a/nginx-example.conf b/nginx-example.conf index 6707e45e..e0ae4061 100644 --- a/nginx-example.conf +++ b/nginx-example.conf @@ -38,14 +38,14 @@ http { location / { add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; add_header Access-Control-Allow-Credentials true; # If OPTIONS method is needed for preflight if ($request_method = 'OPTIONS') { add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; return 204; # No Content } @@ -65,4 +65,4 @@ http { location ~* \.(cgi|shtml|phtml)$ { } } -} \ No newline at end of file +} diff --git a/nginx.conf b/nginx.conf index 1904efbc..ce4bf628 100644 --- a/nginx.conf +++ b/nginx.conf @@ -52,14 +52,14 @@ http { location / { add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; add_header Access-Control-Allow-Credentials true; # If OPTIONS method is needed for preflight if ($request_method = 'OPTIONS') { add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; return 204; # No Content } @@ -80,4 +80,4 @@ http { # Additional SSL options or configurations can be placed here, if necessary. } } -} \ No newline at end of file +} diff --git a/openapi.yaml b/openapi.yaml index 21f601e3..95ad5d13 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -72,6 +72,8 @@ tags: description: Form submissions and management - name: Worker description: System worker status and maintenance + - name: Error Reports + description: Authenticated application error reporting - name: Plate Scans description: License plate scanning operations - name: Config @@ -90,6 +92,160 @@ tags: description: Voice Calls via Bird paths: + /error-reports: + post: + tags: + - Error Reports + summary: Submit an authenticated user error report + operationId: submitErrorReport + security: + - BearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportSubmissionRequest' + responses: + '201': + description: Error report submitted + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportResponse' + '400': + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Authentication required + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /superuser/error-reports: + get: + tags: + - Error Reports + summary: List error reports for superusers + operationId: listSuperuserErrorReports + security: + - BearerAuth: [] + parameters: + - name: status + in: query + required: false + schema: + type: string + enum: [open, resolved, all] + default: open + - name: q + in: query + required: false + schema: + type: string + - name: limit + in: query + required: false + schema: + type: integer + minimum: 1 + maximum: 200 + default: 50 + - name: offset + in: query + required: false + schema: + type: integer + minimum: 0 + default: 0 + responses: + '200': + description: Error reports retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportListResponse' + '403': + description: Missing superuser error report permission + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /superuser/error-reports/{id}: + get: + tags: + - Error Reports + summary: Get an error report detail + operationId: getSuperuserErrorReport + security: + - BearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: integer + minimum: 1 + responses: + '200': + description: Error report retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportResponse' + '404': + description: Error report not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /superuser/error-reports/{id}/status: + patch: + tags: + - Error Reports + summary: Mark an error report open or resolved + operationId: updateSuperuserErrorReportStatus + security: + - BearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: integer + minimum: 1 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportStatusUpdateRequest' + responses: + '200': + description: Error report status updated + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportResponse' + '400': + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Missing superuser error report resolve permission + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + # Bird Voice Calls /bird/voice/calls: post: @@ -11939,6 +12095,182 @@ components: type: integer description: HTTP status code + ErrorReportSubmissionRequest: + type: object + required: + - before_error + - expected + - actual + - data_collection_accepted + - screenshot + properties: + before_error: + type: string + maxLength: 4000 + description: What the user was doing before the error occurred + expected: + type: string + maxLength: 4000 + description: What the user expected would happen + actual: + type: string + maxLength: 4000 + description: What actually happened + data_collection_accepted: + type: boolean + description: Required acceptance of collecting screenshot and diagnostic error data + screenshot: + type: string + description: PNG, JPEG, or WebP data URI of the current app viewport + route_path: + type: string + nullable: true + page_url: + type: string + nullable: true + release_trace_id: + type: string + nullable: true + request_errors: + type: array + items: + type: object + additionalProperties: true + vue_errors: + type: array + items: + type: object + additionalProperties: true + context: + type: object + additionalProperties: true + + ErrorReportStatusUpdateRequest: + type: object + required: + - status + properties: + status: + type: string + enum: [open, resolved] + resolution_note: + type: string + nullable: true + maxLength: 2000 + + ErrorReportResponse: + type: object + properties: + success: + type: boolean + data: + $ref: '#/components/schemas/ErrorReport' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + ErrorReportListResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + items: + type: array + items: + $ref: '#/components/schemas/ErrorReport' + counts: + type: object + properties: + open: + type: integer + resolved: + type: integer + all: + type: integer + limit: + type: integer + offset: + type: integer + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + ErrorReport: + type: object + properties: + id: + type: integer + status: + type: string + enum: [open, resolved] + reporter: + type: object + additionalProperties: true + route_path: + type: string + nullable: true + page_url: + type: string + nullable: true + release_trace_id: + type: string + nullable: true + frontend_version: + type: string + nullable: true + api_version: + type: string + nullable: true + screenshot: + type: object + additionalProperties: true + answers: + type: object + properties: + before_error: + type: string + expected: + type: string + actual: + type: string + request_error_count: + type: integer + vue_error_count: + type: integer + request_errors: + type: array + items: + type: object + additionalProperties: true + vue_errors: + type: array + items: + type: object + additionalProperties: true + runtime_context: + type: object + additionalProperties: true + resolved_at: + type: string + nullable: true + resolved_by_user_id: + type: integer + nullable: true + created_at: + type: string + updated_at: + type: string + nullable: true + SuperuserSystemStatusResponse: type: object properties: diff --git a/services/edge-broker/test/config.test.mjs b/services/edge-broker/test/config.test.mjs index d238aebb..dfb64706 100644 --- a/services/edge-broker/test/config.test.mjs +++ b/services/edge-broker/test/config.test.mjs @@ -45,6 +45,7 @@ test("base docker compose routes edge broker traffic through traefik", () => { assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-local\.priority=200/); assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-api\.rule=Host\(`api\.truckwash\.dk`\) && PathPrefix\(`\/edge-broker`\)/); assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-api-io\.rule=Host\(`api\.truckwash\.io`\) && PathPrefix\(`\/edge-broker`\)/); + assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-api-v2\.rule=Host\(`api-v2\.truckwash\.io`\) && PathPrefix\(`\/edge-broker`\)/); assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-local\.rule=Host\(`localhost`\) && PathPrefix\(`\/api\/edge-broker`\)/); assert.match(serviceBlock, /traefik\.http\.middlewares\.edge-broker-strip\.stripPrefix\.prefixes=\/edge-broker/); assert.match(serviceBlock, /traefik\.http\.middlewares\.edge-broker-strip-local\.stripPrefix\.prefixes=\/api\/edge-broker/); @@ -70,6 +71,7 @@ test("standalone production compose routes edge broker traffic through traefik", assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-local\.priority=200/); assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-api\.rule=Host\(`api\.truckwash\.dk`\) && PathPrefix\(`\/edge-broker`\)/); assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-api-io\.rule=Host\(`api\.truckwash\.io`\) && PathPrefix\(`\/edge-broker`\)/); + assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-api-v2\.rule=Host\(`api-v2\.truckwash\.io`\) && PathPrefix\(`\/edge-broker`\)/); assert.match(serviceBlock, /traefik\.http\.routers\.edge-broker-local\.rule=Host\(`localhost`\) && PathPrefix\(`\/api\/edge-broker`\)/); assert.match(serviceBlock, /traefik\.http\.services\.edge-broker\.loadbalancer\.server\.port=4300/); }); diff --git a/services/nginx/app/classes/coolify.php b/services/nginx/app/classes/coolify.php new file mode 100644 index 00000000..c8840b40 --- /dev/null +++ b/services/nginx/app/classes/coolify.php @@ -0,0 +1,39 @@ +config = new coolify_c(); + } + + /** + * @throws Exception + */ + public function requireModuleEnabled(): void + { + if (!$this->config->enabled->isTrue()) { + throw new Exception('The Coolify module is not enabled'); + } + } + + public function isEnabled(): bool + { + try { + return $this->config->enabled->isTrue(); + } catch (Exception) { + return false; + } + } +} diff --git a/services/nginx/app/classes/coolify_api_client.php b/services/nginx/app/classes/coolify_api_client.php new file mode 100644 index 00000000..47388b35 --- /dev/null +++ b/services/nginx/app/classes/coolify_api_client.php @@ -0,0 +1,215 @@ +baseUrl = self::normalizeBaseUrl($baseUrl); + $this->token = trim($token); + $this->timeoutSeconds = max(1, $timeoutSeconds); + if ($this->baseUrl === '' || $this->token === '') { + throw new RuntimeException('Coolify base URL and API token are required.'); + } + } + + public static function normalizeBaseUrl(string $baseUrl): string + { + $baseUrl = rtrim(trim($baseUrl), '/'); + if ($baseUrl === '') { + return ''; + } + + if (preg_match('#/api/v[0-9]+$#i', $baseUrl) === 1) { + return $baseUrl; + } + + return $baseUrl . '/api/v1'; + } + + public function healthcheck(): array + { + return $this->request('GET', '/health', null, false); + } + + public function version(): array + { + return $this->request('GET', '/version'); + } + + public function listServers(): array + { + return $this->request('GET', '/servers'); + } + + public function listProjects(): array + { + return $this->request('GET', '/projects'); + } + + public function listProjectEnvironments(string $projectUuid): array + { + return $this->request('GET', '/projects/' . rawurlencode($projectUuid) . '/environments'); + } + + public function listServices(): array + { + return $this->request('GET', '/services'); + } + + public function getService(string $uuid): array + { + return $this->request('GET', '/services/' . rawurlencode($uuid)); + } + + public function createService(array $payload): array + { + return $this->request('POST', '/services', $payload); + } + + public function updateService(string $uuid, array $payload): array + { + return $this->request('PATCH', '/services/' . rawurlencode($uuid), $payload); + } + + public function updateServiceEnvsBulk(string $uuid, array $env): array + { + $data = []; + foreach ($env as $key => $value) { + $data[] = [ + 'key' => (string)$key, + 'value' => (string)$value, + 'is_preview' => false, + 'is_literal' => true, + 'is_multiline' => str_contains((string)$value, "\n"), + 'is_shown_once' => false, + ]; + } + + if ($data === []) { + return []; + } + + return $this->request('PATCH', '/services/' . rawurlencode($uuid) . '/envs/bulk', ['data' => $data]); + } + + public function startService(string $uuid): array + { + return $this->request('GET', '/services/' . rawurlencode($uuid) . '/start'); + } + + public function restartService(string $uuid): array + { + return $this->request('GET', '/services/' . rawurlencode($uuid) . '/restart'); + } + + public function deleteService(string $uuid): array + { + return $this->request('DELETE', '/services/' . rawurlencode($uuid)); + } + + public function listDeployments(): array + { + return $this->request('GET', '/deployments'); + } + + protected function request(string $method, string $path, ?array $payload = null, bool $versionedApi = true): array + { + $url = ($versionedApi ? $this->baseUrl : $this->apiRootUrl()) . '/' . ltrim($path, '/'); + $curl = curl_init($url); + if ($curl === false) { + throw new RuntimeException('Could not initialize Coolify API request.'); + } + + $headers = [ + 'Accept: application/json', + 'Authorization: Bearer ' . $this->token, + ]; + + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_CUSTOMREQUEST, strtoupper($method)); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, min(2, $this->timeoutSeconds)); + curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeoutSeconds); + curl_setopt($curl, CURLOPT_NOSIGNAL, true); + curl_setopt($curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + + if ($payload !== null) { + $body = json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($body === false) { + throw new RuntimeException('Could not encode Coolify API payload.'); + } + $headers[] = 'Content-Type: application/json'; + curl_setopt($curl, CURLOPT_POSTFIELDS, $body); + } + + curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); + + $raw = curl_exec($curl); + $error = curl_error($curl); + $status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE); + curl_close($curl); + + if ($raw === false) { + throw new RuntimeException('Coolify API request failed: ' . $error); + } + + $decoded = null; + if (trim((string)$raw) !== '') { + $decoded = json_decode((string)$raw, true); + if (!is_array($decoded)) { + $decoded = ['raw' => (string)$raw]; + } + } + + if ($status < 200 || $status >= 300) { + $message = is_array($decoded) + ? (string)($decoded['message'] ?? $decoded['error'] ?? ('HTTP ' . $status)) + : ('HTTP ' . $status); + if (is_array($decoded)) { + $details = self::validationErrorSummary($decoded); + if ($details !== '') { + $message .= ': ' . $details; + } + } + throw new RuntimeException('Coolify API request failed: ' . $message); + } + + return is_array($decoded) ? $decoded : []; + } + + private static function validationErrorSummary(array $decoded): string + { + $errors = $decoded['errors'] ?? $decoded['data']['errors'] ?? null; + if (!is_array($errors)) { + return ''; + } + + $parts = []; + foreach ($errors as $field => $messages) { + $fieldName = trim((string)$field); + $fieldPrefix = $fieldName !== '' ? $fieldName . ': ' : ''; + if (is_array($messages)) { + $messages = implode(', ', array_filter(array_map(static fn(mixed $message): string => trim((string)$message), $messages))); + } else { + $messages = trim((string)$messages); + } + if ($messages !== '') { + $parts[] = $fieldPrefix . $messages; + } + } + + return implode('; ', array_slice($parts, 0, 5)); + } + + private function apiRootUrl(): string + { + return preg_replace('#/v[0-9]+$#i', '', $this->baseUrl) ?: $this->baseUrl; + } +} diff --git a/services/nginx/app/classes/coolify_manager.php b/services/nginx/app/classes/coolify_manager.php new file mode 100644 index 00000000..7a1e3ed7 --- /dev/null +++ b/services/nginx/app/classes/coolify_manager.php @@ -0,0 +1,2769 @@ + 'http', 'listen_port' => 80, 'destination_port' => 80], + ['protocol' => 'tcp', 'listen_port' => 443, 'destination_port' => 443], + ]; + + /** @var callable|null */ + private $clientFactory; + /** @var callable|null */ + private $hetznerClientFactory; + private bool $schemaEnsured = false; + + public function __construct(?callable $clientFactory = null, ?callable $hetznerClientFactory = null) + { + $this->clientFactory = $clientFactory; + $this->hetznerClientFactory = $hetznerClientFactory; + } + + public function summary(): array + { + $this->ensureSchema(); + + return [ + 'generated_at' => date('c'), + 'instances' => $this->listInstances(), + 'targets' => $this->listTargets(), + 'availability' => $this->availabilitySummary(), + 'load_balancer' => $this->loadBalancerSummary(), + ]; + } + + public function listInstances(): array + { + if (!coolify_schema_bootstrap::tablesExist()) { + return []; + } + + return array_map( + fn(array $instance): array => $this->publicInstance($instance), + $this->selectRows('SELECT * FROM coolify_instances WHERE deleted_at IS NULL ORDER BY id') + ); + } + + public function createInstance(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $label = trim((string)($input['label'] ?? 'Coolify')); + $baseUrl = trim((string)($input['base_url'] ?? $input['url'] ?? '')); + $apiToken = (string)($input['api_token'] ?? $input['token'] ?? ''); + if ($label === '' || $baseUrl === '' || trim($apiToken) === '') { + throw new RuntimeException('Coolify label, base URL, and API token are required.'); + } + + $this->execute( + "INSERT INTO coolify_instances ( + label, base_url, api_token_secret, default_project_uuid, default_environment_uuid, + default_environment_name, default_server_uuid, default_destination_uuid + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + 'ssssssss', + [ + $label, + rtrim($baseUrl, '/'), + replication_secret_box::encrypt($apiToken), + null, + null, + null, + null, + null, + ] + ); + + $id = $this->insertId(); + $this->setModuleEnabled(true); + $this->audit(null, $id, null, 'instance_created', $actorUserId, 'info', [ + 'label' => $label, + 'base_url' => $baseUrl, + ]); + + return $this->publicInstance($this->getInstance($id)); + } + + public function testInstance(int $instanceId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $instance = $this->getInstance($instanceId); + $startedAt = microtime(true); + + try { + $client = $this->clientForInstance($instance); + $health = $client->healthcheck(); + $version = []; + try { + $version = $client->version(); + } catch (Throwable) { + } + + $result = [ + 'ok' => true, + 'status' => 'ok', + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + 'health' => $health, + 'version' => $version, + 'checked_at' => date('c'), + ]; + + $this->execute( + "UPDATE coolify_instances SET status = 'ok', last_checked_at = NOW(), last_error = NULL WHERE id = ?", + 'i', + [$instanceId] + ); + $this->audit(null, $instanceId, null, 'instance_tested', $actorUserId, 'info', $result); + + return [ + 'instance' => $this->publicInstance($this->getInstance($instanceId)), + 'test' => $result, + ]; + } catch (Throwable $throwable) { + $result = [ + 'ok' => false, + 'status' => 'down', + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + 'error' => $throwable->getMessage(), + 'checked_at' => date('c'), + ]; + $this->execute( + "UPDATE coolify_instances SET status = 'down', last_checked_at = NOW(), last_error = ? WHERE id = ?", + 'si', + [$throwable->getMessage(), $instanceId] + ); + $this->audit(null, $instanceId, null, 'instance_test_failed', $actorUserId, 'warning', $result); + + return [ + 'instance' => $this->publicInstance($this->getInstance($instanceId)), + 'test' => $result, + ]; + } + } + + public function discoverInstancePlacement(int $instanceId): array + { + $this->ensureSchema(); + $instance = $this->getInstance($instanceId); + $client = $this->clientForInstance($instance); + $errors = []; + + $servers = []; + try { + $servers = array_map( + fn(array $server): array => $this->publicPlacementServer($server), + $this->coolifyCollection($client->listServers()) + ); + } catch (Throwable $throwable) { + $errors['servers'] = $throwable->getMessage(); + } + + $projects = []; + $environments = []; + try { + $projects = array_map( + fn(array $project): array => $this->publicPlacementProject($project), + $this->coolifyCollection($client->listProjects()) + ); + + foreach ($projects as $project) { + $projectUuid = (string)($project['uuid'] ?? ''); + if ($projectUuid === '') { + continue; + } + + try { + foreach ($this->coolifyCollection($client->listProjectEnvironments($projectUuid)) as $environment) { + $environments[] = $this->publicPlacementEnvironment($environment, $project); + } + } catch (Throwable $throwable) { + $errors['environments'][$projectUuid] = $throwable->getMessage(); + } + } + } catch (Throwable $throwable) { + $errors['projects'] = $throwable->getMessage(); + } + + return [ + 'generated_at' => date('c'), + 'instance' => $this->publicInstance($instance), + 'servers' => array_values(array_filter($servers, static fn(array $server): bool => (string)($server['uuid'] ?? '') !== '')), + 'projects' => array_values(array_filter($projects, static fn(array $project): bool => (string)($project['uuid'] ?? '') !== '')), + 'environments' => array_values(array_filter($environments, static fn(array $environment): bool => (string)($environment['name'] ?? $environment['uuid'] ?? '') !== '')), + 'destination_discovery_supported' => false, + 'errors' => $errors, + ]; + } + + public function listTargets(?string $kind = null): array + { + if (!coolify_schema_bootstrap::tablesExist()) { + return []; + } + + $types = ''; + $params = []; + $where = ['t.deleted_at IS NULL']; + if ($kind !== null && trim($kind) !== '') { + $where[] = 't.kind = ?'; + $types .= 's'; + $params[] = replication_manager::normalizeKind($kind); + } + + $targets = $this->selectRows( + "SELECT t.*, i.label AS instance_label, i.base_url AS instance_base_url, h.label AS replication_label, + h.host AS replication_host, h.port AS replication_port, h.role AS replication_role, + h.status AS replication_status, h.last_status_json AS replication_last_status_json, + h.last_checked_at AS replication_last_checked_at + FROM coolify_targets t + INNER JOIN coolify_instances i ON i.id = t.instance_id + LEFT JOIN replication_hosts h ON h.id = t.replication_host_id + WHERE " . implode(' AND ', $where) . ' + ORDER BY FIELD(t.kind, \'database\', \'redis\', \'minio\'), t.id', + $types, + $params + ); + + return array_map(fn(array $target): array => $this->publicTarget($target), $targets); + } + + public function createTarget(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $kind = replication_manager::normalizeKind((string)($input['kind'] ?? '')); + $role = strtolower(trim((string)($input['role'] ?? 'replica'))); + if ($role !== 'replica') { + throw new RuntimeException('Coolify-managed targets must be deployed as replicas first to avoid planned downtime.'); + } + + $instanceId = (int)($input['instance_id'] ?? 0); + if ($instanceId <= 0) { + $instanceId = $this->defaultInstanceId(); + } + $instance = $this->getInstance($instanceId); + $input = $this->applyCoolifyDeploymentDefaults($input, $instance); + $input = $this->applyCoolifyPortDefaults($kind, $input, $instance); + + $composeInput = $this->composeInputFromRequest($kind, $input, $instance); + $template = replication_manager::composeTemplate($composeInput); + $hostPayload = $this->hostPayloadFromTemplate($kind, $input, $template); + $hostPayload['options'] = array_replace( + is_array($hostPayload['options'] ?? null) ? $hostPayload['options'] : [], + [ + 'deployment_provider' => 'coolify', + 'coolify_instance_id' => $instanceId, + ] + ); + + $replicationHost = (new replication_manager())->addHost($kind, $hostPayload, $actorUserId); + $replicationHostId = (int)$replicationHost['id']; + $label = trim((string)($input['label'] ?? $replicationHost['label'] ?? $template['service_name'] ?? 'Coolify target')); + $resourceName = self::resourceName($kind, (string)($template['service_name'] ?? $label), $replicationHostId); + $targetOptions = $this->targetOptions($input, $template, $composeInput); + + $this->execute( + "INSERT INTO coolify_targets ( + instance_id, replication_host_id, kind, label, role, server_uuid, project_uuid, + environment_uuid, environment_name, destination_uuid, resource_name, deployment_status, + availability_state, desired_compose_hash, options_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 'degraded', ?, ?)", + 'iisssssssssss', + [ + $instanceId, + $replicationHostId, + $kind, + $label, + $role, + $this->targetMapping($input, $instance, 'server_uuid'), + $this->targetMapping($input, $instance, 'project_uuid'), + $this->targetMapping($input, $instance, 'environment_uuid'), + $this->targetMapping($input, $instance, 'environment_name') ?: 'production', + $this->targetMapping($input, $instance, 'destination_uuid'), + $resourceName, + $this->composeHash($template), + self::jsonEncode($targetOptions), + ] + ); + + $targetId = $this->insertId(); + $this->attachTargetToReplicationHost($kind, $replicationHostId, $targetId, $instanceId); + $this->ensureFailoverEnabled($kind); + $this->audit($targetId, $instanceId, $replicationHostId, 'target_created', $actorUserId, 'info', [ + 'kind' => $kind, + 'role' => $role, + 'resource_name' => $resourceName, + ]); + + $target = $this->getTarget($targetId); + $deploy = $this->toBool($input['deploy'] ?? false, false); + if ($deploy) { + try { + $this->deployTarget($targetId, $actorUserId); + } catch (Throwable $throwable) { + $this->markTargetFailure($targetId, 'reconcile_failed', $throwable->getMessage(), [ + 'stage' => 'create_target_deploy', + ]); + } + } + + return [ + 'target' => $this->publicTarget($this->getTarget($targetId)), + 'host' => $replicationHost, + ]; + } + + public function reconcileTarget(int $targetId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $target = $this->getTarget($targetId); + $host = $this->replicationHost((int)$target['replication_host_id']); + if (self::blocksPrimaryMutation($host, 'deploy')) { + return $this->blockedTargetOperation($target, $host, 'deploy', $actorUserId); + } + + $operationId = $this->startOperation($targetId, (int)$target['instance_id'], 'reconcile', $actorUserId); + + try { + $instance = $this->getInstance((int)$target['instance_id']); + $client = $this->clientForInstance($instance); + $host = $this->syncReplicationHostPortsForTarget($target, $host); + $host = $this->syncReplicationHostEndpointForTarget($target, $host, $instance); + $template = $this->composeTemplateForTarget($target, $host); + $env = self::parseEnvFile((string)($template['env'] ?? '')); + $hash = $this->composeHash($template); + $payload = $this->servicePayload($target, $template, false); + $resourceUuid = trim((string)($target['resource_uuid'] ?? '')); + $action = 'in_sync'; + $apiResult = []; + $shouldStart = in_array((string)($target['deployment_status'] ?? ''), ['pending', 'reconcile_failed', 'created', 'deploying', 'provision_blocked'], true); + + if ($resourceUuid === '') { + $apiResult = $client->createService($payload); + $resourceUuid = (string)($apiResult['uuid'] ?? ''); + if ($resourceUuid === '') { + throw new RuntimeException('Coolify did not return a service UUID.'); + } + $this->recordCreatedResource($targetId, $resourceUuid, $hash); + $action = 'created'; + $shouldStart = true; + } elseif ($hash !== (string)($target['desired_compose_hash'] ?? '')) { + $apiResult = $client->updateService($resourceUuid, $this->servicePayload($target, $template, true)); + $action = 'updated'; + $shouldStart = true; + } else { + try { + $apiResult = $client->getService($resourceUuid); + } catch (Throwable) { + $apiResult = []; + } + $action = $shouldStart ? 'start_requested' : 'in_sync'; + } + + if ($env !== []) { + $client->updateServiceEnvsBulk($resourceUuid, $env); + } + $startResult = null; + if ($shouldStart) { + $startResult = $this->startOrRestartService($client, $resourceUuid, $action === 'updated'); + } + + $context = [ + 'action' => $action, + 'resource_uuid' => $resourceUuid, + 'compose_hash' => $hash, + 'coolify' => self::redactCoolifyResponse($apiResult), + 'start' => self::redactCoolifyResponse(is_array($startResult) ? $startResult : []), + ]; + $availabilityState = $this->availabilityStateForHost($host); + $this->execute( + "UPDATE coolify_targets + SET resource_uuid = ?, deployment_status = ?, availability_state = ?, desired_compose_hash = ?, + last_reconcile_status = ?, last_reconcile_json = ?, last_reconciled_at = NOW() + WHERE id = ?", + 'ssssssi', + [ + $resourceUuid, + $action === 'in_sync' ? 'in_sync' : 'deploying', + $availabilityState, + $hash, + $action, + self::jsonEncode($context), + $targetId, + ] + ); + $this->finishOperation($operationId, 'completed', 'Coolify reconcile completed.', []); + $this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_reconciled', $actorUserId, 'info', $context); + + return [ + 'ok' => true, + 'status' => $action, + 'target' => $this->publicTarget($this->getTarget($targetId)), + 'context' => $context, + ]; + } catch (Throwable $throwable) { + $this->finishOperation($operationId, 'failed', null, [$throwable->getMessage()]); + $this->markTargetFailure($targetId, 'reconcile_failed', $throwable->getMessage()); + $this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_reconcile_failed', $actorUserId, 'error', [ + 'error' => $throwable->getMessage(), + ]); + throw $throwable; + } + } + + public function deployTarget(int $targetId, ?int $actorUserId = null): array + { + $reconcile = $this->reconcileTarget($targetId, $actorUserId); + $target = $this->getTarget($targetId); + $hostId = (int)($target['replication_host_id'] ?? 0); + $provision = null; + + if ($hostId > 0) { + $reconcileAction = (string)($reconcile['status'] ?? $reconcile['context']['action'] ?? ''); + if (in_array($reconcileAction, ['created', 'updated'], true)) { + $provision = (string)($target['kind'] ?? '') === 'minio' + ? (new replication_manager())->provisionHost((string)$target['kind'], $hostId, $actorUserId, true) + : $this->deferredProvisionResult(null); + $this->setTargetProvisionState($targetId, $hostId, 'deploying', 'provision_deferred'); + } else { + $provision = $this->attemptTargetProvision($targetId, $target, $hostId, $actorUserId, true); + } + } + + return [ + 'ok' => ($provision['ok'] ?? true) !== false, + 'reconcile' => $reconcile, + 'provision' => $provision, + 'target' => $this->publicTarget($this->getTarget($targetId)), + ]; + } + + private function attemptTargetProvision( + int $targetId, + array $target, + int $hostId, + ?int $actorUserId, + bool $deferLongRunning = false + ): array + { + try { + $provision = (new replication_manager())->provisionHost( + (string)$target['kind'], + $hostId, + $actorUserId, + $deferLongRunning + ); + if (($provision['ok'] ?? false) === false && $this->isTransientProvisionBlock($provision)) { + $this->setTargetProvisionState($targetId, $hostId, 'deploying', 'provision_deferred'); + return $this->deferredProvisionResult($provision); + } + + $completed = (($provision['ok'] ?? false) === true) + && (($provision['operation']['status'] ?? null) !== 'running'); + $this->setTargetProvisionState( + $targetId, + $hostId, + $completed ? 'provisioned' : ((($provision['ok'] ?? false) === true) ? 'deploying' : 'provision_blocked'), + $completed ? 'provisioned' : ((($provision['ok'] ?? false) === true) ? 'provisioning' : 'provision_blocked') + ); + + return $provision; + } catch (Throwable $throwable) { + $this->markTargetFailure($targetId, 'provision_blocked', $throwable->getMessage()); + return [ + 'ok' => false, + 'message' => $throwable->getMessage(), + 'blockers' => [$throwable->getMessage()], + ]; + } + } + + private function setTargetProvisionState(int $targetId, int $hostId, string $deploymentStatus, string $lastReconcileStatus): void + { + $this->execute( + "UPDATE coolify_targets + SET availability_state = ?, deployment_status = ?, last_reconcile_status = ?, last_reconciled_at = NOW() + WHERE id = ?", + 'sssi', + [ + $this->availabilityStateForHost($this->replicationHost($hostId, true)), + $deploymentStatus, + $lastReconcileStatus, + $targetId, + ] + ); + } + + private function deferredProvisionResult(?array $provision): array + { + $blockers = array_values(array_unique(array_filter(array_map( + static fn(mixed $blocker): string => trim((string)$blocker), + is_array($provision['blockers'] ?? null) ? $provision['blockers'] : [] + )))); + + return array_replace($provision ?? [], [ + 'ok' => true, + 'deferred' => true, + 'status' => 'waiting_for_coolify', + 'message' => 'Coolify deployment has started. Replication provisioning will continue after the service port becomes reachable.', + 'blockers' => $blockers, + ]); + } + + private function isTransientProvisionBlock(array $provision): bool + { + $blockers = is_array($provision['blockers'] ?? null) ? $provision['blockers'] : []; + if ($blockers === []) { + return false; + } + + $matched = false; + foreach ($blockers as $blocker) { + $message = strtolower(trim((string)$blocker)); + if ($message === '') { + continue; + } + $isTransient = false; + foreach ([ + 'connection refused', + 'connection timed out', + 'timed out', + 'timeout', + 'failed to connect', + 'could not connect', + 'no route to host', + 'network is unreachable', + 'connection reset', + 'temporarily unavailable', + 'temporary failure', + 'name or service not known', + ] as $needle) { + if (str_contains($message, $needle)) { + $isTransient = true; + $matched = true; + break; + } + } + if (!$isTransient) { + return false; + } + } + + return $matched; + } + + public function restartTarget(int $targetId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $target = $this->getTarget($targetId); + $host = $this->replicationHost((int)$target['replication_host_id']); + if (self::blocksPrimaryMutation($host, 'restart')) { + return $this->blockedTargetOperation($target, $host, 'restart', $actorUserId); + } + + $resourceUuid = trim((string)($target['resource_uuid'] ?? '')); + if ($resourceUuid === '') { + throw new RuntimeException('Coolify target has no resource UUID yet. Reconcile it first.'); + } + + $operationId = $this->startOperation($targetId, (int)$target['instance_id'], 'restart', $actorUserId); + try { + $result = $this->clientForInstance($this->getInstance((int)$target['instance_id']))->restartService($resourceUuid); + $this->execute( + "UPDATE coolify_targets SET deployment_status = 'restarting', last_reconcile_status = 'restart_requested', last_reconciled_at = NOW() WHERE id = ?", + 'i', + [$targetId] + ); + $this->finishOperation($operationId, 'completed', 'Coolify restart requested.', []); + $this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_restart_requested', $actorUserId, 'warning', [ + 'resource_uuid' => $resourceUuid, + 'coolify' => self::redactCoolifyResponse($result), + ]); + + return [ + 'ok' => true, + 'target' => $this->publicTarget($this->getTarget($targetId)), + 'coolify' => self::redactCoolifyResponse($result), + ]; + } catch (Throwable $throwable) { + $this->finishOperation($operationId, 'failed', null, [$throwable->getMessage()]); + $this->markTargetFailure($targetId, 'restart_failed', $throwable->getMessage()); + throw $throwable; + } + } + + public function failoverTarget(int $targetId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $target = $this->getTarget($targetId); + $host = $this->replicationHost((int)$target['replication_host_id']); + $kind = (string)$target['kind']; + + if (($host['role'] ?? '') === 'primary') { + $result = (new replication_manager())->runAutomaticFailoverMonitor($actorUserId); + } else { + $result = (new replication_manager())->promoteHost($kind, (int)$host['id'], $actorUserId); + } + + $this->execute( + "UPDATE coolify_targets SET availability_state = ?, last_reconcile_status = 'failover_checked', last_reconciled_at = NOW() WHERE id = ?", + 'si', + [$this->availabilityStateForHost($this->replicationHost((int)$host['id'], true)), $targetId] + ); + $this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_failover_requested', $actorUserId, 'critical', [ + 'result' => $result, + ]); + + return [ + 'ok' => true, + 'target' => $this->publicTarget($this->getTarget($targetId)), + 'failover' => $result, + ]; + } + + public function deleteTarget(int $targetId, array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $target = $this->getTarget($targetId); + $host = $this->replicationHost((int)$target['replication_host_id'], true); + if (($host['role'] ?? '') === 'primary') { + throw new RuntimeException('Coolify cannot delete an active primary target. Promote a healthy replica first.'); + } + + $confirmation = trim((string)($input['confirm'] ?? $input['confirmation'] ?? '')); + $expected = 'delete-coolify-target-' . $targetId; + if ($confirmation !== $expected) { + throw new RuntimeException('Destructive confirmation is required. Send confirm="' . $expected . '".'); + } + + $deleteResource = $this->toBool($input['delete_resource'] ?? false, false); + $resourceUuid = trim((string)($target['resource_uuid'] ?? '')); + $coolifyResult = null; + if ($deleteResource && $resourceUuid !== '') { + $coolifyResult = $this->clientForInstance($this->getInstance((int)$target['instance_id']))->deleteService($resourceUuid); + } + + $hostRemoved = false; + $canRemoveHost = replication_manager::replicationHostCanBeRemoved($host) + || self::targetAllowsReplicaRemoval($target); + if ((int)($host['id'] ?? 0) > 0 && $canRemoveHost) { + (new replication_manager())->removeHost((string)$target['kind'], (int)$host['id'], $actorUserId, false); + $hostRemoved = true; + } + + $this->execute( + "UPDATE coolify_targets SET deleted_at = NOW(), deployment_status = 'removed', availability_state = 'degraded' WHERE id = ?", + 'i', + [$targetId] + ); + $this->audit($targetId, (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_deleted', $actorUserId, 'warning', [ + 'delete_resource' => $deleteResource, + 'host_removed' => $hostRemoved, + 'coolify' => self::redactCoolifyResponse(is_array($coolifyResult) ? $coolifyResult : []), + ]); + + return [ + 'ok' => true, + 'id' => $targetId, + 'host_removed' => $hostRemoved, + 'coolify' => self::redactCoolifyResponse(is_array($coolifyResult) ? $coolifyResult : []), + ]; + } + + public function runAvailabilityMaintenance(?int $actorUserId = null): array + { + $this->ensureSchema(); + $failover = (new replication_manager())->runAutomaticFailoverMonitor($actorUserId); + $updated = []; + foreach ($this->selectRows('SELECT id, kind, replication_host_id, resource_uuid, deployment_status FROM coolify_targets WHERE deleted_at IS NULL') as $target) { + $hostId = (int)($target['replication_host_id'] ?? 0); + if ($hostId <= 0) { + continue; + } + try { + $provision = null; + $host = $this->replicationHost($hostId, true); + if ($this->shouldRetryProvisioning($target, $host) + || $this->hasRunningReplicationProvisionOperation((string)$target['kind'], $hostId)) { + $provision = $this->attemptTargetProvision((int)$target['id'], $target, $hostId, $actorUserId); + } + $state = $this->availabilityStateForHost($this->replicationHost($hostId, true)); + $this->execute('UPDATE coolify_targets SET availability_state = ? WHERE id = ?', 'si', [$state, (int)$target['id']]); + $updated[] = [ + 'id' => (int)$target['id'], + 'availability_state' => $state, + 'deployment_status' => $this->getTargetDeploymentStatus((int)$target['id']), + 'provision' => $provision, + ]; + } catch (Throwable) { + } + } + + return [ + 'ok' => true, + 'failover' => $failover, + 'targets' => $updated, + ]; + } + + public function listLoadBalancerGateways(bool $includeDeleted = false): array + { + $this->ensureSchema(); + $where = $includeDeleted ? '1=1' : 'deleted_at IS NULL'; + return array_map( + fn(array $gateway): array => $this->publicGateway($gateway), + $this->selectRows( + "SELECT * FROM coolify_instance_gateways WHERE $where ORDER BY priority ASC, id ASC" + ) + ); + } + + public function saveLoadBalancerGateway(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + + $id = (int)($input['id'] ?? 0); + $hostname = trim((string)($input['hostname'] ?? '')); + $targetIp = trim((string)($input['target_ip'] ?? $input['ip'] ?? '')); + $enabled = $this->toBool($input['enabled'] ?? true, true) ? 1 : 0; + $priority = max(0, (int)($input['priority'] ?? 100)); + $instanceId = (int)($input['instance_id'] ?? 0); + $instanceIdValue = $instanceId > 0 ? $instanceId : null; + + if ($hostname === '' || $targetIp === '') { + throw new RuntimeException('Gateway hostname and target IP are required.'); + } + + if (filter_var($targetIp, FILTER_VALIDATE_IP) === false) { + throw new RuntimeException('Gateway target IP must be a valid IPv4 or IPv6 address.'); + } + + if ($id > 0) { + $this->execute( + "UPDATE coolify_instance_gateways + SET instance_id = ?, hostname = ?, target_ip = ?, enabled = ?, priority = ?, deleted_at = NULL + WHERE id = ?", + 'issiii', + [$instanceIdValue, $hostname, $targetIp, $enabled, $priority, $id] + ); + $action = 'load_balancer_gateway_updated'; + } else { + $this->execute( + "INSERT INTO coolify_instance_gateways (instance_id, hostname, target_ip, enabled, priority) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + instance_id = VALUES(instance_id), + hostname = VALUES(hostname), + enabled = VALUES(enabled), + priority = VALUES(priority), + deleted_at = NULL", + 'issii', + [$instanceIdValue, $hostname, $targetIp, $enabled, $priority] + ); + $id = $this->insertId(); + if ($id <= 0) { + $row = $this->selectOne('SELECT id FROM coolify_instance_gateways WHERE target_ip = ? LIMIT 1', 's', [$targetIp]); + $id = (int)($row['id'] ?? 0); + } + $action = 'load_balancer_gateway_saved'; + } + + $gateway = $this->getGateway($id); + $this->audit(null, $instanceIdValue, null, $action, $actorUserId, 'info', [ + 'gateway_id' => $id, + 'hostname' => $hostname, + 'target_ip' => $targetIp, + 'enabled' => (bool)$enabled, + 'priority' => $priority, + ]); + + return $this->publicGateway($gateway); + } + + public function testLoadBalancerGateway(int $gatewayId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $gateway = $this->getGateway($gatewayId); + $publicHost = $this->coolifyConfigValue('public_gateway_host', self::DEFAULT_PUBLIC_GATEWAY_HOST); + $result = $this->probeGatewayTarget((string)$gateway['target_ip'], $publicHost); + $state = ($result['ok'] ?? false) === true ? 'ok' : 'down'; + + $this->execute( + "UPDATE coolify_instance_gateways + SET health_state = ?, last_probe_json = ?, last_probed_at = NOW() + WHERE id = ?", + 'ssi', + [$state, self::jsonEncode($result), $gatewayId] + ); + $this->audit(null, isset($gateway['instance_id']) ? (int)$gateway['instance_id'] : null, null, 'load_balancer_gateway_tested', $actorUserId, $state === 'ok' ? 'info' : 'warning', [ + 'gateway_id' => $gatewayId, + 'target_ip' => $gateway['target_ip'] ?? null, + 'result' => $result, + ]); + + return [ + 'gateway' => $this->publicGateway($this->getGateway($gatewayId)), + 'test' => $result, + ]; + } + + public function loadBalancerSummary(): array + { + $this->ensureSchema(); + $config = $this->loadBalancerConfig(); + $gateways = $this->listLoadBalancerGateways(); + $base = [ + 'configured' => $config['load_balancer_id'] !== '' && $config['token_set'], + 'status' => 'not_configured', + 'config' => $this->publicLoadBalancerConfig($config), + 'gateways' => $gateways, + 'load_balancer' => null, + 'drift' => [], + 'last_error' => null, + ]; + + if (!$base['configured']) { + return $base; + } + + try { + $loadBalancer = $this->hetznerClient($config['token'])->getLoadBalancer($config['load_balancer_id']); + $drift = $this->planLoadBalancerReconcile($loadBalancer, $gateways); + $this->syncGatewayLoadBalancerStates($gateways, $drift['actual_target_ips']); + + return array_replace($base, [ + 'status' => $drift['has_drift'] ? 'degraded' : 'ok', + 'gateways' => $this->listLoadBalancerGateways(), + 'load_balancer' => $this->publicLoadBalancer($loadBalancer), + 'drift' => $drift, + ]); + } catch (Throwable $throwable) { + return array_replace($base, [ + 'status' => 'down', + 'last_error' => $throwable->getMessage(), + ]); + } + } + + public function reconcileLoadBalancer(bool $dryRun = true, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $config = $this->loadBalancerConfig(); + if ($config['load_balancer_id'] === '' || !$config['token_set']) { + throw new RuntimeException('Hetzner Load Balancer ID and API token are required.'); + } + + $client = $this->hetznerClient($config['token']); + $loadBalancer = $client->getLoadBalancer($config['load_balancer_id']); + $gateways = $this->listLoadBalancerGateways(true); + $plan = $this->planLoadBalancerReconcile($loadBalancer, $gateways); + $canMutate = !$dryRun && $config['automation_enabled'] && $config['automation_mode'] === 'enforce'; + $applied = []; + $skipped = []; + $errors = []; + + foreach ($plan['actions'] as $action) { + $type = (string)($action['type'] ?? ''); + if ($type === 'skip_remove_target') { + $skipped[] = $action; + continue; + } + + if (!$canMutate) { + $skipped[] = array_replace($action, ['reason' => $action['reason'] ?? 'report_only']); + continue; + } + + try { + if ($type === 'add_target') { + $client->addIpTarget($config['load_balancer_id'], (string)$action['target_ip']); + } elseif ($type === 'remove_target') { + $client->removeIpTarget($config['load_balancer_id'], (string)$action['target_ip']); + } elseif ($type === 'add_service') { + $client->addService( + $config['load_balancer_id'], + (string)$action['protocol'], + (int)$action['listen_port'], + (int)$action['destination_port'] + ); + } else { + $skipped[] = array_replace($action, ['reason' => 'unknown_action']); + continue; + } + $applied[] = $action; + } catch (hetzner_cloud_api_exception $exception) { + if (($action['type'] ?? '') === 'add_target' && $exception->apiCode() === 'target_already_defined') { + $applied[] = array_replace($action, ['already_defined' => true]); + continue; + } + $errors[] = array_replace($action, [ + 'error' => $exception->getMessage(), + 'api_code' => $exception->apiCode(), + ]); + } catch (Throwable $throwable) { + $errors[] = array_replace($action, ['error' => $throwable->getMessage()]); + } + } + + $this->audit(null, null, null, $canMutate ? 'load_balancer_reconcile_applied' : 'load_balancer_reconcile_planned', $actorUserId, $errors === [] ? 'info' : 'warning', [ + 'dry_run' => $dryRun, + 'can_mutate' => $canMutate, + 'automation_enabled' => $config['automation_enabled'], + 'automation_mode' => $config['automation_mode'], + 'load_balancer_id' => $config['load_balancer_id'], + 'actions' => $plan['actions'], + 'applied' => $applied, + 'skipped' => $skipped, + 'errors' => $errors, + ]); + + $freshLoadBalancer = $loadBalancer; + if ($canMutate && $errors === []) { + $freshLoadBalancer = $client->getLoadBalancer($config['load_balancer_id']); + } + $freshPlan = $this->planLoadBalancerReconcile($freshLoadBalancer, $this->listLoadBalancerGateways(true)); + $this->syncGatewayLoadBalancerStates($this->listLoadBalancerGateways(), $freshPlan['actual_target_ips']); + + return [ + 'ok' => $errors === [], + 'dry_run' => $dryRun, + 'mutated' => $canMutate, + 'config' => $this->publicLoadBalancerConfig($config), + 'load_balancer' => $this->publicLoadBalancer($freshLoadBalancer), + 'drift' => $freshPlan, + 'planned' => $plan['actions'], + 'applied' => $applied, + 'skipped' => $skipped, + 'errors' => $errors, + 'gateways' => $this->listLoadBalancerGateways(), + ]; + } + + public function loadBalancerAutomationEnabled(): bool + { + $this->ensureSchema(); + $config = $this->loadBalancerConfig(); + return $config['automation_enabled'] + && $config['load_balancer_id'] !== '' + && $config['token_set']; + } + + private function shouldRetryProvisioning(array $target, ?array $host = null): bool + { + if (trim((string)($target['resource_uuid'] ?? '')) === '') { + return false; + } + + if (in_array((string)($target['deployment_status'] ?? ''), ['created', 'deploying', 'provision_blocked'], true)) { + return true; + } + + return $host !== null && self::replicationHostStillNeedsProvisioning($host); + } + + private function hasRunningReplicationProvisionOperation(string $kind, int $hostId): bool + { + if ($hostId <= 0) { + return false; + } + + return $this->selectOne( + "SELECT id FROM replication_operations + WHERE kind = ? AND host_id = ? AND operation = 'provision' AND status = 'running' + LIMIT 1", + 'si', + [$kind, $hostId] + ) !== null; + } + + private static function replicationHostStillNeedsProvisioning(array $host): bool + { + if ((string)($host['role'] ?? '') === 'primary') { + return false; + } + + $status = self::jsonDecode($host['last_status_json'] ?? null); + $effectiveStatus = (string)($status['status'] ?? $host['status'] ?? 'unknown'); + $percent = round((float)($status['replication_percent'] ?? 0), 2); + $blockers = array_values(array_filter($status['blockers'] ?? [])); + + return $effectiveStatus !== 'ok' || $percent < 100.0 || $blockers !== []; + } + + private function getTargetDeploymentStatus(int $targetId): string + { + try { + $target = $this->selectOne('SELECT deployment_status FROM coolify_targets WHERE id = ? LIMIT 1', 'i', [$targetId]); + return (string)($target['deployment_status'] ?? 'unknown'); + } catch (Throwable) { + return 'unknown'; + } + } + + public static function parseEnvFile(string $env): array + { + $values = []; + foreach (preg_split('/\r\n|\r|\n/', $env) ?: [] as $line) { + $line = trim($line); + if ($line === '' || str_starts_with($line, '#') || !str_contains($line, '=')) { + continue; + } + [$key, $value] = explode('=', $line, 2); + $key = trim($key); + if ($key === '') { + continue; + } + $values[$key] = trim($value); + } + return $values; + } + + public static function blocksPrimaryMutation(array $host, string $operation): bool + { + return in_array($operation, ['deploy', 'restart', 'delete', 'stop', 'replace'], true) + && (string)($host['role'] ?? '') === 'primary'; + } + + public static function targetAllowsReplicaRemoval(?array $target): bool + { + if ($target === null || (string)($target['role'] ?? $target['replication_role'] ?? '') === 'primary') { + return false; + } + + $deploymentStatus = (string)($target['deployment_status'] ?? ''); + $lastReconcileStatus = (string)($target['last_reconcile_status'] ?? ''); + if (in_array($deploymentStatus, ['reconcile_failed', 'removed', 'delete_failed'], true) + || in_array($lastReconcileStatus, ['reconcile_failed', 'delete_failed'], true)) { + return true; + } + + $lastReconcile = self::jsonDecode($target['last_reconcile_json'] ?? null); + $message = strtolower((string)($lastReconcile['message'] ?? $lastReconcile['error'] ?? '')); + return $message !== '' && (str_contains($message, 'not found') || str_contains($message, '404')); + } + + public static function replicationHostCanBeRemoved(array $host): bool + { + if ((string)($host['role'] ?? '') === 'primary') { + return false; + } + + $hostId = (int)($host['id'] ?? 0); + if ($hostId <= 0) { + return false; + } + + try { + if (!coolify_schema_bootstrap::tablesExist()) { + return false; + } + + $manager = new self(); + $target = $manager->selectOne( + 'SELECT * FROM coolify_targets WHERE replication_host_id = ? AND deleted_at IS NULL ORDER BY id DESC LIMIT 1', + 'i', + [$hostId] + ); + if ($target === null) { + return self::hostHasCoolifyMetadata($host); + } + + return self::targetAllowsReplicaRemoval($target); + } catch (Throwable) { + return false; + } + } + + public static function markTargetsRemovedForReplicationHost(int $hostId, ?int $actorUserId = null): void + { + if ($hostId <= 0) { + return; + } + + try { + if (!coolify_schema_bootstrap::tablesExist()) { + return; + } + + $manager = new self(); + $targets = $manager->selectRows( + 'SELECT id, instance_id, replication_host_id FROM coolify_targets WHERE replication_host_id = ? AND deleted_at IS NULL', + 'i', + [$hostId] + ); + if ($targets === []) { + return; + } + + $manager->execute( + "UPDATE coolify_targets + SET deleted_at = NOW(), deployment_status = 'removed', availability_state = 'degraded' + WHERE replication_host_id = ? AND deleted_at IS NULL", + 'i', + [$hostId] + ); + + foreach ($targets as $target) { + $manager->audit( + (int)$target['id'], + (int)$target['instance_id'], + (int)$target['replication_host_id'], + 'target_removed_with_replication_host', + $actorUserId, + 'warning', + ['host_id' => $hostId] + ); + } + } catch (Throwable) { + // Removing the replication host should not be blocked by optional Coolify metadata cleanup. + } + } + + public static function deploymentMetadataForReplicationHost(int $hostId): ?array + { + if ($hostId <= 0) { + return null; + } + + try { + global $db; + if (!coolify_schema_bootstrap::tablesExist()) { + return null; + } + $stmt = $db->prepare( + "SELECT t.*, i.label AS instance_label, i.base_url AS instance_base_url + FROM coolify_targets t + INNER JOIN coolify_instances i ON i.id = t.instance_id + WHERE t.replication_host_id = ? AND t.deleted_at IS NULL + ORDER BY t.id DESC LIMIT 1" + ); + if ($stmt === false) { + return null; + } + $stmt->bind_param('i', $hostId); + $stmt->execute(); + $result = $stmt->get_result(); + $target = $result ? $result->fetch_assoc() : null; + if (!is_array($target)) { + return null; + } + + return [ + 'target_id' => (int)$target['id'], + 'instance_id' => (int)$target['instance_id'], + 'instance_label' => (string)($target['instance_label'] ?? ''), + 'base_url' => (string)($target['instance_base_url'] ?? ''), + 'server_uuid' => $target['server_uuid'] ?? null, + 'project_uuid' => $target['project_uuid'] ?? null, + 'environment_uuid' => $target['environment_uuid'] ?? null, + 'environment_name' => $target['environment_name'] ?? null, + 'destination_uuid' => $target['destination_uuid'] ?? null, + 'resource_uuid' => $target['resource_uuid'] ?? null, + 'resource_type' => (string)($target['resource_type'] ?? self::RESOURCE_TYPE_SERVICE), + 'resource_name' => $target['resource_name'] ?? null, + 'deployment_status' => (string)($target['deployment_status'] ?? 'unknown'), + 'last_reconcile_status' => $target['last_reconcile_status'] ?? null, + 'last_reconciled_at' => $target['last_reconciled_at'] ?? null, + 'availability_state' => (string)($target['availability_state'] ?? 'degraded'), + ]; + } catch (Throwable) { + return null; + } + } + + public static function syncDeploymentStateForReplicationHost(int $hostId): void + { + if ($hostId <= 0) { + return; + } + + try { + if (!coolify_schema_bootstrap::tablesExist()) { + return; + } + + (new self())->syncTargetsForReplicationHost($hostId); + } catch (Throwable) { + // Replication health checks must not fail just because Coolify metadata cannot be updated. + } + } + + public static function syncLabelForReplicationHost(int $hostId, string $label): void + { + if ($hostId <= 0 || trim($label) === '') { + return; + } + + try { + if (!coolify_schema_bootstrap::tablesExist()) { + return; + } + + (new self())->execute( + 'UPDATE coolify_targets SET label = ? WHERE replication_host_id = ? AND deleted_at IS NULL', + 'si', + [$label, $hostId] + ); + } catch (Throwable) { + // Renaming a replication host should not fail because optional Coolify metadata is unavailable. + } + } + + private function syncTargetsForReplicationHost(int $hostId): void + { + $host = $this->replicationHost($hostId, true); + $availabilityState = $this->availabilityStateForHost($host); + $hostIsReady = $this->replicationHostIsReady($host); + + foreach ($this->selectRows( + 'SELECT id, deployment_status FROM coolify_targets WHERE replication_host_id = ? AND deleted_at IS NULL', + 'i', + [$hostId] + ) as $target) { + $deploymentStatus = (string)($target['deployment_status'] ?? 'unknown'); + $nextDeploymentStatus = $deploymentStatus; + if ($hostIsReady && in_array($deploymentStatus, ['pending', 'created', 'deploying', 'provision_blocked', 'restarting'], true)) { + $nextDeploymentStatus = 'provisioned'; + } + + $this->execute( + 'UPDATE coolify_targets SET availability_state = ?, deployment_status = ? WHERE id = ?', + 'ssi', + [$availabilityState, $nextDeploymentStatus, (int)$target['id']] + ); + } + } + + private function ensureSchema(): void + { + if ($this->schemaEnsured) { + return; + } + + coolify_schema_bootstrap::ensureTables(); + $this->schemaEnsured = true; + } + + private function composeInputFromRequest(string $kind, array $input, array $instance): array + { + $hostPort = (int)($input['host_port'] ?? $input['port'] ?? match ($kind) { + 'database' => 3307, + 'redis' => 6380, + default => 9010, + }); + + $base = [ + 'kind' => $kind, + 'role' => 'replica', + 'service_name' => $input['service_name'] ?? $input['resource_name'] ?? null, + 'host_port' => $hostPort, + ]; + + if ($kind === 'database') { + $base['database'] = (string)($input['database'] ?? $input['database_name'] ?? 'nnks_db'); + $base['username'] = (string)($input['username'] ?? 'nnks_db_user'); + $base['server_id'] = (int)($input['server_id'] ?? max(2, time() % 4294967295)); + [$base['primary_host'], $base['primary_port']] = $this->primaryAddress('database'); + } elseif ($kind === 'redis') { + [$base['primary_host'], $base['primary_port']] = $this->primaryAddress('redis'); + } else { + $base['host'] = (string)($input['host'] ?? ''); + $base['scheme'] = (string)($input['scheme'] ?? 'http'); + $base['console_port'] = (int)($input['console_port'] ?? ($hostPort + 1)); + $base['buckets'] = $this->normalizeBuckets($input['buckets'] ?? null); + $base['replication_transfer_limit'] = (string)($input['replication_transfer_limit'] + ?? ($input['options']['replication_transfer_limit'] ?? '')); + [$base['primary_host'], $base['primary_port']] = $this->primaryAddress('minio'); + } + + return $base; + } + + private function hostPayloadFromTemplate(string $kind, array $input, array $template): array + { + $credentials = is_array($template['credentials'] ?? null) ? $template['credentials'] : []; + $host = trim((string)($input['host'] ?? $input['endpoint'] ?? '')); + if ($host === '') { + throw new RuntimeException('Target host is required so replication can reach the Coolify-managed container.'); + } + + $payload = [ + 'label' => trim((string)($input['label'] ?? $credentials['label'] ?? $template['service_name'] ?? '')), + 'host' => $host, + 'port' => (int)($credentials['port'] ?? $input['port'] ?? $template['host_port'] ?? 0), + 'username' => (string)($credentials['username'] ?? $input['username'] ?? ''), + 'password' => (string)($credentials['password'] ?? $input['password'] ?? ''), + ]; + + if ($kind === 'database') { + $payload['database'] = (string)($credentials['database'] ?? $input['database'] ?? $input['database_name'] ?? ''); + $payload['admin_username'] = (string)($credentials['admin_username'] ?? $input['admin_username'] ?? 'root'); + $payload['admin_password'] = (string)($credentials['admin_password'] ?? $input['admin_password'] ?? ''); + $payload['replication_username'] = (string)($credentials['replication_username'] ?? $input['replication_username'] ?? 'replication'); + $payload['replication_password'] = (string)($credentials['replication_password'] ?? $input['replication_password'] ?? ''); + $payload['ssl_mode'] = (string)($credentials['ssl_mode'] ?? $input['ssl_mode'] ?? 'DISABLED'); + $payload['options'] = ['allow_preseeded_replica' => true]; + } elseif ($kind === 'redis') { + $payload['database'] = (int)($credentials['database'] ?? $input['database'] ?? 0); + } else { + $payload['scheme'] = (string)($credentials['scheme'] ?? $input['scheme'] ?? 'http'); + $payload['buckets'] = $credentials['buckets'] ?? $this->normalizeBuckets($input['buckets'] ?? null); + $payload['console_port'] = (int)($credentials['console_port'] ?? $input['console_port'] ?? 9001); + $payload['replication_transfer_limit'] = (string)($credentials['replication_transfer_limit'] + ?? $input['replication_transfer_limit'] + ?? ($input['options']['replication_transfer_limit'] ?? '')); + $payload['options'] = [ + 'scheme' => $payload['scheme'], + 'buckets' => $payload['buckets'], + 'console_port' => $payload['console_port'], + 'replication_transfer_limit' => $payload['replication_transfer_limit'], + 'space_headroom_percent' => (float)($credentials['space_headroom_percent'] ?? 20.0), + ]; + } + + return $payload; + } + + private function composeTemplateForTarget(array $target, array $host): array + { + $kind = (string)$target['kind']; + $options = self::jsonDecode($target['options_json'] ?? null); + $credentials = $this->hostCredentials($host); + $input = is_array($options['compose_input'] ?? null) ? $options['compose_input'] : []; + $input['kind'] = $kind; + $input['role'] = 'replica'; + $input['service_name'] = $target['resource_name'] ?? $target['label'] ?? null; + $input['host_port'] = (int)($host['port'] ?? $input['host_port'] ?? 0); + + if ($kind === 'database') { + $input['database'] = (string)($host['database_name'] ?? $input['database'] ?? ''); + $input['username'] = (string)($host['username'] ?? $input['username'] ?? ''); + $input['password'] = $credentials['password']; + $input['admin_username'] = $credentials['admin_username'] ?: 'root'; + $input['admin_password'] = $credentials['admin_password']; + $input['replication_username'] = $credentials['replication_username'] ?: 'replication'; + $input['replication_password'] = $credentials['replication_password']; + [$input['primary_host'], $input['primary_port']] = $this->primaryAddress('database'); + $primaryCredentials = $this->primaryCredentials('database'); + if ($primaryCredentials !== []) { + $input['primary_admin_username'] = $primaryCredentials['admin_username'] + ?: ($primaryCredentials['username'] ?: 'root'); + $input['primary_admin_password'] = $primaryCredentials['admin_password'] + ?: $primaryCredentials['password']; + } + } elseif ($kind === 'redis') { + $input['password'] = $credentials['password']; + [$input['primary_host'], $input['primary_port']] = $this->primaryAddress('redis'); + } else { + $hostOptions = self::jsonDecode($host['options_json'] ?? null); + $input['host'] = (string)($host['host'] ?? $input['host'] ?? ''); + $input['scheme'] = (string)($hostOptions['scheme'] ?? $input['scheme'] ?? 'http'); + $input['username'] = $credentials['username']; + $input['password'] = $credentials['password']; + $input['buckets'] = $hostOptions['buckets'] ?? $input['buckets'] ?? []; + $input['console_port'] = (int)($hostOptions['console_port'] ?? $input['console_port'] ?? 9001); + $input['replication_transfer_limit'] = (string)($hostOptions['replication_transfer_limit'] + ?? $input['replication_transfer_limit'] + ?? ''); + [$input['primary_host'], $input['primary_port']] = $this->primaryAddress('minio'); + } + + return replication_manager::composeTemplate($input); + } + + private function primaryCredentials(string $kind): array + { + $primary = $this->selectOne( + "SELECT * FROM replication_hosts WHERE kind = ? AND role = 'primary' AND deleted_at IS NULL ORDER BY id DESC LIMIT 1", + 's', + [$kind] + ); + if ($primary === null) { + return []; + } + + return $this->hostCredentials($primary); + } + + private function startOrRestartService(coolify_api_client $client, string $resourceUuid, bool $restartIfRunning): array + { + try { + return $client->startService($resourceUuid); + } catch (Throwable $throwable) { + if (!str_contains(strtolower($throwable->getMessage()), 'already running')) { + throw $throwable; + } + + if ($restartIfRunning) { + return array_replace( + ['already_running' => true, 'action' => 'restart_requested'], + $client->restartService($resourceUuid) + ); + } + + return [ + 'already_running' => true, + 'action' => 'start_noop', + 'message' => 'Service is already running.', + ]; + } + } + + private function recordCreatedResource(int $targetId, string $resourceUuid, string $hash): void + { + $this->execute( + "UPDATE coolify_targets + SET resource_uuid = ?, deployment_status = 'created', desired_compose_hash = ?, + last_reconcile_status = 'created', last_reconciled_at = NOW() + WHERE id = ?", + 'ssi', + [$resourceUuid, $hash, $targetId] + ); + } + + private function servicePayload(array $target, array $template, bool $update): array + { + $payload = [ + 'name' => (string)($target['resource_name'] ?? $target['label']), + 'description' => 'Truckwash managed ' . $target['kind'] . ' replication target. Do not stop the active primary here.', + 'instant_deploy' => false, + 'docker_compose_raw' => $this->encodedDockerCompose($template), + 'force_domain_override' => false, + ]; + + if (!$update) { + $payload = array_replace($payload, [ + 'project_uuid' => $target['project_uuid'] ?? null, + 'environment_name' => $target['environment_name'] ?: 'production', + 'environment_uuid' => $target['environment_uuid'] ?? null, + 'server_uuid' => $target['server_uuid'] ?? null, + 'destination_uuid' => $target['destination_uuid'] ?? null, + ]); + } + + return array_filter($payload, static fn($value): bool => $value !== null && $value !== ''); + } + + private function encodedDockerCompose(array $template): string + { + return base64_encode((string)($template['compose'] ?? '')); + } + + private function targetOptions(array $input, array $template, array $composeInput): array + { + return [ + 'compose_input' => $composeInput, + 'compose_service_name' => (string)($template['service_name'] ?? ''), + 'engine' => (string)($template['engine'] ?? ''), + 'coolify_docs' => [ + 'services_endpoint' => '/api/v1/services', + 'envs_bulk_endpoint' => '/api/v1/services/{uuid}/envs/bulk', + ], + ]; + } + + private function attachTargetToReplicationHost(string $kind, int $hostId, int $targetId, int $instanceId): void + { + $host = $this->replicationHost($hostId, true); + $options = self::jsonDecode($host['options_json'] ?? null); + $options['deployment_provider'] = 'coolify'; + $options['coolify_instance_id'] = $instanceId; + $options['coolify_target_id'] = $targetId; + $this->execute( + 'UPDATE replication_hosts SET options_json = ? WHERE id = ? AND kind = ?', + 'sis', + [self::jsonEncode($options), $hostId, $kind] + ); + } + + private function blockedTargetOperation(array $target, array $host, string $operation, ?int $actorUserId): array + { + $context = [ + 'operation' => $operation, + 'reason' => 'active_primary_guard', + 'message' => 'Coolify will not mutate the active primary. Promote a healthy replica first.', + ]; + $this->execute( + "UPDATE coolify_targets SET availability_state = 'destructive_action_required', last_reconcile_status = ?, last_reconcile_json = ?, last_reconciled_at = NOW() WHERE id = ?", + 'ssi', + ['blocked', self::jsonEncode($context), (int)$target['id']] + ); + $this->audit((int)$target['id'], (int)$target['instance_id'], (int)$target['replication_host_id'], 'target_operation_blocked', $actorUserId, 'warning', $context); + + return [ + 'ok' => false, + 'status' => 'destructive_action_required', + 'message' => $context['message'], + 'target' => $this->publicTarget($this->getTarget((int)$target['id'])), + 'host' => [ + 'id' => (int)($host['id'] ?? 0), + 'role' => (string)($host['role'] ?? ''), + 'status' => (string)($host['status'] ?? ''), + ], + ]; + } + + private function availabilitySummary(): array + { + $summary = []; + foreach (self::KINDS as $kind) { + $targets = $this->listTargets($kind); + $states = array_map(static fn(array $target): string => (string)($target['availability_state'] ?? 'degraded'), $targets); + $summary[$kind] = [ + 'status' => in_array('protected', $states, true) ? 'protected' : ($targets === [] ? 'not_configured' : 'degraded'), + 'targets' => count($targets), + 'protected' => count(array_filter($states, static fn(string $state): bool => $state === 'protected' || $state === 'failover_ready')), + 'blocked' => count(array_filter($states, static fn(string $state): bool => str_contains($state, 'blocked') || $state === 'destructive_action_required')), + ]; + } + return $summary; + } + + private function loadBalancerConfig(): array + { + $mode = $this->coolifyConfigValue('lb_automation_mode', 'report_only'); + $mode = in_array($mode, ['report_only', 'enforce'], true) ? $mode : 'report_only'; + $token = $this->hetznerCloudToken(); + $tokenSource = trim((string)(getenv('HETZNER_CLOUD_API_TOKEN') ?: '')) !== '' ? 'env' : 'config'; + + return [ + 'automation_enabled' => $this->coolifyConfigBool('lb_automation_enabled', false), + 'automation_mode' => $mode, + 'load_balancer_id' => $this->coolifyConfigValue('hetzner_load_balancer_id', ''), + 'public_gateway_host' => $this->coolifyConfigValue('public_gateway_host', self::DEFAULT_PUBLIC_GATEWAY_HOST), + 'token' => $token, + 'token_set' => trim($token) !== '', + 'token_source' => trim($token) !== '' ? $tokenSource : null, + 'required_services' => self::REQUIRED_LOAD_BALANCER_SERVICES, + ]; + } + + private function publicLoadBalancerConfig(array $config): array + { + unset($config['token']); + return $config; + } + + private function coolifyConfigValue(string $variable, string $default = ''): string + { + $row = $this->selectOne( + "SELECT value FROM module_config WHERE module = 'Coolify' AND variable = ? LIMIT 1", + 's', + [$variable] + ); + $value = trim((string)($row['value'] ?? '')); + return $value !== '' ? $value : $default; + } + + private function coolifyConfigBool(string $variable, bool $default = false): bool + { + $row = $this->selectOne( + "SELECT value FROM module_config WHERE module = 'Coolify' AND variable = ? LIMIT 1", + 's', + [$variable] + ); + if ($row === null) { + return $default; + } + return $this->toBool($row['value'] ?? null, $default); + } + + private function hetznerCloudToken(): string + { + $envToken = trim((string)(getenv('HETZNER_CLOUD_API_TOKEN') ?: '')); + if ($envToken !== '') { + return $envToken; + } + + $stored = $this->coolifyConfigValue('hetzner_cloud_api_token', ''); + if ($stored === '') { + return ''; + } + + return replication_secret_box::decrypt($stored); + } + + private function hetznerClient(string $token): object + { + if ($this->hetznerClientFactory !== null) { + $client = call_user_func($this->hetznerClientFactory, $token); + foreach (['getLoadBalancer', 'addIpTarget', 'removeIpTarget', 'addService'] as $method) { + if (!is_object($client) || !method_exists($client, $method)) { + throw new RuntimeException('Hetzner client factory returned an invalid client.'); + } + } + return $client; + } + + return new hetzner_cloud_client($token); + } + + private function planLoadBalancerReconcile(array $loadBalancer, array $gateways): array + { + $actualTargetIps = self::loadBalancerIpTargets($loadBalancer); + $actualServices = self::loadBalancerServices($loadBalancer); + $enabledIps = []; + $actions = []; + $missingTargets = []; + $disabledPresentTargets = []; + $missingServices = []; + + foreach ($gateways as $gateway) { + $targetIp = trim((string)($gateway['target_ip'] ?? '')); + if ($targetIp === '') { + continue; + } + + if (empty($gateway['deleted_at']) && !empty($gateway['enabled'])) { + $enabledIps[] = $targetIp; + if (!in_array($targetIp, $actualTargetIps, true)) { + $missingTargets[] = $targetIp; + $actions[] = [ + 'type' => 'add_target', + 'target_ip' => $targetIp, + 'hostname' => $gateway['hostname'] ?? null, + ]; + } + continue; + } + + if (in_array($targetIp, $actualTargetIps, true)) { + $disabledPresentTargets[] = $targetIp; + $actions[] = [ + 'type' => 'remove_target', + 'target_ip' => $targetIp, + 'hostname' => $gateway['hostname'] ?? null, + ]; + } + } + + foreach (self::REQUIRED_LOAD_BALANCER_SERVICES as $requiredService) { + if (self::hasLoadBalancerService($actualServices, $requiredService)) { + continue; + } + $missingServices[] = $requiredService; + $actions[] = array_replace(['type' => 'add_service'], $requiredService); + } + + $actions = $this->guardLastLoadBalancerTarget($actions, $actualTargetIps); + + return [ + 'has_drift' => $actions !== [], + 'actions' => array_values($actions), + 'missing_targets' => array_values($missingTargets), + 'disabled_present_targets' => array_values($disabledPresentTargets), + 'missing_services' => array_values($missingServices), + 'actual_target_ips' => $actualTargetIps, + 'expected_target_ips' => array_values(array_unique($enabledIps)), + 'actual_services' => $actualServices, + 'required_services' => self::REQUIRED_LOAD_BALANCER_SERVICES, + ]; + } + + private function guardLastLoadBalancerTarget(array $actions, array $actualTargetIps): array + { + $remainingTargets = count($actualTargetIps); + $guarded = []; + + foreach ($actions as $action) { + if (($action['type'] ?? '') !== 'remove_target') { + $guarded[] = $action; + continue; + } + + if ($remainingTargets <= 1) { + $guarded[] = array_replace($action, [ + 'type' => 'skip_remove_target', + 'reason' => 'last_reachable_target_guard', + ]); + continue; + } + + $remainingTargets--; + $guarded[] = $action; + } + + return $guarded; + } + + private function syncGatewayLoadBalancerStates(array $gateways, array $actualTargetIps): void + { + foreach ($gateways as $gateway) { + $targetIp = trim((string)($gateway['target_ip'] ?? '')); + if ($targetIp === '') { + continue; + } + $enabled = !empty($gateway['enabled']); + $present = in_array($targetIp, $actualTargetIps, true); + $state = match (true) { + $enabled && $present => 'in_lb', + $enabled && !$present => 'missing', + !$enabled && $present => 'disabled_present', + default => 'disabled_absent', + }; + $this->execute( + 'UPDATE coolify_instance_gateways SET lb_state = ?, last_reconciled_at = NOW() WHERE id = ?', + 'si', + [$state, (int)$gateway['id']] + ); + } + } + + private static function loadBalancerIpTargets(array $loadBalancer): array + { + $ips = []; + foreach (($loadBalancer['targets'] ?? []) as $target) { + if (!is_array($target)) { + continue; + } + $type = strtolower((string)($target['type'] ?? '')); + $ip = ''; + if ($type === 'ip') { + $ipPayload = is_array($target['ip'] ?? null) ? $target['ip'] : []; + $ip = (string)($ipPayload['ip'] ?? ''); + } elseif (isset($target['server']['public_net']['ipv4']['ip'])) { + $ip = (string)$target['server']['public_net']['ipv4']['ip']; + } + $ip = trim($ip); + if ($ip !== '') { + $ips[] = $ip; + } + } + + return array_values(array_unique($ips)); + } + + private static function loadBalancerServices(array $loadBalancer): array + { + $services = []; + foreach (($loadBalancer['services'] ?? []) as $service) { + if (!is_array($service)) { + continue; + } + $services[] = [ + 'protocol' => strtolower((string)($service['protocol'] ?? '')), + 'listen_port' => (int)($service['listen_port'] ?? 0), + 'destination_port' => (int)($service['destination_port'] ?? 0), + 'proxyprotocol' => (bool)($service['proxyprotocol'] ?? false), + ]; + } + return $services; + } + + private static function hasLoadBalancerService(array $services, array $required): bool + { + foreach ($services as $service) { + if ((string)$service['protocol'] === (string)$required['protocol'] + && (int)$service['listen_port'] === (int)$required['listen_port'] + && (int)$service['destination_port'] === (int)$required['destination_port'] + && empty($service['proxyprotocol'])) { + return true; + } + } + + return false; + } + + private function publicLoadBalancer(array $loadBalancer): array + { + return [ + 'id' => isset($loadBalancer['id']) ? (int)$loadBalancer['id'] : null, + 'name' => (string)($loadBalancer['name'] ?? ''), + 'ipv4' => $loadBalancer['public_net']['ipv4']['ip'] ?? null, + 'ipv6' => $loadBalancer['public_net']['ipv6']['ip'] ?? null, + 'location' => $loadBalancer['location']['name'] ?? null, + 'algorithm' => $loadBalancer['algorithm']['type'] ?? null, + 'targets' => self::loadBalancerIpTargets($loadBalancer), + 'services' => self::loadBalancerServices($loadBalancer), + ]; + } + + private function publicGateway(array $gateway): array + { + return [ + 'id' => (int)$gateway['id'], + 'instance_id' => isset($gateway['instance_id']) ? (int)$gateway['instance_id'] : null, + 'hostname' => (string)$gateway['hostname'], + 'target_ip' => (string)$gateway['target_ip'], + 'enabled' => (bool)$gateway['enabled'], + 'priority' => (int)$gateway['priority'], + 'health_state' => (string)($gateway['health_state'] ?? 'unknown'), + 'lb_state' => (string)($gateway['lb_state'] ?? 'unknown'), + 'last_probe' => self::jsonDecode($gateway['last_probe_json'] ?? null), + 'last_probed_at' => $gateway['last_probed_at'] ?? null, + 'last_reconciled_at' => $gateway['last_reconciled_at'] ?? null, + 'deleted_at' => $gateway['deleted_at'] ?? null, + 'created_at' => $gateway['created_at'] ?? null, + 'updated_at' => $gateway['updated_at'] ?? null, + ]; + } + + private function getGateway(int $id): array + { + $gateway = $this->selectOne( + 'SELECT * FROM coolify_instance_gateways WHERE id = ? AND deleted_at IS NULL LIMIT 1', + 'i', + [$id] + ); + if ($gateway === null) { + throw new RuntimeException('Coolify gateway target was not found.'); + } + return $gateway; + } + + private function probeGatewayTarget(string $targetIp, string $publicHost): array + { + $startedAt = microtime(true); + $url = 'https://' . $publicHost . '/ping'; + $curl = curl_init($url); + if ($curl === false) { + throw new RuntimeException('Could not initialize gateway probe.'); + } + + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 3); + curl_setopt($curl, CURLOPT_TIMEOUT, 5); + curl_setopt($curl, CURLOPT_NOSIGNAL, true); + curl_setopt($curl, CURLOPT_HTTPHEADER, ['Accept: application/json']); + curl_setopt($curl, CURLOPT_RESOLVE, [$publicHost . ':443:' . $targetIp]); + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false); + + $raw = curl_exec($curl); + $error = curl_error($curl); + $status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE); + curl_close($curl); + + return [ + 'ok' => $raw !== false && $status >= 200 && $status < 300, + 'status_code' => $status ?: null, + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + 'host' => $publicHost, + 'target_ip' => $targetIp, + 'path' => '/ping', + 'error' => $raw === false ? $error : null, + 'checked_at' => date('c'), + ]; + } + + private function availabilityStateForHost(array $host): string + { + $role = (string)($host['role'] ?? ''); + $status = self::jsonDecode($host['last_status_json'] ?? null); + $effectiveStatus = (string)($status['status'] ?? $host['status'] ?? 'unknown'); + $percent = round((float)($status['replication_percent'] ?? ($role === 'primary' ? 100 : 0)), 2); + $blockers = is_array($status['blockers'] ?? null) ? $status['blockers'] : []; + + if ($role === 'primary') { + return $this->hasHealthyReplica((string)$host['kind'], (int)$host['id']) ? 'protected' : 'degraded'; + } + if ($effectiveStatus === 'ok' && $percent >= 100.0 && $blockers === []) { + return 'failover_ready'; + } + if (in_array($effectiveStatus, ['down', 'removed'], true)) { + return 'degraded'; + } + return 'failover_blocked'; + } + + private function replicationHostIsReady(array $host): bool + { + $status = self::jsonDecode($host['last_status_json'] ?? null); + $effectiveStatus = (string)($status['status'] ?? $host['status'] ?? 'unknown'); + $percent = round((float)($status['replication_percent'] ?? (($host['role'] ?? '') === 'primary' ? 100 : 0)), 2); + $blockers = is_array($status['blockers'] ?? null) ? $status['blockers'] : []; + + return $effectiveStatus === 'ok' && $percent >= 100.0 && $blockers === []; + } + + private function hasHealthyReplica(string $kind, int $primaryId): bool + { + foreach ($this->selectRows( + "SELECT * FROM replication_hosts WHERE kind = ? AND role = 'replica' AND deleted_at IS NULL AND id <> ?", + 'si', + [$kind, $primaryId] + ) as $host) { + $status = self::jsonDecode($host['last_status_json'] ?? null); + if (($status['status'] ?? '') === 'ok' + && round((float)($status['replication_percent'] ?? 0), 2) >= 100.0 + && (is_array($status['blockers'] ?? null) ? $status['blockers'] : []) === []) { + return true; + } + } + return false; + } + + private function coolifyCollection(array $response): array + { + if (self::isListArray($response)) { + return array_values(array_filter($response, 'is_array')); + } + + foreach (['data', 'items', 'servers', 'projects', 'environments', 'resources'] as $key) { + if (!is_array($response[$key] ?? null)) { + continue; + } + + $collection = $response[$key]; + if (self::isListArray($collection)) { + return array_values(array_filter($collection, 'is_array')); + } + + return array_values(array_filter($collection, 'is_array')); + } + + return []; + } + + private function publicPlacementServer(array $server): array + { + $settings = is_array($server['settings'] ?? null) ? $server['settings'] : []; + $publicHost = self::publicServerHostFromCoolifyServer($server, null, false) + ?? self::resolvedPublicDnsServerHostFromCoolifyServer($server); + + return [ + 'id' => isset($server['id']) ? (int)$server['id'] : null, + 'uuid' => $this->placementString($server['uuid'] ?? ''), + 'name' => $this->placementString($server['name'] ?? $server['uuid'] ?? ''), + 'description' => $this->placementString($server['description'] ?? ''), + 'ip' => $this->placementString($server['ip'] ?? $server['public_ip'] ?? $server['address'] ?? ''), + 'public_host' => $publicHost, + 'user' => $this->placementString($server['user'] ?? ''), + 'port' => isset($server['port']) ? (int)$server['port'] : null, + 'proxy_type' => $this->placementString($server['proxy_type'] ?? ''), + 'swarm_cluster' => $this->placementString($server['swarm_cluster'] ?? ''), + 'is_reachable' => array_key_exists('is_reachable', $settings) ? (bool)$settings['is_reachable'] : null, + 'is_usable' => array_key_exists('is_usable', $settings) ? (bool)$settings['is_usable'] : null, + ]; + } + + private function publicPlacementProject(array $project): array + { + return [ + 'id' => isset($project['id']) ? (int)$project['id'] : null, + 'uuid' => $this->placementString($project['uuid'] ?? ''), + 'name' => $this->placementString($project['name'] ?? $project['uuid'] ?? ''), + 'description' => $this->placementString($project['description'] ?? ''), + ]; + } + + private function publicPlacementEnvironment(array $environment, array $project): array + { + return [ + 'id' => isset($environment['id']) ? (int)$environment['id'] : null, + 'uuid' => $this->placementString($environment['uuid'] ?? ''), + 'name' => $this->placementString($environment['name'] ?? $environment['uuid'] ?? ''), + 'description' => $this->placementString($environment['description'] ?? ''), + 'project_id' => isset($environment['project_id']) ? (int)$environment['project_id'] : null, + 'project_uuid' => $this->placementString($project['uuid'] ?? ''), + 'project_name' => $this->placementString($project['name'] ?? ''), + ]; + } + + private function placementString(mixed $value): string + { + return trim((string)($value ?? '')); + } + + private function publicInstance(array $instance): array + { + return [ + 'id' => (int)$instance['id'], + 'label' => (string)$instance['label'], + 'base_url' => (string)$instance['base_url'], + 'api_token_set' => trim((string)($instance['api_token_secret'] ?? '')) !== '', + 'default_project_uuid' => $instance['default_project_uuid'] ?? null, + 'default_environment_uuid' => $instance['default_environment_uuid'] ?? null, + 'default_environment_name' => $instance['default_environment_name'] ?? null, + 'default_server_uuid' => $instance['default_server_uuid'] ?? null, + 'default_destination_uuid' => $instance['default_destination_uuid'] ?? null, + 'status' => (string)($instance['status'] ?? 'unknown'), + 'last_checked_at' => $instance['last_checked_at'] ?? null, + 'last_error' => $instance['last_error'] ?? null, + 'created_at' => $instance['created_at'] ?? null, + 'updated_at' => $instance['updated_at'] ?? null, + ]; + } + + private function publicTarget(array $target): array + { + $replication = [ + 'host_id' => isset($target['replication_host_id']) ? (int)$target['replication_host_id'] : null, + 'label' => $target['replication_label'] ?? null, + 'host' => $target['replication_host'] ?? null, + 'port' => isset($target['replication_port']) ? (int)$target['replication_port'] : null, + 'role' => $target['replication_role'] ?? null, + 'status' => $target['replication_status'] ?? null, + 'last_status' => self::jsonDecode($target['replication_last_status_json'] ?? null), + 'last_checked_at' => $target['replication_last_checked_at'] ?? null, + ]; + + return [ + 'id' => (int)$target['id'], + 'instance_id' => (int)$target['instance_id'], + 'instance_label' => (string)($target['instance_label'] ?? ''), + 'kind' => (string)$target['kind'], + 'label' => (string)$target['label'], + 'role' => (string)$target['role'], + 'server_uuid' => $target['server_uuid'] ?? null, + 'project_uuid' => $target['project_uuid'] ?? null, + 'environment_uuid' => $target['environment_uuid'] ?? null, + 'environment_name' => $target['environment_name'] ?? null, + 'destination_uuid' => $target['destination_uuid'] ?? null, + 'resource_uuid' => $target['resource_uuid'] ?? null, + 'resource_type' => (string)($target['resource_type'] ?? self::RESOURCE_TYPE_SERVICE), + 'resource_name' => $target['resource_name'] ?? null, + 'deployment_status' => (string)($target['deployment_status'] ?? 'unknown'), + 'availability_state' => (string)($target['availability_state'] ?? 'degraded'), + 'last_reconcile_status' => $target['last_reconcile_status'] ?? null, + 'last_reconcile' => self::jsonDecode($target['last_reconcile_json'] ?? null), + 'last_reconciled_at' => $target['last_reconciled_at'] ?? null, + 'replication' => $replication, + 'created_at' => $target['created_at'] ?? null, + 'updated_at' => $target['updated_at'] ?? null, + ]; + } + + private function clientForInstance(array $instance): coolify_api_client + { + $token = replication_secret_box::decrypt($instance['api_token_secret'] ?? ''); + if ($this->clientFactory !== null) { + $client = call_user_func($this->clientFactory, $instance, $token); + if (!$client instanceof coolify_api_client) { + throw new RuntimeException('Coolify client factory returned an invalid client.'); + } + return $client; + } + return new coolify_api_client((string)$instance['base_url'], $token); + } + + private function getInstance(int $id): array + { + $instance = $this->selectOne('SELECT * FROM coolify_instances WHERE id = ? AND deleted_at IS NULL LIMIT 1', 'i', [$id]); + if ($instance === null) { + throw new RuntimeException('Coolify instance was not found.'); + } + return $instance; + } + + private function getTarget(int $id): array + { + $target = $this->selectOne( + "SELECT t.*, i.label AS instance_label, i.base_url AS instance_base_url, h.label AS replication_label, + h.host AS replication_host, h.port AS replication_port, h.role AS replication_role, + h.status AS replication_status, h.last_status_json AS replication_last_status_json, + h.last_checked_at AS replication_last_checked_at + FROM coolify_targets t + INNER JOIN coolify_instances i ON i.id = t.instance_id + LEFT JOIN replication_hosts h ON h.id = t.replication_host_id + WHERE t.id = ? AND t.deleted_at IS NULL LIMIT 1", + 'i', + [$id] + ); + if ($target === null) { + throw new RuntimeException('Coolify target was not found.'); + } + return $target; + } + + private function replicationHost(int $id, bool $includeDeleted = false): array + { + $sql = 'SELECT * FROM replication_hosts WHERE id = ?'; + if (!$includeDeleted) { + $sql .= ' AND deleted_at IS NULL'; + } + $host = $this->selectOne($sql . ' LIMIT 1', 'i', [$id]); + if ($host === null) { + throw new RuntimeException('Linked replication host was not found.'); + } + return $host; + } + + private function hostCredentials(array $host): array + { + return [ + 'username' => (string)($host['username'] ?? ''), + 'password' => replication_secret_box::decrypt($host['password_secret'] ?? ''), + 'admin_username' => (string)($host['admin_username'] ?? ''), + 'admin_password' => replication_secret_box::decrypt($host['admin_password_secret'] ?? ''), + 'replication_username' => (string)($host['replication_username'] ?? ''), + 'replication_password' => replication_secret_box::decrypt($host['replication_password_secret'] ?? ''), + ]; + } + + private function primaryAddress(string $kind): array + { + $primary = $this->selectOne( + "SELECT * FROM replication_hosts WHERE kind = ? AND role = 'primary' AND deleted_at IS NULL ORDER BY id DESC LIMIT 1", + 's', + [$kind] + ); + if ($primary === null) { + return match ($kind) { + 'database' => ['', 3306], + 'redis' => ['redis-primary', 6379], + default => ['http://minio-primary:9000', 9000], + }; + } + + if ($kind === 'minio') { + $options = self::jsonDecode($primary['options_json'] ?? null); + $endpoint = (string)($options['endpoint'] ?? (($options['scheme'] ?? 'http') . '://' . $primary['host'] . ':' . $primary['port'])); + return [$endpoint, (int)$primary['port']]; + } + + return [(string)$primary['host'], (int)$primary['port']]; + } + + private function defaultInstanceId(): int + { + $instance = $this->selectOne('SELECT id FROM coolify_instances WHERE deleted_at IS NULL ORDER BY id LIMIT 1'); + if ($instance === null) { + throw new RuntimeException('No Coolify instance is configured.'); + } + return (int)$instance['id']; + } + + private function applyCoolifyDeploymentDefaults(array $input, array $instance): array + { + $serverUuid = $this->targetMapping($input, $instance, 'server_uuid'); + if ($serverUuid === null) { + return $input; + } + + $serverHost = $this->resolveCoolifyServerHost( + $instance, + $serverUuid, + (int)($input['host_port'] ?? $input['port'] ?? 0), + 0 + ); + if ($serverHost !== null) { + $input['host'] = $serverHost; + } + + return $input; + } + + private function applyCoolifyPortDefaults(string $kind, array $input, array $instance): array + { + $serverUuid = $this->targetMapping($input, $instance, 'server_uuid'); + if ($serverUuid === null) { + return $input; + } + + $port = (int)($input['host_port'] ?? $input['port'] ?? match ($kind) { + 'database' => 3307, + 'redis' => 6380, + default => 9010, + }); + $consolePort = $kind === 'minio' ? (int)($input['console_port'] ?? ($port + 1)) : null; + [$nextPort, $nextConsolePort] = $this->nextAvailablePublicPorts( + $kind, + $port, + $consolePort, + $this->usedPublicPortsForCoolifyServer($serverUuid, 0) + ); + + $input['host_port'] = $nextPort; + $input['port'] = $nextPort; + if ($kind === 'minio' && $nextConsolePort !== null) { + $input['console_port'] = $nextConsolePort; + } + + return $input; + } + + private function resolveCoolifyServerHost(array $instance, string $serverUuid, ?int $port = null, int $excludeHostId = 0): ?string + { + try { + foreach ($this->coolifyCollection($this->clientForInstance($instance)->listServers()) as $server) { + if ($this->placementString($server['uuid'] ?? '') !== $serverUuid) { + continue; + } + + $publicHost = self::publicServerHostFromCoolifyServer($server, $port, false); + if ($publicHost !== null) { + return $publicHost; + } + + $knownHost = $this->knownPublicHostForCoolifyServer($serverUuid, $port, $excludeHostId); + if ($knownHost !== null) { + return $knownHost; + } + + return self::resolvedPublicDnsServerHostFromCoolifyServer($server); + } + } catch (Throwable) { + } + + return $this->knownPublicHostForCoolifyServer($serverUuid, $port, $excludeHostId); + } + + private function syncReplicationHostEndpointForTarget(array $target, array $host, array $instance): array + { + $serverUuid = trim((string)($target['server_uuid'] ?? '')); + $hostId = (int)($host['id'] ?? 0); + if ($serverUuid === '' || $hostId <= 0) { + return $host; + } + + $port = (int)($host['port'] ?? 0); + $publicHost = $this->resolveCoolifyServerHost($instance, $serverUuid, $port, $hostId); + if ($publicHost === null || $publicHost === trim((string)($host['host'] ?? ''))) { + return $host; + } + + $options = self::jsonDecode($host['options_json'] ?? null); + if ((string)($target['kind'] ?? '') === 'minio') { + $scheme = strtolower(trim((string)($options['scheme'] ?? 'http'))) ?: 'http'; + $options['endpoint'] = $scheme . '://' . $publicHost . ':' . $port; + } + + $optionsJson = self::jsonEncode($options); + $this->execute( + 'UPDATE replication_hosts SET host = ?, options_json = ? WHERE id = ?', + 'ssi', + [$publicHost, $optionsJson, $hostId] + ); + + $host['host'] = $publicHost; + $host['options_json'] = $optionsJson; + return $host; + } + + private function syncReplicationHostPortsForTarget(array $target, array $host): array + { + $serverUuid = trim((string)($target['server_uuid'] ?? '')); + $hostId = (int)($host['id'] ?? 0); + $kind = (string)($target['kind'] ?? ''); + if ($serverUuid === '' || $hostId <= 0 || (string)($host['role'] ?? '') === 'primary') { + return $host; + } + + $options = self::jsonDecode($host['options_json'] ?? null); + $port = (int)($host['port'] ?? 0); + $consolePort = $kind === 'minio' ? (int)($options['console_port'] ?? ($port + 1)) : null; + if ($port <= 0) { + return $host; + } + + $usedPorts = $this->usedPublicPortsForCoolifyServer($serverUuid, $hostId); + [$nextPort, $nextConsolePort] = $this->nextAvailablePublicPorts($kind, $port, $consolePort, $usedPorts); + if ($nextPort === $port && ($kind !== 'minio' || $nextConsolePort === $consolePort)) { + return $host; + } + + if ($kind === 'minio') { + $scheme = strtolower(trim((string)($options['scheme'] ?? 'http'))) ?: 'http'; + $options['console_port'] = $nextConsolePort; + $options['endpoint'] = $scheme . '://' . (string)$host['host'] . ':' . $nextPort; + } + + $optionsJson = self::jsonEncode($options); + $this->execute( + 'UPDATE replication_hosts SET port = ?, options_json = ? WHERE id = ?', + 'isi', + [$nextPort, $optionsJson, $hostId] + ); + + $host['port'] = $nextPort; + $host['options_json'] = $optionsJson; + return $host; + } + + private function usedPublicPortsForCoolifyServer(string $serverUuid, int $excludeHostId): array + { + $rows = $this->selectRows( + "SELECT h.port, h.options_json, t.last_reconcile_json + FROM coolify_targets t + INNER JOIN replication_hosts h ON h.id = t.replication_host_id + WHERE t.server_uuid = ? AND h.id <> ? AND t.deleted_at IS NULL AND h.deleted_at IS NULL + LIMIT 100", + 'si', + [$serverUuid, $excludeHostId] + ); + + $ports = []; + foreach ($rows as $row) { + $port = (int)($row['port'] ?? 0); + if ($port > 0) { + $ports[$port] = true; + } + $options = self::jsonDecode($row['options_json'] ?? null); + $consolePort = (int)($options['console_port'] ?? 0); + if ($consolePort > 0) { + $ports[$consolePort] = true; + } + foreach (self::coolifyApplicationPortsFromContext(self::jsonDecode($row['last_reconcile_json'] ?? null)) as $applicationPort) { + $ports[$applicationPort] = true; + } + } + + return array_keys($ports); + } + + private function nextAvailablePublicPorts(string $kind, int $port, ?int $consolePort, array $usedPorts): array + { + $used = array_fill_keys(array_map('intval', $usedPorts), true); + if ($kind !== 'minio') { + while (isset($used[$port]) && $port < 65535) { + $port++; + } + return [$port, null]; + } + + $consolePort = $consolePort !== null && $consolePort > 0 ? $consolePort : ($port + 1); + while ((isset($used[$port]) || isset($used[$consolePort])) && $consolePort < 65535) { + $port += 2; + $consolePort = $port + 1; + } + + return [$port, $consolePort]; + } + + private static function coolifyApplicationPortsFromContext(array $context): array + { + $ports = []; + $applications = $context['coolify']['applications'] ?? []; + if (!is_array($applications)) { + return []; + } + + foreach ($applications as $application) { + if (!is_array($application)) { + continue; + } + foreach (preg_split('/\s*,\s*/', (string)($application['ports'] ?? '')) ?: [] as $mapping) { + if (preg_match('/^(\d+)\s*:/', trim($mapping), $matches) === 1) { + $ports[] = (int)$matches[1]; + } + } + } + + return array_values(array_unique(array_filter($ports))); + } + + private function knownPublicHostForCoolifyServer(string $serverUuid, ?int $port = null, int $excludeHostId = 0): ?string + { + if ($serverUuid === '') { + return null; + } + + $where = 't.server_uuid = ? AND t.deleted_at IS NULL AND h.deleted_at IS NULL'; + $types = 's'; + $params = [$serverUuid]; + if ($excludeHostId > 0) { + $where .= ' AND h.id <> ?'; + $types .= 'i'; + $params[] = $excludeHostId; + } + + $rows = $this->selectRows( + "SELECT h.host, h.status, h.last_status_json + FROM coolify_targets t + INNER JOIN replication_hosts h ON h.id = t.replication_host_id + WHERE $where + ORDER BY (h.status = 'ok') DESC, h.last_checked_at DESC, h.updated_at DESC, h.id DESC + LIMIT 20", + $types, + $params + ); + + $fallback = null; + foreach ($rows as $row) { + $host = self::publicServerHostCandidate($row['host'] ?? null); + if ($host === null) { + continue; + } + $lastStatus = self::jsonDecode($row['last_status_json'] ?? null); + $isHealthy = (string)($row['status'] ?? '') === 'ok' || (string)($lastStatus['status'] ?? '') === 'ok'; + if ($fallback === null && $isHealthy) { + $fallback = $host; + } + if ($port !== null && $port > 0 && self::tcpPortIsOpen($host, $port)) { + return $host; + } + } + + return $fallback; + } + + public static function publicServerHostFromCoolifyServer(array $server, ?int $port = null, bool $includeDisplayName = true): ?string + { + $candidates = []; + foreach ([ + 'public_host', + 'publicHost', + 'public_ip', + 'publicIp', + 'public_ipv4', + 'publicIpv4', + 'public_ipv6', + 'publicIpv6', + 'address', + 'hostname', + 'fqdn', + 'domain', + 'ip', + ] as $key) { + $host = self::publicServerHostCandidate($server[$key] ?? null); + if ($host !== null && !in_array($host, $candidates, true)) { + $candidates[] = $host; + } + } + + if ($includeDisplayName) { + $host = self::publicServerHostCandidate($server['name'] ?? null); + if ($host !== null && !in_array($host, $candidates, true)) { + $candidates[] = $host; + } + } + + if ($port !== null && $port > 0) { + foreach ($candidates as $host) { + if (self::tcpPortIsOpen($host, $port)) { + return $host; + } + } + } + + return $candidates[0] ?? null; + } + + public static function publicDnsServerNameFromCoolifyServer(array $server): ?string + { + $host = self::publicServerHostCandidate($server['name'] ?? null); + if ($host === null || !self::isPublicDnsName($host)) { + return null; + } + + return $host; + } + + private static function resolvedPublicDnsServerHostFromCoolifyServer(array $server): ?string + { + $host = self::publicDnsServerNameFromCoolifyServer($server); + if ($host === null) { + return null; + } + + foreach (@gethostbynamel($host) ?: [] as $address) { + $address = self::publicServerHostCandidate($address); + if ($address !== null) { + return $address; + } + } + + return $host; + } + + private static function isPublicDnsName(string $host): bool + { + $host = strtolower(trim($host, '.')); + return str_contains($host, '.') + && preg_match('/^[a-z0-9][a-z0-9.-]*[a-z0-9]$/', $host) === 1 + && preg_match('/[a-z]/', $host) === 1 + && !str_contains($host, '..'); + } + + private static function tcpPortIsOpen(string $host, int $port): bool + { + if ($port <= 0 || $port > 65535) { + return false; + } + + $errno = 0; + $errstr = ''; + $socket = @fsockopen($host, $port, $errno, $errstr, 0.4); + if (is_resource($socket)) { + fclose($socket); + return true; + } + + return false; + } + + private static function publicServerHostCandidate(mixed $value): ?string + { + $host = trim((string)($value ?? '')); + if ($host === '') { + return null; + } + + if (str_contains($host, '://')) { + $parsed = parse_url($host, PHP_URL_HOST); + $host = is_string($parsed) ? $parsed : $host; + } + + $host = trim($host); + if (str_contains($host, '/')) { + $host = strtok($host, '/') ?: ''; + } + if (str_contains($host, ':') && substr_count($host, ':') === 1) { + $host = explode(':', $host, 2)[0]; + } + + $host = trim($host, " \t\n\r\0\x0B[]"); + if ($host === '' || preg_match('/\s/', $host) === 1 || self::isDockerLocalOrLoopbackHost($host)) { + return null; + } + + return $host; + } + + private static function isDockerLocalOrLoopbackHost(string $host): bool + { + $normalized = strtolower(trim($host, '[]')); + if (in_array($normalized, [ + 'localhost', + 'host.docker.internal', + 'host.containers.internal', + 'docker.for.win.localhost', + 'docker.for.mac.localhost', + '0.0.0.0', + '::', + '::1', + '0:0:0:0:0:0:0:1', + ], true)) { + return true; + } + + return str_starts_with($normalized, '127.') + || str_starts_with($normalized, '169.254.') + || str_starts_with($normalized, 'fe80:'); + } + + private function targetMapping(array $input, array $instance, string $key): ?string + { + $defaultKey = 'default_' . $key; + return $this->nullableString($input[$key] ?? $instance[$defaultKey] ?? null); + } + + private function nullableString(mixed $value): ?string + { + $value = trim((string)($value ?? '')); + return $value === '' ? null : $value; + } + + private function normalizeBuckets(mixed $value): array + { + if (is_array($value)) { + return array_values(array_filter(array_map('strval', $value))); + } + return array_values(array_filter(array_map('trim', preg_split('/[,\s]+/', (string)$value) ?: []))); + } + + private static function resourceName(string $kind, string $name, int $hostId): string + { + $name = strtolower(trim($name)); + $name = preg_replace('/[^a-z0-9-]+/', '-', $name) ?: ''; + $name = trim($name, '-'); + if ($name === '') { + $name = 'truckwash-' . $kind . '-replica'; + } + return substr($name . '-' . $hostId, 0, 120); + } + + private function composeHash(array $template): string + { + return hash('sha256', (string)($template['compose'] ?? '') . "\n---env---\n" . (string)($template['env'] ?? '')); + } + + private function startOperation(?int $targetId, ?int $instanceId, string $operation, ?int $actorUserId): int + { + $this->execute( + "INSERT INTO coolify_operations (target_id, instance_id, operation, status, actor_user_id) + VALUES (?, ?, ?, 'running', ?)", + 'iisi', + [$targetId, $instanceId, $operation, $actorUserId] + ); + return $this->insertId(); + } + + private function finishOperation(int $operationId, string $status, ?string $message, array $errors): void + { + $this->execute( + "UPDATE coolify_operations SET status = ?, message = ?, error_message = ?, completed_at = NOW() WHERE id = ?", + 'sssi', + [$status, $message, implode("\n", $errors), $operationId] + ); + } + + private function markTargetFailure(int $targetId, string $status, string $message, array $context = []): void + { + $payload = array_replace($context, ['message' => $message, 'status' => $status]); + $this->execute( + "UPDATE coolify_targets + SET deployment_status = ?, availability_state = 'degraded', last_reconcile_status = ?, last_reconcile_json = ?, last_reconciled_at = NOW() + WHERE id = ?", + 'sssi', + [$status, $status, self::jsonEncode($payload), $targetId] + ); + } + + private function audit(?int $targetId, ?int $instanceId, ?int $hostId, string $action, ?int $actorUserId, string $severity, array $context): void + { + $this->execute( + "INSERT INTO coolify_audit_logs (target_id, instance_id, replication_host_id, action, actor_user_id, severity, context_json) + VALUES (?, ?, ?, ?, ?, ?, ?)", + 'iiisiss', + [$targetId, $instanceId, $hostId, $action, $actorUserId, $severity, self::jsonEncode($context)] + ); + } + + private function setModuleEnabled(bool $enabled): void + { + $value = $enabled ? 'true' : 'false'; + $row = $this->selectOne("SELECT value FROM module_config WHERE module = 'Coolify' AND variable = 'enabled' LIMIT 1"); + if ($row === null) { + $this->execute("INSERT INTO module_config (module, variable, value, type) VALUES ('Coolify', 'enabled', ?, 'bool')", 's', [$value]); + return; + } + $this->execute("UPDATE module_config SET value = ? WHERE module = 'Coolify' AND variable = 'enabled'", 's', [$value]); + } + + private function ensureFailoverEnabled(string $kind): void + { + $this->setModuleConfigValue('Failover', 'enabled', 'true', 'bool'); + $this->setModuleConfigValue('Failover', $kind . '_enabled', 'true', 'bool'); + } + + private function setModuleConfigValue(string $module, string $variable, string $value, string $type): void + { + $row = $this->selectOne( + 'SELECT value FROM module_config WHERE module = ? AND variable = ? LIMIT 1', + 'ss', + [$module, $variable] + ); + if ($row === null) { + $this->execute( + 'INSERT INTO module_config (module, variable, value, type) VALUES (?, ?, ?, ?)', + 'ssss', + [$module, $variable, $value, $type] + ); + return; + } + $this->execute( + 'UPDATE module_config SET value = ?, type = ? WHERE module = ? AND variable = ?', + 'ssss', + [$value, $type, $module, $variable] + ); + } + + private function selectOne(string $sql, string $types = '', array $params = []): ?array + { + $rows = $this->selectRows($sql, $types, $params); + return $rows[0] ?? null; + } + + private function selectRows(string $sql, string $types = '', array $params = []): array + { + global $db; + if ($types === '') { + $result = $db->query($sql); + return $result ? $result->fetch_all(MYSQLI_ASSOC) : []; + } + + $stmt = $db->prepare($sql); + if ($stmt === false) { + throw new RuntimeException('Could not prepare Coolify query.'); + } + $stmt->bind_param($types, ...$params); + $stmt->execute(); + $result = $stmt->get_result(); + return $result ? $result->fetch_all(MYSQLI_ASSOC) : []; + } + + private function execute(string $sql, string $types = '', array $params = []): void + { + global $db; + if ($types === '') { + $db->query($sql); + return; + } + + $stmt = $db->prepare($sql); + if ($stmt === false) { + throw new RuntimeException('Could not prepare Coolify statement.'); + } + $stmt->bind_param($types, ...$params); + $stmt->execute(); + } + + private function insertId(): int + { + global $db; + return (int)$db->insert_id(); + } + + private function toBool(mixed $value, bool $default): bool + { + if (is_bool($value)) { + return $value; + } + if ($value === null) { + return $default; + } + $normalized = strtolower(trim((string)$value)); + if (in_array($normalized, ['1', 'true', 'yes', 'on'], true)) { + return true; + } + if (in_array($normalized, ['0', 'false', 'no', 'off'], true)) { + return false; + } + return $default; + } + + private static function hostHasCoolifyMetadata(array $host): bool + { + $options = isset($host['options']) && is_array($host['options']) + ? $host['options'] + : self::jsonDecode($host['options_json'] ?? null); + + return (string)($options['deployment_provider'] ?? '') === 'coolify' + || isset($options['coolify_target_id']) + || isset($options['coolify_instance_id']); + } + + private static function redactCoolifyResponse(array $response): array + { + foreach (['token', 'api_token', 'password', 'secret', 'real_value'] as $key) { + if (array_key_exists($key, $response)) { + $response[$key] = '[redacted]'; + } + } + return $response; + } + + private static function jsonEncode(mixed $value): string + { + $json = json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($json === false) { + throw new RuntimeException('Could not encode Coolify JSON payload.'); + } + return $json; + } + + private static function jsonDecode(mixed $value): array + { + if (!is_string($value) || trim($value) === '') { + return []; + } + $decoded = json_decode($value, true); + return is_array($decoded) ? $decoded : []; + } + + private static function isListArray(array $value): bool + { + if ($value === []) { + return true; + } + + return array_keys($value) === range(0, count($value) - 1); + } +} diff --git a/services/nginx/app/classes/coolify_schema_bootstrap.php b/services/nginx/app/classes/coolify_schema_bootstrap.php new file mode 100644 index 00000000..633d6f35 --- /dev/null +++ b/services/nginx/app/classes/coolify_schema_bootstrap.php @@ -0,0 +1,236 @@ +query($sql); + } + + self::ensureColumn('coolify_instances', 'default_destination_uuid', 'VARCHAR(128) NULL'); + self::ensureColumn('coolify_targets', 'availability_state', "VARCHAR(32) NOT NULL DEFAULT 'degraded'"); + self::ensureColumn('coolify_targets', 'desired_compose_hash', 'CHAR(64) NULL'); + self::ensureColumn('coolify_targets', 'last_reconcile_json', 'LONGTEXT NULL'); + self::ensureColumn('coolify_operations', 'guarded', 'TINYINT(1) NOT NULL DEFAULT 1'); + self::ensureColumn('coolify_instance_gateways', 'last_reconciled_at', 'DATETIME NULL'); + + self::ensureModuleConfigDefault('Coolify', 'enabled', 'false', 'bool'); + self::ensureModuleConfigDefault('Coolify', 'lb_automation_enabled', 'false', 'bool'); + self::ensureModuleConfigDefault('Coolify', 'lb_automation_mode', 'report_only', 'string'); + self::ensureModuleConfigDefault('Coolify', 'hetzner_load_balancer_id', '', 'string'); + self::ensureModuleConfigDefault('Coolify', 'hetzner_cloud_api_token', '', 'string'); + self::ensureModuleConfigDefault('Coolify', 'public_gateway_host', 'api-v2.truckwash.io', 'string'); + + self::ensureDefaultGateway('node1.truckwash.io', '94.130.142.41', 10); + self::ensureDefaultGateway('node2.truckwash.io', '65.21.214.30', 20); + self::ensureDefaultGateway('node3.truckwash.io', '23.88.23.183', 30); + + self::$initialized = true; + self::$tablesExist = true; + } + + public static function tablesExist(): bool + { + if (self::$tablesExist !== null) { + return self::$tablesExist; + } + + global $db; + + foreach (['coolify_instances', 'coolify_targets', 'coolify_operations', 'coolify_audit_logs', 'coolify_instance_gateways'] as $table) { + $tableSql = $db->escape_string($table); + $result = $db->query("SHOW TABLES LIKE '$tableSql'"); + if ($result === false || $result->num_rows === 0) { + self::$tablesExist = false; + return false; + } + } + + self::$tablesExist = true; + return self::$tablesExist; + } + + private static function ensureColumn(string $table, string $column, string $definition): void + { + global $db; + + $table = preg_replace('/[^a-zA-Z0-9_]/', '', $table); + $column = preg_replace('/[^a-zA-Z0-9_]/', '', $column); + if ($table === '' || $column === '') { + return; + } + + $result = $db->query("SHOW COLUMNS FROM `$table` LIKE '$column'"); + if ($result !== false && $result->num_rows > 0) { + return; + } + + $db->query("ALTER TABLE `$table` ADD COLUMN `$column` $definition"); + } + + private static function ensureModuleConfigDefault(string $module, string $variable, string $value, string $type): void + { + global $db; + + $moduleSql = $db->escape_string($module); + $variableSql = $db->escape_string($variable); + $result = $db->query("SELECT value FROM module_config WHERE module = '$moduleSql' AND variable = '$variableSql' LIMIT 1"); + if ($result !== false && $result->num_rows > 0) { + return; + } + + $valueSql = $db->escape_string($value); + $typeSql = $db->escape_string($type); + $db->query("INSERT INTO module_config (module, variable, value, type) VALUES ('$moduleSql', '$variableSql', '$valueSql', '$typeSql')"); + } + + private static function ensureDefaultGateway(string $hostname, string $targetIp, int $priority): void + { + global $db; + + $targetIpSql = $db->escape_string($targetIp); + $result = $db->query("SELECT id FROM coolify_instance_gateways WHERE target_ip = '$targetIpSql' LIMIT 1"); + if ($result !== false && $result->num_rows > 0) { + return; + } + + $hostnameSql = $db->escape_string($hostname); + $db->query( + "INSERT INTO coolify_instance_gateways (hostname, target_ip, enabled, priority) + VALUES ('$hostnameSql', '$targetIpSql', 1, " . (int)$priority . ")" + ); + } +} diff --git a/services/nginx/app/classes/error_report_schema_bootstrap.php b/services/nginx/app/classes/error_report_schema_bootstrap.php new file mode 100644 index 00000000..3f2ad1f7 --- /dev/null +++ b/services/nginx/app/classes/error_report_schema_bootstrap.php @@ -0,0 +1,76 @@ +query("CREATE TABLE IF NOT EXISTS error_reports ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + status VARCHAR(16) NOT NULL DEFAULT 'open', + reporter_type VARCHAR(16) NOT NULL, + reporter_user_id INT NULL, + reporter_subuser_id INT NULL, + reporter_customer_number INT NULL, + reporter_customer_number_context INT NULL, + reporter_name VARCHAR(255) NULL, + reporter_email VARCHAR(255) NULL, + route_path VARCHAR(512) NULL, + page_url VARCHAR(1024) NULL, + release_trace_id VARCHAR(64) NULL, + frontend_version VARCHAR(128) NULL, + api_version VARCHAR(128) NULL, + screenshot_object_key VARCHAR(512) NOT NULL, + screenshot_mime_type VARCHAR(64) NOT NULL, + screenshot_size_bytes INT UNSIGNED NOT NULL DEFAULT 0, + before_error TEXT NOT NULL, + expected TEXT NOT NULL, + actual TEXT NOT NULL, + request_error_count INT UNSIGNED NOT NULL DEFAULT 0, + vue_error_count INT UNSIGNED NOT NULL DEFAULT 0, + request_errors_json LONGTEXT NULL, + vue_errors_json LONGTEXT NULL, + runtime_context_json LONGTEXT NULL, + data_collection_accepted TINYINT(1) NOT NULL DEFAULT 0, + data_collection_accepted_at DATETIME NOT NULL, + data_collection_policy_version VARCHAR(64) NOT NULL DEFAULT 'error-report-v1', + resolved_at DATETIME NULL, + resolved_by_user_id INT NULL, + resolution_note TEXT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_error_reports_status_created (status, created_at), + INDEX idx_error_reports_reporter_user (reporter_user_id, created_at), + INDEX idx_error_reports_reporter_subuser (reporter_subuser_id, created_at), + INDEX idx_error_reports_customer (reporter_customer_number, reporter_customer_number_context), + INDEX idx_error_reports_trace (release_trace_id), + INDEX idx_error_reports_route (route_path) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"); + + self::$initialized = true; + self::$tablesExist = true; + } + + public static function tablesExist(): bool + { + if (self::$tablesExist !== null) { + return self::$tablesExist; + } + + global $db; + + $result = $db->query("SHOW TABLES LIKE 'error_reports'"); + self::$tablesExist = $result !== false && $result->num_rows > 0; + return self::$tablesExist; + } +} diff --git a/services/nginx/app/classes/error_report_service.php b/services/nginx/app/classes/error_report_service.php new file mode 100644 index 00000000..10507955 --- /dev/null +++ b/services/nginx/app/classes/error_report_service.php @@ -0,0 +1,567 @@ +store = $store ?? new error_report_store(); + } + + public static function redactPayload(mixed $value, int $depth = 0): mixed + { + if ($depth > 8) { + return '[depth-limit]'; + } + + if (is_array($value)) { + $redacted = []; + $index = 0; + foreach ($value as $key => $item) { + $index++; + if ($index > 80) { + $redacted['[truncated]'] = 'More than 80 keys omitted.'; + break; + } + + $keyString = (string)$key; + if (preg_match('/authorization|cookie|password|passwd|secret|token|api[_-]?key|session|credential|card|cpr|ssn/i', $keyString) === 1) { + $redacted[$key] = '[redacted]'; + continue; + } + + $redacted[$key] = self::redactPayload($item, $depth + 1); + } + return $redacted; + } + + if (is_object($value)) { + return self::redactPayload((array)$value, $depth + 1); + } + + if (is_string($value) && strlen($value) > 4000) { + return substr($value, 0, 4000) . "\n... [truncated]"; + } + + return $value; + } + + public static function decodeScreenshotDataUri(string $dataUri): array + { + if (!preg_match('/^data:(image\/(?:png|jpeg|webp));base64,([a-zA-Z0-9+\/=\r\n]+)$/', trim($dataUri), $matches)) { + throw new RuntimeException('Screenshot must be a PNG, JPEG, or WebP data URI.'); + } + + $contents = base64_decode(preg_replace('/\s+/', '', $matches[2]) ?? '', true); + if ($contents === false || $contents === '') { + throw new RuntimeException('Screenshot could not be decoded.'); + } + + if (strlen($contents) > self::SCREENSHOT_MAX_BYTES) { + throw new RuntimeException('Screenshot is too large.'); + } + + return [ + 'mime_type' => $matches[1], + 'contents' => $contents, + 'size_bytes' => strlen($contents), + ]; + } + + public function createFromCurrentPrincipal(array $payload): array + { + $this->ensureSchema(); + $principal = $this->resolvePrincipal(); + $answers = $this->validatedAnswers($payload); + + if (!$this->acceptedDataCollection($payload['data_collection_accepted'] ?? null)) { + throw new RuntimeException('Data collection acceptance is required.'); + } + + $screenshot = self::decodeScreenshotDataUri((string)($payload['screenshot'] ?? '')); + $storedScreenshot = $this->store->storeScreenshot($screenshot['mime_type'], $screenshot['contents']); + $context = is_array($payload['context'] ?? null) ? $payload['context'] : []; + $requestErrors = $this->boundedArray($payload['request_errors'] ?? ($context['request_errors'] ?? []), 25); + $vueErrors = $this->boundedArray($payload['vue_errors'] ?? ($context['vue_errors'] ?? []), 25); + $runtimeContext = $this->runtimeContext($payload, $context); + + $this->execute( + "INSERT INTO error_reports ( + status, + reporter_type, + reporter_user_id, + reporter_subuser_id, + reporter_customer_number, + reporter_customer_number_context, + reporter_name, + reporter_email, + route_path, + page_url, + release_trace_id, + frontend_version, + api_version, + screenshot_object_key, + screenshot_mime_type, + screenshot_size_bytes, + before_error, + expected, + actual, + request_error_count, + vue_error_count, + request_errors_json, + vue_errors_json, + runtime_context_json, + data_collection_accepted, + data_collection_accepted_at, + data_collection_policy_version + ) VALUES ( + 'open', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, NOW(), ? + )", + 'siiiisssssssssisssiissss', + [ + $principal['type'], + $principal['user_id'], + $principal['subuser_id'], + $principal['customer_number'], + $principal['customer_number_context'], + $principal['name'], + $principal['email'], + $runtimeContext['route_path'], + $runtimeContext['page_url'], + $runtimeContext['release_trace_id'], + $runtimeContext['frontend_version'], + $runtimeContext['api_version'], + $storedScreenshot['key'], + $storedScreenshot['mime_type'], + (int)$storedScreenshot['size_bytes'], + $answers['before_error'], + $answers['expected'], + $answers['actual'], + count($requestErrors), + count($vueErrors), + $this->jsonEncodeLimited(self::redactPayload($requestErrors)), + $this->jsonEncodeLimited(self::redactPayload($vueErrors)), + $this->jsonEncodeLimited(self::redactPayload($runtimeContext)), + $runtimeContext['data_collection_policy_version'], + ] + ); + + return $this->get($this->insertId()); + } + + public function list(array $filters = []): array + { + $this->ensureSchema(); + + $where = ['1 = 1']; + $types = ''; + $params = []; + $status = $this->statusFilter($filters['status'] ?? self::STATUS_OPEN); + if ($status !== 'all') { + $where[] = 'status = ?'; + $types .= 's'; + $params[] = $status; + } + + $search = trim((string)($filters['q'] ?? $filters['search'] ?? '')); + if ($search !== '') { + $where[] = '(route_path LIKE ? OR page_url LIKE ? OR before_error LIKE ? OR actual LIKE ? OR reporter_name LIKE ? OR reporter_email LIKE ?)'; + $types .= 'ssssss'; + $like = '%' . $search . '%'; + array_push($params, $like, $like, $like, $like, $like, $like); + } + + $limit = min(200, max(1, (int)($filters['limit'] ?? 50))); + $offset = max(0, (int)($filters['offset'] ?? 0)); + $types .= 'ii'; + $params[] = $limit; + $params[] = $offset; + + $items = $this->selectRows( + "SELECT id, status, reporter_type, reporter_user_id, reporter_subuser_id, + reporter_customer_number, reporter_customer_number_context, reporter_name, reporter_email, + route_path, page_url, release_trace_id, frontend_version, api_version, + screenshot_mime_type, screenshot_size_bytes, before_error, expected, actual, + request_error_count, vue_error_count, resolved_at, resolved_by_user_id, created_at, updated_at + FROM error_reports + WHERE " . implode(' AND ', $where) . " + ORDER BY created_at DESC + LIMIT ? OFFSET ?", + $types, + $params + ); + + return [ + 'items' => array_map(fn(array $row): array => $this->publicReport($row, false), $items), + 'counts' => $this->counts(), + 'limit' => $limit, + 'offset' => $offset, + ]; + } + + public function get(int $id): array + { + $this->ensureSchema(); + $row = $this->selectOne('SELECT * FROM error_reports WHERE id = ? LIMIT 1', 'i', [$id]); + if ($row === null) { + throw new RuntimeException('Error report not found.'); + } + + return $this->publicReport($row, true); + } + + public function updateStatus(int $id, string $status, ?string $resolutionNote, ?int $actorUserId): array + { + $this->ensureSchema(); + $status = self::normalizeStatus($status); + $note = $resolutionNote !== null ? $this->trimmedString($resolutionNote, self::NOTE_MAX_LENGTH, false) : null; + + if ($status === self::STATUS_RESOLVED) { + $this->execute( + 'UPDATE error_reports SET status = ?, resolved_at = NOW(), resolved_by_user_id = ?, resolution_note = ? WHERE id = ?', + 'sisi', + [$status, $actorUserId, $note, $id] + ); + } else { + $this->execute( + 'UPDATE error_reports SET status = ?, resolved_at = NULL, resolved_by_user_id = NULL, resolution_note = ? WHERE id = ?', + 'ssi', + [$status, $note, $id] + ); + } + + return $this->get($id); + } + + public static function normalizeStatus(string $status): string + { + $status = strtolower(trim($status)); + if (!in_array($status, [self::STATUS_OPEN, self::STATUS_RESOLVED], true)) { + throw new RuntimeException('Invalid error report status.'); + } + return $status; + } + + private function validatedAnswers(array $payload): array + { + return [ + 'before_error' => $this->requiredAnswer($payload, ['before_error', 'what_were_you_doing_before_error_occurred']), + 'expected' => $this->requiredAnswer($payload, ['expected', 'what_did_you_expect_would_happen']), + 'actual' => $this->requiredAnswer($payload, ['actual', 'what_actually_happened']), + ]; + } + + private function requiredAnswer(array $payload, array $keys): string + { + foreach ($keys as $key) { + if (array_key_exists($key, $payload)) { + return $this->trimmedString((string)$payload[$key], self::ANSWER_MAX_LENGTH, true); + } + } + + throw new RuntimeException('Missing required answer.'); + } + + private function trimmedString(string $value, int $maxLength, bool $required): string + { + $value = trim($value); + if ($required && $value === '') { + throw new RuntimeException('Required text fields must not be empty.'); + } + + if (strlen($value) > $maxLength) { + return substr($value, 0, $maxLength); + } + + return $value; + } + + private function acceptedDataCollection(mixed $value): bool + { + return $value === true || $value === 1 || $value === '1' || $value === 'true'; + } + + private function runtimeContext(array $payload, array $context): array + { + return [ + 'route_path' => $this->nullableString($payload['route_path'] ?? $context['route_path'] ?? $context['route'] ?? null, 512), + 'page_url' => $this->nullableString($payload['page_url'] ?? $context['page_url'] ?? $context['url'] ?? null, 1024), + 'release_trace_id' => $this->nullableString($payload['release_trace_id'] ?? $context['release_trace_id'] ?? $context['trace_id'] ?? $this->releaseRequestContext('trace_id'), 64), + 'frontend_version' => $this->nullableString($payload['frontend_version'] ?? $context['frontend_version'] ?? $this->releaseRequestContext('frontend_version'), 128), + 'api_version' => $this->nullableString($payload['api_version'] ?? $context['api_version'] ?? $this->releaseRequestContext('backend_version'), 128), + 'viewport' => is_array($context['viewport'] ?? null) ? $context['viewport'] : null, + 'user_agent' => $this->nullableString($context['user_agent'] ?? ($_SERVER['HTTP_USER_AGENT'] ?? null), 1024), + 'captured_at' => $this->nullableString($context['captured_at'] ?? null, 64), + 'data_collection_policy_version' => $this->nullableString($payload['data_collection_policy_version'] ?? $context['data_collection_policy_version'] ?? 'error-report-v1', 64) ?? 'error-report-v1', + ]; + } + + private function releaseRequestContext(string $key): ?string + { + $context = is_array($GLOBALS['RELEASE_REQUEST_CONTEXT'] ?? null) ? $GLOBALS['RELEASE_REQUEST_CONTEXT'] : []; + return isset($context[$key]) ? (string)$context[$key] : null; + } + + private function nullableString(mixed $value, int $maxLength): ?string + { + if ($value === null) { + return null; + } + $value = trim((string)$value); + if ($value === '') { + return null; + } + return substr($value, 0, $maxLength); + } + + private function boundedArray(mixed $value, int $limit): array + { + return is_array($value) ? array_slice(array_values($value), 0, $limit) : []; + } + + private function statusFilter(mixed $status): string + { + $status = strtolower(trim((string)$status)); + if ($status === '' || $status === self::STATUS_OPEN) { + return self::STATUS_OPEN; + } + if ($status === self::STATUS_RESOLVED || $status === 'all') { + return $status; + } + return self::STATUS_OPEN; + } + + private function counts(): array + { + $rows = $this->selectRows('SELECT status, COUNT(*) AS count FROM error_reports GROUP BY status'); + $counts = [ + self::STATUS_OPEN => 0, + self::STATUS_RESOLVED => 0, + 'all' => 0, + ]; + foreach ($rows as $row) { + $status = (string)($row['status'] ?? ''); + $count = (int)($row['count'] ?? 0); + if (isset($counts[$status])) { + $counts[$status] = $count; + } + $counts['all'] += $count; + } + return $counts; + } + + private function resolvePrincipal(): array + { + $auth = new authentication(); + + try { + $subuser = $auth->get_subuser(); + if ($subuser !== false) { + return [ + 'type' => 'subuser', + 'user_id' => null, + 'subuser_id' => (int)$subuser->id, + 'customer_number' => null, + 'customer_number_context' => $this->headerInt('X-Customer-Number'), + 'name' => $this->safeObjectValue($subuser, 'name') ?: $this->safeObjectValue($subuser, 'username'), + 'email' => $this->safeObjectValue($subuser, 'email'), + ]; + } + } catch (Throwable) { + } + + try { + $user = $auth->get_user(); + if ($user !== false) { + return [ + 'type' => 'user', + 'user_id' => (int)$user->id, + 'subuser_id' => null, + 'customer_number' => isset($user->customer_number) ? (int)$user->customer_number->value() : null, + 'customer_number_context' => null, + 'name' => $this->safeObjectValue($user, 'display_name'), + 'email' => $this->safeObjectValue($user, 'email'), + ]; + } + } catch (Throwable) { + } + + throw new RuntimeException('Authentication failed. Invalid or missing token.'); + } + + private function headerInt(string $name): ?int + { + $headers = function_exists('getallheaders') ? getallheaders() : []; + foreach ($headers as $key => $value) { + if (strcasecmp((string)$key, $name) === 0) { + $int = (int)$value; + return $int > 0 ? $int : null; + } + } + $serverKey = 'HTTP_' . strtoupper(str_replace('-', '_', $name)); + $int = (int)($_SERVER[$serverKey] ?? 0); + return $int > 0 ? $int : null; + } + + private function safeObjectValue(object $object, string $property): ?string + { + try { + if (!isset($object->{$property}) || !method_exists($object->{$property}, 'value')) { + return null; + } + $value = $object->{$property}->value(); + return $value === null ? null : substr((string)$value, 0, 255); + } catch (Throwable) { + return null; + } + } + + private function publicReport(array $row, bool $includeDetail): array + { + $report = [ + 'id' => (int)$row['id'], + 'status' => (string)$row['status'], + 'reporter' => [ + 'type' => $row['reporter_type'] ?? null, + 'user_id' => isset($row['reporter_user_id']) ? (int)$row['reporter_user_id'] : null, + 'subuser_id' => isset($row['reporter_subuser_id']) ? (int)$row['reporter_subuser_id'] : null, + 'customer_number' => isset($row['reporter_customer_number']) ? (int)$row['reporter_customer_number'] : null, + 'customer_number_context' => isset($row['reporter_customer_number_context']) ? (int)$row['reporter_customer_number_context'] : null, + 'name' => $row['reporter_name'] ?? null, + 'email' => $row['reporter_email'] ?? null, + ], + 'route_path' => $row['route_path'] ?? null, + 'page_url' => $row['page_url'] ?? null, + 'release_trace_id' => $row['release_trace_id'] ?? null, + 'frontend_version' => $row['frontend_version'] ?? null, + 'api_version' => $row['api_version'] ?? null, + 'screenshot' => [ + 'mime_type' => $row['screenshot_mime_type'] ?? null, + 'size_bytes' => isset($row['screenshot_size_bytes']) ? (int)$row['screenshot_size_bytes'] : 0, + ], + 'answers' => [ + 'before_error' => $row['before_error'] ?? '', + 'expected' => $row['expected'] ?? '', + 'actual' => $row['actual'] ?? '', + ], + 'request_error_count' => isset($row['request_error_count']) ? (int)$row['request_error_count'] : 0, + 'vue_error_count' => isset($row['vue_error_count']) ? (int)$row['vue_error_count'] : 0, + 'resolved_at' => $row['resolved_at'] ?? null, + 'resolved_by_user_id' => isset($row['resolved_by_user_id']) ? (int)$row['resolved_by_user_id'] : null, + 'created_at' => $row['created_at'] ?? null, + 'updated_at' => $row['updated_at'] ?? null, + ]; + + if ($includeDetail) { + $report['screenshot']['url'] = $this->store->screenshotUrl((string)($row['screenshot_object_key'] ?? '')); + $report['screenshot']['object_key'] = $row['screenshot_object_key'] ?? null; + $report['request_errors'] = $this->jsonDecode($row['request_errors_json'] ?? null); + $report['vue_errors'] = $this->jsonDecode($row['vue_errors_json'] ?? null); + $report['runtime_context'] = $this->jsonDecode($row['runtime_context_json'] ?? null); + $report['data_collection'] = [ + 'accepted' => (bool)($row['data_collection_accepted'] ?? false), + 'accepted_at' => $row['data_collection_accepted_at'] ?? null, + 'policy_version' => $row['data_collection_policy_version'] ?? null, + ]; + $report['resolution_note'] = $row['resolution_note'] ?? null; + } + + return $report; + } + + private function ensureSchema(): void + { + if ($this->schemaEnsured) { + return; + } + error_report_schema_bootstrap::ensureTables(); + $this->schemaEnsured = true; + } + + private function selectOne(string $sql, string $types = '', array $params = []): ?array + { + $rows = $this->selectRows($sql, $types, $params); + return $rows[0] ?? null; + } + + private function selectRows(string $sql, string $types = '', array $params = []): array + { + global $db; + if ($types === '') { + $result = $db->query($sql); + return $result ? $result->fetch_all(MYSQLI_ASSOC) : []; + } + + $stmt = $db->prepare($sql); + if ($stmt === false) { + throw new RuntimeException('Could not prepare error report query.'); + } + $stmt->bind_param($types, ...$params); + $stmt->execute(); + $result = $stmt->get_result(); + return $result ? $result->fetch_all(MYSQLI_ASSOC) : []; + } + + private function execute(string $sql, string $types = '', array $params = []): void + { + global $db; + if ($types === '') { + $db->query($sql); + return; + } + + $stmt = $db->prepare($sql); + if ($stmt === false) { + throw new RuntimeException('Could not prepare error report statement.'); + } + $stmt->bind_param($types, ...$params); + $stmt->execute(); + } + + private function insertId(): int + { + global $db; + return (int)$db->insert_id(); + } + + private function jsonEncodeLimited(mixed $value): string + { + $json = json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($json === false) { + throw new RuntimeException('Could not encode error report JSON payload.'); + } + if (strlen($json) <= self::JSON_MAX_LENGTH) { + return $json; + } + + $truncated = [ + '[truncated]' => 'Payload exceeded ' . self::JSON_MAX_LENGTH . ' bytes.', + 'preview' => substr($json, 0, self::JSON_MAX_LENGTH), + ]; + $encoded = json_encode($truncated, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + return $encoded === false ? '{}' : $encoded; + } + + private function jsonDecode(mixed $value): array + { + if (!is_string($value) || trim($value) === '') { + return []; + } + $decoded = json_decode($value, true); + return is_array($decoded) ? $decoded : []; + } +} diff --git a/services/nginx/app/classes/error_report_store.php b/services/nginx/app/classes/error_report_store.php new file mode 100644 index 00000000..7dc93fc0 --- /dev/null +++ b/services/nginx/app/classes/error_report_store.php @@ -0,0 +1,47 @@ + 'webp', + 'image/jpeg' => 'jpg', + default => 'png', + }; + + $datePath = date('Y/m'); + $key = sprintf('error-reports/%s/%s.%s', $datePath, bin2hex(random_bytes(16)), $extension); + + if (!self::createObject($key, $contents)) { + throw new \RuntimeException('Could not store error report screenshot.'); + } + + return [ + 'key' => $key, + 'mime_type' => $mimeType, + 'size_bytes' => strlen($contents), + ]; + } + + public function screenshotUrl(string $key): ?string + { + $key = trim($key); + if ($key === '') { + return null; + } + + return self::getPresignedUrl($key, 1200, false); + } +} diff --git a/services/nginx/app/classes/failover.php b/services/nginx/app/classes/failover.php new file mode 100644 index 00000000..65f7c14f --- /dev/null +++ b/services/nginx/app/classes/failover.php @@ -0,0 +1,17 @@ +config = new failover_c(); + } +} diff --git a/services/nginx/app/classes/hetzner_cloud_client.php b/services/nginx/app/classes/hetzner_cloud_client.php new file mode 100644 index 00000000..9698c2d8 --- /dev/null +++ b/services/nginx/app/classes/hetzner_cloud_client.php @@ -0,0 +1,123 @@ +statusCode; + } + + public function apiCode(): string + { + return $this->apiCode; + } +} + +class hetzner_cloud_client +{ + private const BASE_URL = 'https://api.hetzner.cloud/v1'; + + public function __construct(private readonly string $token, private readonly int $timeoutSeconds = 8) + { + if (trim($token) === '') { + throw new RuntimeException('Hetzner Cloud API token is required.'); + } + } + + public function getLoadBalancer(int|string $id): array + { + return $this->request('GET', '/load_balancers/' . rawurlencode((string)$id))['load_balancer'] ?? []; + } + + public function addIpTarget(int|string $loadBalancerId, string $ip): array + { + return $this->request('POST', '/load_balancers/' . rawurlencode((string)$loadBalancerId) . '/actions/add_target', [ + 'type' => 'ip', + 'ip' => ['ip' => $ip], + ]); + } + + public function removeIpTarget(int|string $loadBalancerId, string $ip): array + { + return $this->request('POST', '/load_balancers/' . rawurlencode((string)$loadBalancerId) . '/actions/remove_target', [ + 'type' => 'ip', + 'ip' => ['ip' => $ip], + ]); + } + + public function addService(int|string $loadBalancerId, string $protocol, int $listenPort, int $destinationPort): array + { + return $this->request('POST', '/load_balancers/' . rawurlencode((string)$loadBalancerId) . '/actions/add_service', [ + 'protocol' => strtolower($protocol), + 'listen_port' => $listenPort, + 'destination_port' => $destinationPort, + 'proxyprotocol' => false, + ]); + } + + private function request(string $method, string $path, ?array $payload = null): array + { + $curl = curl_init(self::BASE_URL . $path); + if ($curl === false) { + throw new RuntimeException('Could not initialize Hetzner Cloud API request.'); + } + + $headers = [ + 'Accept: application/json', + 'Authorization: Bearer ' . trim($this->token), + ]; + + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_CUSTOMREQUEST, strtoupper($method)); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, min(3, $this->timeoutSeconds)); + curl_setopt($curl, CURLOPT_TIMEOUT, max(1, $this->timeoutSeconds)); + curl_setopt($curl, CURLOPT_NOSIGNAL, true); + curl_setopt($curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + + if ($payload !== null) { + $body = json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($body === false) { + throw new RuntimeException('Could not encode Hetzner Cloud API payload.'); + } + $headers[] = 'Content-Type: application/json'; + curl_setopt($curl, CURLOPT_POSTFIELDS, $body); + } + + curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); + + $raw = curl_exec($curl); + $error = curl_error($curl); + $status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE); + curl_close($curl); + + if ($raw === false) { + throw new RuntimeException('Hetzner Cloud API request failed: ' . $error); + } + + $decoded = trim((string)$raw) === '' ? [] : json_decode((string)$raw, true); + if (!is_array($decoded)) { + $decoded = ['raw' => (string)$raw]; + } + + if ($status < 200 || $status >= 300) { + $errorPayload = is_array($decoded['error'] ?? null) ? $decoded['error'] : []; + $apiCode = (string)($errorPayload['code'] ?? $decoded['code'] ?? ''); + $message = (string)($errorPayload['message'] ?? $decoded['message'] ?? ('HTTP ' . $status)); + throw new hetzner_cloud_api_exception('Hetzner Cloud API request failed: ' . $message, $status, $apiCode); + } + + return $decoded; + } +} diff --git a/services/nginx/app/classes/release_manager.php b/services/nginx/app/classes/release_manager.php new file mode 100644 index 00000000..400e6ec5 --- /dev/null +++ b/services/nginx/app/classes/release_manager.php @@ -0,0 +1,3963 @@ + bin2hex(random_bytes(16)), + 'requested_channel' => '', + 'frontend_version' => '', + 'backend_version' => self::backendVersion(), + 'request_started_at' => date('c'), + ]; + + $GLOBALS['RELEASE_REQUEST_CONTEXT'] = $context; + return $context; + } + + public static function moduleKeys(): array + { + return self::MODULE_KEYS; + } + + public static function backendVersion(): string + { + foreach (['RELEASE_VERSION', 'GITHUB_SHA', 'COMMIT_SHA', 'VITE_COMMIT_HASH'] as $key) { + $value = trim((string)(getenv($key) ?: ($_SERVER[$key] ?? ''))); + if ($value !== '') { + return self::safeIdentifier($value, 128); + } + } + return 'unknown'; + } + + public static function verifyGithubSignature(string $secret, string $payload, string $signatureHeader): bool + { + $secret = trim($secret); + $signatureHeader = trim($signatureHeader); + if ($secret === '' || $signatureHeader === '' || !str_starts_with($signatureHeader, 'sha256=')) { + return false; + } + + $expected = 'sha256=' . hash_hmac('sha256', $payload, $secret); + return hash_equals($expected, $signatureHeader); + } + + public static function normalizeGithubRepositoryName(string $value): string + { + $repository = trim($value); + if ($repository === '') { + return ''; + } + + if (preg_match('#^git@github\.com:(.+)$#i', $repository, $matches) === 1) { + $repository = $matches[1]; + } elseif (preg_match('#^https?://#i', $repository) === 1) { + $path = parse_url($repository, PHP_URL_PATH); + $repository = is_string($path) ? ltrim($path, '/') : $repository; + } else { + $repository = preg_replace('#^github\.com/#i', '', $repository) ?? $repository; + } + + $repository = preg_replace('#\.git$#i', '', $repository) ?? $repository; + $repository = trim($repository, "/ \t\n\r\0\x0B"); + return preg_match('/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/', $repository) === 1 ? $repository : ''; + } + + public static function redactPayload(mixed $value, int $depth = 0): mixed + { + if ($depth > 8) { + return '[depth-limit]'; + } + + if (is_array($value)) { + $redacted = []; + $index = 0; + foreach ($value as $key => $item) { + $index++; + if ($index > 80) { + $redacted['[truncated]'] = 'More than 80 keys omitted.'; + break; + } + + $keyString = (string)$key; + if (self::isSensitiveKey($keyString)) { + $redacted[$key] = '[redacted]'; + continue; + } + $redacted[$key] = self::redactPayload($item, $depth + 1); + } + return $redacted; + } + + if (is_object($value)) { + return self::redactPayload((array)$value, $depth + 1); + } + + if (is_string($value)) { + if (strlen($value) > 4000) { + return substr($value, 0, 4000) . "\n... [truncated]"; + } + return $value; + } + + return $value; + } + + public static function deploymentCanBePromoted(string $status): bool + { + return in_array(strtolower(trim($status)), ['deployed'], true); + } + + public static function deploymentPromotionBlockedReason(array $deployment): string + { + $status = strtolower(trim((string)($deployment['status'] ?? 'unknown'))) ?: 'unknown'; + $result = self::jsonDecode($deployment['result_json'] ?? null); + if ($result === [] && is_array($deployment['result'] ?? null)) { + $result = $deployment['result']; + } + $failure = is_array($result['failure_summary'] ?? null) ? $result['failure_summary'] : []; + $rootCause = trim((string)($failure['root_cause'] ?? $deployment['error_message'] ?? '')); + + if ($status === 'active') { + return 'Deployment is already active.'; + } + + if ($status === 'failed') { + return 'Deployment failed and cannot be promoted.' . ($rootCause !== '' ? ' Cause: ' . $rootCause : ''); + } + + return 'Only successfully deployed release deployments can be promoted. Current status: ' . $status . '.'; + } + + public static function deploymentFailureSummary(Throwable $throwable, array $context = []): array + { + $message = trim($throwable->getMessage()) ?: 'Deployment failed without an error message.'; + $normalized = strtolower($message); + $category = 'unknown'; + $stage = trim((string)($context['stage'] ?? 'deployment')) ?: 'deployment'; + $nextAction = 'Open the Coolify deployment logs for the service and compare the failing commit with the last successful deployment.'; + + if (str_contains($normalized, 'github repository access') || str_contains($normalized, 'github api')) { + $category = 'github_access'; + $stage = 'source_access'; + $nextAction = 'Verify the Release Manager GitHub token, repository, branch, and selected commit before deploying again.'; + } elseif ( + str_contains($normalized, 'coolify instance') + || str_contains($normalized, 'base url') + || str_contains($normalized, 'api token') + ) { + $category = 'coolify_connection'; + $stage = 'provider_connection'; + $nextAction = 'Test the configured Coolify instance and API token from Release Manager settings.'; + } elseif ( + str_contains($normalized, 'service uuid') + || str_contains($normalized, 'select an existing coolify service') + || str_contains($normalized, 'http 404') + ) { + $category = 'coolify_target'; + $stage = 'provider_target'; + $nextAction = 'Check that the saved deployment target points at the correct Coolify service UUID and instance.'; + } elseif ( + str_contains($normalized, 'docker_compose') + || str_contains($normalized, 'image/repository') + || str_contains($normalized, 'validation') + ) { + $category = 'configuration'; + $stage = 'provider_configuration'; + $nextAction = 'Review the deployment target deploy context, image, compose payload, and required environment variables.'; + } elseif (str_contains($normalized, 'health') || str_contains($normalized, 'smoke')) { + $category = 'smoke_test'; + $stage = 'post_deploy_smoke_test'; + $nextAction = 'Check container startup logs and the configured health URL before promoting the deployment.'; + } elseif ( + str_contains($normalized, 'timeout') + || str_contains($normalized, 'timed out') + || str_contains($normalized, 'could not connect') + || str_contains($normalized, 'network') + ) { + $category = 'network'; + $stage = 'provider_connection'; + $nextAction = 'Check network access from the API container to GitHub and Coolify, then retry the deployment.'; + } + + $evidence = array_filter([ + 'message' => $message, + 'app' => $context['app'] ?? null, + 'repository' => $context['repository'] ?? null, + 'branch' => $context['branch'] ?? null, + 'commit_sha' => $context['commit_sha'] ?? null, + 'target_id' => $context['target_id'] ?? null, + 'coolify_instance_id' => $context['coolify_instance_id'] ?? null, + 'coolify_service_uuid' => $context['coolify_service_uuid'] ?? null, + ], static fn(mixed $value): bool => $value !== null && $value !== ''); + + return [ + 'category' => $category, + 'stage' => $stage, + 'root_cause' => $message, + 'next_action' => $nextAction, + 'promotion_blocked' => true, + 'captured_at' => date('c'), + 'evidence' => self::redactPayload($evidence), + ]; + } + + public static function recordBackendFailure(bool $success, mixed $data, ?int $status): void + { + if ($success || ($status !== null && $status < 400)) { + return; + } + + $uri = (string)($_SERVER['REQUEST_URI'] ?? ''); + if (str_starts_with($uri, '/release/timeline/events')) { + return; + } + + try { + if (!isset($GLOBALS['db']) || !release_manager_schema_bootstrap::tablesExist()) { + return; + } + + $context = is_array($GLOBALS['RELEASE_REQUEST_CONTEXT'] ?? null) + ? $GLOBALS['RELEASE_REQUEST_CONTEXT'] + : self::initializeRequestContext(); + + $manager = new self(); + $principalContext = $manager->currentPrincipalContext(); + $channel = $manager->resolveChannel($principalContext); + $manager->ingestTimelineEvents([ + [ + 'type' => 'backend_response_failed', + 'severity' => ($status ?? 500) >= 500 ? 'error' : 'warning', + 'module_key' => $manager->inferModuleKeyFromUri($uri), + 'route' => explode('?', $uri)[0] ?: '/', + 'occurred_at' => date('c'), + 'payload' => [ + 'status' => $status, + 'method' => $_SERVER['REQUEST_METHOD'] ?? 'GET', + 'response' => $data, + ], + ], + ], [ + 'trace_id' => $context['trace_id'] ?? '', + 'channel_slug' => $channel['slug'] ?? '', + 'principal_type' => $principalContext['principal_type'] ?? null, + 'principal_id' => $principalContext['principal_id'] ?? null, + 'customer_number' => $principalContext['customer_number'] ?? null, + ], false); + } catch (Throwable) { + // Release telemetry must never block API responses. + } + } + + public function bootstrap(): array + { + $this->ensureSchema(); + $channel = $this->defaultChannel(); + $versions = $this->currentVersionsForChannel((int)$channel['id']); + + return [ + 'generated_at' => date('c'), + 'trace_id' => $this->requestTraceId(), + 'channel' => $this->publicChannel($channel), + 'versions' => $versions, + 'api_base_url' => $channel['api_base_url'] ?? null, + 'frontend_base_url' => $channel['frontend_base_url'] ?? null, + 'availability' => $this->channelAvailability($channel), + 'capture_policy' => [ + 'enabled' => false, + 'capture_level' => 'metadata', + 'all_failure_metadata' => true, + 'retention_days' => (int)($channel['retention_days'] ?? 14), + ], + ]; + } + + public function runtimeForPayload(array $payload): array + { + $this->ensureSchema(); + + $context = [ + 'principal_type' => 'user', + 'principal_id' => isset($payload['id']) ? (string)$payload['id'] : null, + 'customer_number' => isset($payload['customer_number']) ? (int)$payload['customer_number'] : null, + ]; + + $channel = $this->resolveChannel($context); + $versions = $this->currentVersionsForChannel((int)$channel['id']); + $capturePolicy = $this->capturePolicyFor($context, $channel); + + return [ + 'generated_at' => date('c'), + 'trace_id' => $this->requestTraceId(), + 'channel' => $this->publicChannel($channel), + 'versions' => $versions, + 'api_base_url' => $channel['api_base_url'] ?? null, + 'frontend_base_url' => $channel['frontend_base_url'] ?? null, + 'availability' => $this->channelAvailability($channel), + 'capture_policy' => $capturePolicy, + 'module_keys' => self::MODULE_KEYS, + ]; + } + + public function runtimeForCurrentPrincipal(): array + { + $this->ensureSchema(); + $context = $this->currentPrincipalContext(); + $channel = $this->resolveChannel($context); + + return [ + 'generated_at' => date('c'), + 'trace_id' => $this->requestTraceId(), + 'channel' => $this->publicChannel($channel), + 'versions' => $this->currentVersionsForChannel((int)$channel['id']), + 'api_base_url' => $channel['api_base_url'] ?? null, + 'frontend_base_url' => $channel['frontend_base_url'] ?? null, + 'availability' => $this->channelAvailability($channel), + 'capture_policy' => $this->capturePolicyFor($context, $channel), + 'module_keys' => self::MODULE_KEYS, + ]; + } + + public function summary(): array + { + $this->ensureSchema(); + + return [ + 'generated_at' => date('c'), + 'channels' => $this->listChannels(), + 'assignments' => $this->listAssignments(), + 'deployment_targets' => $this->listDeploymentTargets(), + 'service_sets' => $this->listServiceSets(), + 'bundles' => $this->listBundles(25), + 'deployments' => $this->listDeployments(25), + 'timeline' => $this->timelineSummary(), + 'module_health' => $this->latestModuleHealth(), + 'module_keys' => self::MODULE_KEYS, + 'suggestions' => $this->releaseSuggestions(), + ]; + } + + public function suggestions(): array + { + $this->ensureSchema(); + return $this->releaseSuggestions(); + } + + public function releaseConfig(): array + { + $this->ensureSchema(); + $storedToken = trim((string)$this->moduleConfigValue('ReleaseManager', 'github_token', '')); + $storedWebhookSecret = trim((string)$this->moduleConfigValue('ReleaseManager', 'github_webhook_secret', '')); + + return [ + 'github_token_configured' => $this->hasGithubApiToken(), + 'github_token_env_configured' => $this->githubEnvToken() !== '', + 'github_token_module_configured' => $storedToken !== '', + 'github_token_variable' => 'ReleaseManager.github_token', + 'github_token_env_variable' => 'RELEASE_MANAGER_GITHUB_TOKEN', + 'github_api_url' => $this->githubApiBaseUrl(), + 'github_api_url_variable' => 'ReleaseManager.github_api_url', + 'github_webhook_secret_configured' => $storedWebhookSecret !== '', + 'github_webhook_secret_variable' => 'ReleaseManager.github_webhook_secret', + ]; + } + + public function updateReleaseConfig(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $updated = []; + + if (array_key_exists('github_api_url', $input)) { + $apiUrl = rtrim(trim((string)$input['github_api_url']), '/'); + if ($apiUrl === '') { + $apiUrl = 'https://api.github.com'; + } + if (preg_match('#^https?://#i', $apiUrl) !== 1) { + throw new RuntimeException('GitHub API URL must start with http:// or https://.'); + } + $this->upsertModuleConfigValue('ReleaseManager', 'github_api_url', $apiUrl, 'string'); + $updated[] = 'github_api_url'; + } + + if (array_key_exists('github_token', $input)) { + $token = trim((string)$input['github_token']); + if ($token !== '' && $token !== '[redacted]' && !str_starts_with($token, 'twsec:v1:') && class_exists(replication_secret_box::class)) { + $token = replication_secret_box::encrypt($token); + } + if ($token !== '' && $token !== '[redacted]') { + $this->upsertModuleConfigValue('ReleaseManager', 'github_token', $token, 'string'); + $updated[] = 'github_token'; + } + } + + if ($this->toBool($input['clear_github_token'] ?? false)) { + $this->upsertModuleConfigValue('ReleaseManager', 'github_token', '', 'string'); + $updated[] = 'github_token'; + } + + if (array_key_exists('github_webhook_secret', $input)) { + $secret = trim((string)$input['github_webhook_secret']); + if ($secret !== '' && $secret !== '[redacted]') { + $this->upsertModuleConfigValue('ReleaseManager', 'github_webhook_secret', $secret, 'string'); + $updated[] = 'github_webhook_secret'; + } + } + + $this->audit(null, null, 'release_config_updated', $actorUserId, 'info', [ + 'updated' => array_values(array_unique($updated)), + ]); + + return $this->releaseConfig(); + } + + public function listGithubRepositories(array $filters = []): array + { + $this->ensureSchema(); + if (!$this->hasGithubApiToken()) { + return $this->githubTokenMissingResponse(); + } + + $query = strtolower(trim((string)($filters['query'] ?? $filters['search'] ?? ''))); + $repositories = []; + for ($page = 1; $page <= 5; $page++) { + $rows = $this->githubRequest('GET', '/user/repos', [ + 'visibility' => 'all', + 'affiliation' => 'owner,collaborator,organization_member', + 'sort' => 'updated', + 'direction' => 'desc', + 'per_page' => 100, + 'page' => $page, + ]); + if (!is_array($rows)) { + break; + } + + foreach ($rows as $row) { + if (!is_array($row)) { + continue; + } + $repository = $this->publicGithubRepository($row); + if ($query !== '') { + $haystack = strtolower(($repository['full_name'] ?? '') . ' ' . ($repository['description'] ?? '')); + if (!str_contains($haystack, $query)) { + continue; + } + } + $repositories[$repository['full_name']] = $repository; + } + + if (count($rows) < 100) { + break; + } + } + + return [ + 'token_configured' => true, + 'github_api_url' => $this->githubApiBaseUrl(), + 'repositories' => array_values($repositories), + ]; + } + + public function listGithubBranches(array $input): array + { + $this->ensureSchema(); + $repository = self::normalizeGithubRepositoryName((string)($input['repository'] ?? '')); + if ($repository === '') { + throw new RuntimeException('GitHub repository must use owner/repo format.'); + } + if (!$this->hasGithubApiToken()) { + return $this->githubTokenMissingResponse($repository); + } + + $branches = []; + for ($page = 1; $page <= 5; $page++) { + $rows = $this->githubRequest('GET', '/repos/' . $this->githubRepositoryPath($repository) . '/branches', [ + 'per_page' => 100, + 'page' => $page, + ]); + if (!is_array($rows)) { + break; + } + + foreach ($rows as $row) { + if (is_array($row)) { + $branch = $this->publicGithubBranch($row); + $branches[$branch['name']] = $branch; + } + } + + if (count($rows) < 100) { + break; + } + } + + return [ + 'token_configured' => true, + 'repository' => $repository, + 'branches' => array_values($branches), + ]; + } + + public function listGithubCommits(array $input): array + { + $this->ensureSchema(); + $repository = self::normalizeGithubRepositoryName((string)($input['repository'] ?? '')); + if ($repository === '') { + throw new RuntimeException('GitHub repository must use owner/repo format.'); + } + if (!$this->hasGithubApiToken()) { + return $this->githubTokenMissingResponse($repository, (string)($input['branch'] ?? '')); + } + + $branch = trim((string)($input['branch'] ?? 'main')) ?: 'main'; + $commit = trim((string)($input['commit_sha'] ?? $input['commit'] ?? '')); + $query = strtolower(trim((string)($input['query'] ?? $input['search'] ?? ''))); + + if ($commit !== '' && !in_array(strtolower($commit), ['latest', 'head'], true)) { + $row = $this->githubRequest('GET', '/repos/' . $this->githubRepositoryPath($repository) . '/commits/' . rawurlencode($commit)); + $publicCommit = is_array($row) ? $this->publicGithubCommit($row) : []; + return [ + 'token_configured' => true, + 'repository' => $repository, + 'branch' => $branch, + 'commits' => $publicCommit !== [] ? [$publicCommit] : [], + 'latest' => $publicCommit !== [] ? $publicCommit : null, + ]; + } + + $rows = $this->githubRequest('GET', '/repos/' . $this->githubRepositoryPath($repository) . '/commits', [ + 'sha' => $branch, + 'per_page' => 25, + ]); + $commits = []; + foreach (is_array($rows) ? $rows : [] as $row) { + if (!is_array($row)) { + continue; + } + $publicCommit = $this->publicGithubCommit($row); + if ($query !== '') { + $haystack = strtolower(($publicCommit['sha'] ?? '') . ' ' . ($publicCommit['message'] ?? '') . ' ' . ($publicCommit['author_name'] ?? '')); + if (!str_contains($haystack, $query)) { + continue; + } + } + $commits[] = $publicCommit; + } + + return [ + 'token_configured' => true, + 'repository' => $repository, + 'branch' => $branch, + 'commits' => $commits, + 'latest' => $commits[0] ?? null, + ]; + } + + public function testGithubRepositoryAccess(array $input): array + { + $this->ensureSchema(); + return $this->githubRepositoryAccess($input); + } + + public function listChannels(): array + { + if (!release_manager_schema_bootstrap::tablesExist()) { + return []; + } + + $channels = $this->selectRows( + "SELECT * FROM release_channels WHERE deleted_at IS NULL ORDER BY default_channel DESC, slug" + ); + + return array_map(function (array $channel): array { + $public = $this->publicChannel($channel); + $public['versions'] = $this->currentVersionsForChannel((int)$channel['id']); + return $public; + }, $channels); + } + + public function createChannel(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $normalized = $this->normalizeChannelInput($input, true); + + $this->execute( + "INSERT INTO release_channels ( + slug, name, description, enabled, default_channel, rollout_percent, + frontend_base_url, api_base_url, replay_enabled, capture_level, retention_days, metadata_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + 'sssiidssisis', + [ + $normalized['slug'], + $normalized['name'], + $normalized['description'], + $normalized['enabled'], + $normalized['default_channel'], + $normalized['rollout_percent'], + $normalized['frontend_base_url'], + $normalized['api_base_url'], + $normalized['replay_enabled'], + $normalized['capture_level'], + $normalized['retention_days'], + self::jsonEncode($normalized['metadata']), + ] + ); + + $id = $this->insertId(); + if ($normalized['default_channel'] === 1) { + $this->clearOtherDefaultChannels($id); + } + $this->audit($id, null, 'channel_created', $actorUserId, 'info', $normalized); + + return $this->publicChannel($this->getChannel($id)); + } + + public function updateChannel(int $id, array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $channel = $this->getChannel($id); + $normalized = $this->normalizeChannelInput(array_replace($channel, $input), false); + + $this->execute( + "UPDATE release_channels + SET slug = ?, name = ?, description = ?, enabled = ?, default_channel = ?, rollout_percent = ?, + frontend_base_url = ?, api_base_url = ?, replay_enabled = ?, capture_level = ?, + retention_days = ?, metadata_json = ? + WHERE id = ?", + 'sssiidssisisi', + [ + $normalized['slug'], + $normalized['name'], + $normalized['description'], + $normalized['enabled'], + $normalized['default_channel'], + $normalized['rollout_percent'], + $normalized['frontend_base_url'], + $normalized['api_base_url'], + $normalized['replay_enabled'], + $normalized['capture_level'], + $normalized['retention_days'], + self::jsonEncode($normalized['metadata']), + $id, + ] + ); + + if ($normalized['default_channel'] === 1) { + $this->clearOtherDefaultChannels($id); + } + $this->audit($id, null, 'channel_updated', $actorUserId, 'info', $normalized); + + return $this->publicChannel($this->getChannel($id)); + } + + public function listAssignments(): array + { + if (!release_manager_schema_bootstrap::tablesExist()) { + return []; + } + + return array_map( + fn(array $row): array => $this->publicAssignment($row), + $this->selectRows( + "SELECT a.*, c.slug AS channel_slug, c.name AS channel_name + FROM release_assignments a + INNER JOIN release_channels c ON c.id = a.channel_id + WHERE a.deleted_at IS NULL AND (a.expires_at IS NULL OR a.expires_at > NOW()) + ORDER BY a.created_at DESC + LIMIT 250" + ) + ); + } + + public function createAssignment(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $subjectType = strtolower(trim((string)($input['subject_type'] ?? ''))); + if (!in_array($subjectType, self::SUBJECT_TYPES, true)) { + throw new RuntimeException('Invalid release assignment subject type.'); + } + + $subjectId = trim((string)($input['subject_id'] ?? '')); + if ($subjectId === '') { + throw new RuntimeException('Release assignment subject_id is required.'); + } + + $channel = $this->channelFromInput($input); + $reason = trim((string)($input['reason'] ?? '')); + $expiresAt = $this->normalizeDateTime($input['expires_at'] ?? null); + + $this->execute( + "INSERT INTO release_assignments (subject_type, subject_id, channel_id, reason, expires_at, actor_user_id) + VALUES (?, ?, ?, ?, ?, ?)", + 'ssissi', + [$subjectType, $subjectId, (int)$channel['id'], $reason !== '' ? $reason : null, $expiresAt, $actorUserId] + ); + + $id = $this->insertId(); + $this->clearAssignmentCache($subjectType, $subjectId); + $this->audit((int)$channel['id'], null, 'assignment_created', $actorUserId, 'info', [ + 'subject_type' => $subjectType, + 'subject_id' => $subjectId, + 'channel_slug' => $channel['slug'], + ]); + + $row = $this->selectOne( + "SELECT a.*, c.slug AS channel_slug, c.name AS channel_name + FROM release_assignments a INNER JOIN release_channels c ON c.id = a.channel_id + WHERE a.id = ?", + 'i', + [$id] + ); + return $this->publicAssignment($row ?? []); + } + + public function deleteAssignment(int $id, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $assignment = $this->selectOne('SELECT * FROM release_assignments WHERE id = ? AND deleted_at IS NULL', 'i', [$id]); + if ($assignment === null) { + throw new RuntimeException('Release assignment not found.'); + } + + $this->execute('UPDATE release_assignments SET deleted_at = NOW() WHERE id = ?', 'i', [$id]); + $this->clearAssignmentCache((string)$assignment['subject_type'], (string)$assignment['subject_id']); + $this->audit((int)$assignment['channel_id'], null, 'assignment_deleted', $actorUserId, 'info', [ + 'assignment_id' => $id, + ]); + + return ['deleted' => true, 'id' => $id]; + } + + public function listDeploymentTargets(): array + { + if (!release_manager_schema_bootstrap::tablesExist()) { + return []; + } + + return array_map( + fn(array $row): array => $this->publicDeploymentTarget($row), + $this->selectRows( + "SELECT t.*, c.slug AS channel_slug, c.name AS channel_name, i.label AS coolify_instance_label + FROM release_deployment_targets t + INNER JOIN release_channels c ON c.id = t.channel_id + LEFT JOIN coolify_instances i ON i.id = t.coolify_instance_id + WHERE t.deleted_at IS NULL + ORDER BY c.slug, FIELD(t.app, 'frontend', 'api'), t.repository" + ) + ); + } + + public function upsertDeploymentTarget(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $id = (int)($input['id'] ?? 0); + $channel = $this->channelFromInput($input); + $app = $this->normalizeApp((string)($input['app'] ?? '')); + $repository = trim((string)($input['repository'] ?? '')); + $branch = trim((string)($input['branch'] ?? 'main')) ?: 'main'; + if ($repository === '') { + throw new RuntimeException('Repository is required for release deployment targets.'); + } + $normalizedRepository = self::normalizeGithubRepositoryName($repository); + if ($normalizedRepository !== '') { + $repository = $normalizedRepository; + } + + $githubAccess = $this->githubRepositoryAccess([ + 'repository' => $repository, + 'branch' => $branch, + 'commit_mode' => 'latest', + ]); + if (($githubAccess['token_configured'] ?? false) && !($githubAccess['ok'] ?? false)) { + throw new RuntimeException('GitHub repository access test failed: ' . (string)($githubAccess['message'] ?? 'Repository is not accessible.')); + } + + $deployContext = is_array($input['deploy_context'] ?? null) ? $input['deploy_context'] : []; + foreach (['coolify_auto_create', 'coolify_enable_ssl', 'coolify_deploy_now'] as $key) { + if (array_key_exists($key, $input)) { + $deployContext[$key] = $this->toBool($input[$key]); + } + } + foreach (['coolify_domain', 'coolify_public_url', 'coolify_url_name'] as $key) { + if (array_key_exists($key, $input)) { + $deployContext[$key] = trim((string)$input[$key]); + } + } + foreach ([ + 'coolify_project_uuid', + 'project_uuid', + 'coolify_environment_uuid', + 'environment_uuid', + 'coolify_environment_name', + 'environment_name', + ] as $key) { + if (array_key_exists($key, $input)) { + $deployContext[$key] = trim((string)$input[$key]); + } + } + + $payload = [ + 'channel_id' => (int)$channel['id'], + 'app' => $app, + 'coolify_instance_id' => $this->nullablePositiveInt($input['coolify_instance_id'] ?? null), + 'coolify_service_uuid' => trim((string)($input['coolify_service_uuid'] ?? '')) ?: null, + 'repository' => $repository, + 'branch' => $branch, + 'auto_deploy' => $this->toBool($input['auto_deploy'] ?? true) ? 1 : 0, + 'health_url' => trim((string)($input['health_url'] ?? '')) ?: null, + 'deploy_context' => $deployContext, + ]; + + if ($id > 0) { + $this->execute( + "UPDATE release_deployment_targets + SET channel_id = ?, app = ?, coolify_instance_id = ?, coolify_service_uuid = ?, + repository = ?, branch = ?, auto_deploy = ?, health_url = ?, deploy_context_json = ? + WHERE id = ? AND deleted_at IS NULL", + 'isisssissi', + [ + $payload['channel_id'], + $payload['app'], + $payload['coolify_instance_id'], + $payload['coolify_service_uuid'], + $payload['repository'], + $payload['branch'], + $payload['auto_deploy'], + $payload['health_url'], + self::jsonEncode($payload['deploy_context']), + $id, + ] + ); + $targetId = $id; + $action = 'deployment_target_updated'; + } else { + $this->execute( + "INSERT INTO release_deployment_targets ( + channel_id, app, coolify_instance_id, coolify_service_uuid, + repository, branch, auto_deploy, health_url, deploy_context_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + 'isisssiss', + [ + $payload['channel_id'], + $payload['app'], + $payload['coolify_instance_id'], + $payload['coolify_service_uuid'], + $payload['repository'], + $payload['branch'], + $payload['auto_deploy'], + $payload['health_url'], + self::jsonEncode($payload['deploy_context']), + ] + ); + $targetId = $this->insertId(); + $action = 'deployment_target_created'; + } + + $this->audit((int)$channel['id'], null, $action, $actorUserId, 'info', $payload + [ + 'github_access' => $githubAccess, + ]); + return $this->publicDeploymentTarget($this->getDeploymentTarget($targetId)); + } + + public function deleteDeploymentTarget(int $id, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $target = $this->getDeploymentTarget($id); + $this->execute('UPDATE release_deployment_targets SET deleted_at = NOW() WHERE id = ?', 'i', [$id]); + $this->audit((int)$target['channel_id'], null, 'deployment_target_deleted', $actorUserId, 'warning', [ + 'target_id' => $id, + ]); + return ['deleted' => true, 'id' => $id]; + } + + public function listServiceSets(): array + { + if (!release_manager_schema_bootstrap::tablesExist()) { + return []; + } + + return array_map( + fn(array $row): array => $this->publicServiceSet($row), + $this->selectRows( + "SELECT s.*, c.slug AS channel_slug, c.name AS channel_name + FROM release_service_sets s + LEFT JOIN release_channels c ON c.id = s.channel_id + WHERE s.deleted_at IS NULL + ORDER BY s.updated_at DESC, s.created_at DESC, s.id DESC" + ) + ); + } + + public function createServiceSet(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + + $mode = $this->normalizeServiceSetMode((string)($input['mode'] ?? $input['dataset_mode'] ?? 'attach_existing')); + $sourceId = $this->nullablePositiveInt($input['source_service_set_id'] ?? $input['source_id'] ?? null); + if ($mode === 'isolated_stack') { + $sourceId = null; + } + $source = $sourceId !== null ? $this->getServiceSet($sourceId) : null; + $channel = $this->channelFromInputOrDefault($input, $source); + + $frontendTargetId = $this->serviceSetTargetIdFromInput($input, 'frontend', $source); + $apiTargetId = $this->serviceSetTargetIdFromInput($input, 'api', $source); + $dataTargets = []; + foreach (self::STACK_DATA_KINDS as $kind) { + $dataTargets[$kind] = $this->serviceSetDataTargetIdFromInput($input, $kind, $source); + } + + if ($mode === 'isolated_stack') { + $this->assertIsolatedStackTarget($frontendTargetId, 'frontend'); + $this->assertIsolatedStackTarget($apiTargetId, 'api'); + foreach (self::STACK_DATA_KINDS as $kind) { + $dataTargets[$kind] = null; + } + } + + if (!in_array($mode, ['fresh_empty', 'isolated_stack'], true) && $source === null && $frontendTargetId === null && $apiTargetId === null) { + throw new RuntimeException('Select an existing release deployment or target before creating a reusable service set.'); + } + + $metadata = is_array($input['metadata'] ?? null) ? $input['metadata'] : self::jsonDecode($input['metadata_json'] ?? null); + $metadata['dataset_mode'] = $mode; + $metadata['source_service_set_id'] = $sourceId; + $metadata['replica_integration'] = $this->replicaProvisioningPlan($mode, $source, $dataTargets); + if ($mode === 'fresh_empty') { + $metadata['isolated_empty_services'] = true; + $metadata['production_replication_attached'] = false; + } + if ($mode === 'isolated_stack') { + $metadata['isolated_stack'] = true; + $metadata['isolated_empty_services'] = true; + $metadata['production_replication_attached'] = false; + $metadata['production_code_targets_attached'] = false; + } + + $name = trim((string)($input['name'] ?? '')); + if ($name === '') { + $name = $source !== null + ? sprintf('%s %s', (string)($source['name'] ?? 'Release service set'), str_replace('_', ' ', $mode)) + : sprintf('%s service set', ucfirst(str_replace('_', ' ', $mode))); + } + $slug = $this->uniqueServiceSetSlug(self::safeSlug((string)($input['slug'] ?? $name))); + $status = $this->serviceSetStatus($mode, $frontendTargetId, $apiTargetId, $dataTargets); + $stackComplete = $mode === 'isolated_stack' + ? $frontendTargetId !== null && $apiTargetId !== null + : $frontendTargetId !== null && $apiTargetId !== null && !in_array(null, $dataTargets, true); + $health = [ + 'status' => $status, + 'stack_complete' => $stackComplete, + 'checked_at' => date('c'), + ]; + + $this->execute( + "INSERT INTO release_service_sets ( + channel_id, name, slug, mode, source_service_set_id, + frontend_target_id, api_target_id, + database_coolify_target_id, redis_coolify_target_id, minio_coolify_target_id, + status, health_json, metadata_json, actor_user_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + 'isssiiiiiisssi', + [ + (int)$channel['id'], + substr($name, 0, 128), + $slug, + $mode, + $sourceId, + $frontendTargetId, + $apiTargetId, + $dataTargets['database'], + $dataTargets['redis'], + $dataTargets['minio'], + $status, + self::jsonEncode($health), + self::jsonEncode($metadata), + $actorUserId, + ] + ); + $id = $this->insertId(); + + $this->audit((int)$channel['id'], null, 'service_set_created', $actorUserId, 'info', [ + 'service_set_id' => $id, + 'mode' => $mode, + 'source_service_set_id' => $sourceId, + 'data_targets' => $dataTargets, + ]); + + return $this->publicServiceSet($this->getServiceSet($id)); + } + + public function listBundles(int $limit = 50): array + { + if (!release_manager_schema_bootstrap::tablesExist()) { + return []; + } + + $limit = max(1, min(250, $limit)); + return array_map( + fn(array $row): array => $this->publicBundle($row), + $this->selectRows( + "SELECT b.*, c.slug AS channel_slug, c.name AS channel_name, s.name AS service_set_name, s.slug AS service_set_slug + FROM release_bundles b + INNER JOIN release_channels c ON c.id = b.channel_id + INNER JOIN release_service_sets s ON s.id = b.service_set_id + WHERE b.deleted_at IS NULL + ORDER BY b.created_at DESC, b.id DESC + LIMIT $limit" + ) + ); + } + + public function createBundle(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + + $serviceSetId = $this->nullablePositiveInt($input['service_set_id'] ?? null); + if ($serviceSetId === null) { + throw new RuntimeException('A release service set is required before creating a bundle.'); + } + $serviceSet = $this->getServiceSet($serviceSetId); + $channel = $this->channelFromInputOrDefault($input, $serviceSet); + + $versionLabel = trim((string)($input['version_label'] ?? '')); + if ($versionLabel === '') { + $versionLabel = sprintf('%s-bundle-%s', (string)($channel['slug'] ?? 'release'), date('Ymd-His')); + } + + $frontendTarget = $this->nullableDeploymentTarget((int)($serviceSet['frontend_target_id'] ?? 0) ?: null); + $apiTarget = $this->nullableDeploymentTarget((int)($serviceSet['api_target_id'] ?? 0) ?: null); + $frontend = $this->bundleAppInput($input, 'frontend', $frontendTarget, $versionLabel); + $api = $this->bundleAppInput($input, 'api', $apiTarget, $versionLabel); + + $frontendVersionId = $this->createBundleVersion($frontend, 'frontend'); + $apiVersionId = $this->createBundleVersion($api, 'api'); + + $metadata = is_array($input['metadata'] ?? null) ? $input['metadata'] : self::jsonDecode($input['metadata_json'] ?? null); + $metadata['service_set_id'] = $serviceSetId; + $metadata['stack_services'] = array_merge(['frontend', 'api'], self::STACK_DATA_KINDS); + $metadata['promotion_policy'] = 'attach_code_and_service_set_only'; + + $this->execute( + "INSERT INTO release_bundles ( + channel_id, service_set_id, version_label, + frontend_version_id, api_version_id, + frontend_repository, frontend_branch, frontend_commit_sha, + api_repository, api_branch, api_commit_sha, + metadata_json, actor_user_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + 'iisiisssssssi', + [ + (int)$channel['id'], + $serviceSetId, + $versionLabel, + $frontendVersionId, + $apiVersionId, + $frontend['repository'], + $frontend['branch'], + $frontend['commit_sha'], + $api['repository'], + $api['branch'], + $api['commit_sha'], + self::jsonEncode($metadata), + $actorUserId, + ] + ); + $id = $this->insertId(); + + $this->audit((int)$channel['id'], null, 'bundle_created', $actorUserId, 'info', [ + 'bundle_id' => $id, + 'service_set_id' => $serviceSetId, + 'frontend_repository' => $frontend['repository'], + 'api_repository' => $api['repository'], + ]); + + return $this->publicBundle($this->getBundle($id)); + } + + public function deployBundle(int $bundleId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $bundle = $this->getBundle($bundleId); + $serviceSet = $this->getServiceSet((int)$bundle['service_set_id']); + $results = []; + $deploymentIds = ['frontend' => null, 'api' => null]; + + foreach (['frontend', 'api'] as $app) { + $versionId = $this->nullablePositiveInt($bundle[$app . '_version_id'] ?? null); + if ($versionId === null) { + $results[$app] = ['status' => 'skipped', 'message' => 'No release version is attached to this app.']; + continue; + } + + $deployment = $this->startDeployment([ + 'channel_id' => (int)$bundle['channel_id'], + 'target_id' => $this->nullablePositiveInt($serviceSet[$app . '_target_id'] ?? null), + 'version_id' => $versionId, + 'service_set_id' => (int)$bundle['service_set_id'], + 'bundle_id' => $bundleId, + 'deployment_kind' => 'bundle_member', + 'app' => $app, + 'repository' => $bundle[$app . '_repository'] ?? '', + 'branch' => $bundle[$app . '_branch'] ?? 'main', + 'commit_mode' => trim((string)($bundle[$app . '_commit_sha'] ?? '')) !== '' ? 'specific' : 'latest', + 'commit_sha' => $bundle[$app . '_commit_sha'] ?? '', + 'version_label' => $bundle['version_label'] ?? null, + ], $actorUserId); + $deploymentIds[$app] = (int)($deployment['id'] ?? 0) ?: null; + $results[$app] = $deployment; + } + + $statuses = array_map(static fn(array $result): string => strtolower((string)($result['status'] ?? 'unknown')), $results); + $status = in_array('failed', $statuses, true) + ? 'failed' + : (count(array_intersect($statuses, ['queued', 'deploying', 'unknown', 'skipped'])) > 0 ? 'deploying' : 'deployed'); + + $this->execute( + "UPDATE release_bundles + SET status = ?, frontend_deployment_id = ?, api_deployment_id = ?, + deployment_result_json = ?, deployed_at = CASE WHEN ? IN ('deployed', 'deploying') THEN NOW() ELSE deployed_at END + WHERE id = ?", + 'siissi', + [ + $status, + $deploymentIds['frontend'], + $deploymentIds['api'], + self::jsonEncode(self::redactPayload($results)), + $status, + $bundleId, + ] + ); + + $this->audit((int)$bundle['channel_id'], null, 'bundle_deployed', $actorUserId, $status === 'failed' ? 'error' : 'info', [ + 'bundle_id' => $bundleId, + 'service_set_id' => (int)$bundle['service_set_id'], + 'status' => $status, + ]); + + return $this->publicBundle($this->getBundle($bundleId)); + } + + public function promoteBundle(int $bundleId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $bundle = $this->getBundle($bundleId); + $status = strtolower((string)($bundle['status'] ?? '')); + if (!in_array($status, ['deployed', 'active', 'promoted'], true)) { + throw new RuntimeException('Only deployed release bundles can be promoted.'); + } + + $channelId = (int)$bundle['channel_id']; + $frontendVersionId = $this->nullablePositiveInt($bundle['frontend_version_id'] ?? null); + $apiVersionId = $this->nullablePositiveInt($bundle['api_version_id'] ?? null); + $deploymentId = $this->nullablePositiveInt($bundle['api_deployment_id'] ?? null) + ?? $this->nullablePositiveInt($bundle['frontend_deployment_id'] ?? null); + $serviceSetId = (int)$bundle['service_set_id']; + + $this->execute('UPDATE release_channel_versions SET active = 0 WHERE channel_id = ?', 'i', [$channelId]); + $this->execute( + "INSERT INTO release_channel_versions ( + channel_id, frontend_version_id, api_version_id, deployment_id, + service_set_id, bundle_id, actor_user_id, active + ) VALUES (?, ?, ?, ?, ?, ?, ?, 1)", + 'iiiiiii', + [$channelId, $frontendVersionId, $apiVersionId, $deploymentId, $serviceSetId, $bundleId, $actorUserId] + ); + + $this->execute( + "UPDATE release_bundles SET status = 'promoted', promoted_at = NOW() WHERE id = ?", + 'i', + [$bundleId] + ); + $this->execute( + "UPDATE release_deployments SET status = 'active', completed_at = COALESCE(completed_at, NOW()) + WHERE bundle_id = ? AND status IN ('deployed', 'queued', 'deploying')", + 'i', + [$bundleId] + ); + foreach ([$frontendVersionId, $apiVersionId] as $versionId) { + if ($versionId !== null) { + $this->execute( + "UPDATE release_versions SET status = 'active', deployed_at = COALESCE(deployed_at, NOW()) WHERE id = ?", + 'i', + [$versionId] + ); + } + } + + $this->audit($channelId, $deploymentId, 'bundle_promoted', $actorUserId, 'info', [ + 'bundle_id' => $bundleId, + 'service_set_id' => $serviceSetId, + 'data_promotion' => false, + 'replica_failover' => false, + ]); + + return [ + 'channel' => $this->publicChannel($this->getChannel($channelId)), + 'versions' => $this->currentVersionsForChannel($channelId), + 'service_set' => $this->publicServiceSet($this->getServiceSet($serviceSetId)), + 'bundle' => $this->publicBundle($this->getBundle($bundleId)), + ]; + } + + public function listDeployments(int $limit = 50): array + { + if (!release_manager_schema_bootstrap::tablesExist()) { + return []; + } + + $limit = max(1, min(250, $limit)); + return array_map( + fn(array $row): array => $this->publicDeployment($row), + $this->selectRows( + "SELECT d.*, c.slug AS channel_slug, c.name AS channel_name, v.version_label, v.deployed_url + FROM release_deployments d + INNER JOIN release_channels c ON c.id = d.channel_id + LEFT JOIN release_versions v ON v.id = d.version_id + ORDER BY d.created_at DESC + LIMIT $limit" + ) + ); + } + + public function startDeployment(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $channel = $this->channelFromInput($input); + $app = $this->normalizeApp((string)($input['app'] ?? '')); + $target = $this->deploymentTargetFromInput($input, (int)$channel['id'], $app); + + $repository = trim((string)($input['repository'] ?? $target['repository'] ?? '')); + $branch = trim((string)($input['branch'] ?? $target['branch'] ?? 'main')) ?: 'main'; + $normalizedRepository = self::normalizeGithubRepositoryName($repository); + if ($normalizedRepository !== '') { + $repository = $normalizedRepository; + } + $rawCommitSha = trim((string)($input['commit_sha'] ?? $input['commit'] ?? '')); + $commitMode = $this->normalizeCommitMode((string)($input['commit_mode'] ?? ''), $rawCommitSha); + $commitSha = $commitMode === 'specific' && $rawCommitSha !== '' ? $rawCommitSha : null; + $githubAccess = null; + if ($repository !== '') { + $githubAccess = $this->githubRepositoryAccess([ + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'commit_mode' => $commitMode, + ]); + if (($githubAccess['token_configured'] ?? false) && !($githubAccess['ok'] ?? false)) { + throw new RuntimeException('GitHub repository access test failed: ' . (string)($githubAccess['message'] ?? 'Repository is not accessible.')); + } + if (($githubAccess['ok'] ?? false) && !empty($githubAccess['commit_sha'])) { + $commitSha = (string)$githubAccess['commit_sha']; + $branch = (string)($githubAccess['branch'] ?? $branch); + } + } + $versionLabel = trim((string)($input['version_label'] ?? $input['tag'] ?? $commitSha ?? date('Ymd-His'))) ?: date('Ymd-His'); + $deployedUrl = trim((string)($input['deployed_url'] ?? $target['health_url'] ?? '')) ?: null; + + $versionId = $this->nullablePositiveInt($input['version_id'] ?? null); + if ($versionId !== null) { + $version = $this->getVersion($versionId); + if ((string)($version['app'] ?? '') !== $app) { + throw new RuntimeException('Release bundle version does not match the deployment app.'); + } + $this->execute( + "UPDATE release_versions + SET repository = COALESCE(NULLIF(?, ''), repository), + branch = COALESCE(NULLIF(?, ''), branch), + commit_sha = COALESCE(?, commit_sha), + version_label = COALESCE(NULLIF(?, ''), version_label), + deployed_url = COALESCE(?, deployed_url), + status = 'deploying' + WHERE id = ?", + 'sssssi', + [$repository, $branch, $commitSha, $versionLabel, $deployedUrl, $versionId] + ); + } else { + $versionId = $this->createVersion([ + 'app' => $app, + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'tag' => trim((string)($input['tag'] ?? '')) ?: null, + 'version_label' => $versionLabel, + 'build_url' => trim((string)($input['build_url'] ?? '')) ?: null, + 'artifact_url' => trim((string)($input['artifact_url'] ?? '')) ?: null, + 'deployed_url' => $deployedUrl, + 'status' => 'deploying', + 'metadata' => is_array($input['metadata'] ?? null) ? $input['metadata'] : [], + ]); + } + + $requestedPayload = self::redactPayload($input); + if (is_array($requestedPayload)) { + $requestedPayload['commit_mode'] = $commitMode; + $requestedPayload['github_access'] = $githubAccess; + } + $targetId = isset($target['id']) ? (int)$target['id'] : null; + $serviceSetId = $this->nullablePositiveInt($input['service_set_id'] ?? null); + $bundleId = $this->nullablePositiveInt($input['bundle_id'] ?? null); + $deploymentKind = self::safeIdentifier((string)($input['deployment_kind'] ?? 'single_app'), 32) ?: 'single_app'; + $this->execute( + "INSERT INTO release_deployments ( + channel_id, target_id, version_id, service_set_id, bundle_id, deployment_kind, app, provider, repository, branch, + commit_sha, status, actor_user_id, requested_payload_json, started_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'coolify', ?, ?, ?, 'deploying', ?, ?, NOW())", + 'iiiiisssssis', + [ + (int)$channel['id'], + $targetId, + $versionId, + $serviceSetId, + $bundleId, + $deploymentKind, + $app, + $repository, + $branch, + $commitSha, + $actorUserId, + self::jsonEncode($requestedPayload), + ] + ); + $deploymentId = $this->insertId(); + + try { + $result = ['message' => 'Deployment recorded; no Coolify service target is configured.']; + $status = 'queued'; + if ($target !== null && !empty($target['coolify_instance_id'])) { + $result = $this->deployCoolifyReleaseTarget($target); + $status = 'deployed'; + } + + $this->execute( + "UPDATE release_deployments + SET status = ?, result_json = ?, deployment_url = ?, completed_at = CASE WHEN ? = 'deployed' THEN NOW() ELSE NULL END + WHERE id = ?", + 'ssssi', + [$status, self::jsonEncode(self::redactPayload($result)), $deployedUrl, $status, $deploymentId] + ); + $this->execute( + "UPDATE release_versions SET status = ?, deployed_at = CASE WHEN ? = 'deployed' THEN NOW() ELSE deployed_at END WHERE id = ?", + 'ssi', + [$status === 'deployed' ? 'deployed' : 'deploying', $status, $versionId] + ); + $this->audit((int)$channel['id'], $deploymentId, 'deployment_started', $actorUserId, 'info', [ + 'app' => $app, + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'commit_mode' => $commitMode, + 'github_access_status' => $githubAccess['status'] ?? null, + 'status' => $status, + ]); + } catch (Throwable $throwable) { + $failureSummary = self::deploymentFailureSummary($throwable, [ + 'app' => $app, + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'target_id' => $targetId, + 'coolify_instance_id' => is_array($target) ? ($target['coolify_instance_id'] ?? null) : null, + 'coolify_service_uuid' => is_array($target) ? ($target['coolify_service_uuid'] ?? null) : null, + ]); + $failureResult = [ + 'message' => 'Deployment failed before promotion. A successful deployment is required before promotion.', + 'failure_summary' => $failureSummary, + ]; + $this->execute( + "UPDATE release_deployments SET status = 'failed', result_json = ?, error_message = ?, completed_at = NOW() WHERE id = ?", + 'ssi', + [self::jsonEncode($failureResult), $throwable->getMessage(), $deploymentId] + ); + $this->execute("UPDATE release_versions SET status = 'failed' WHERE id = ?", 'i', [$versionId]); + $this->audit((int)$channel['id'], $deploymentId, 'deployment_failed', $actorUserId, 'error', [ + 'error' => $throwable->getMessage(), + 'failure_summary' => $failureSummary, + 'app' => $app, + ]); + } + + return $this->publicDeployment($this->getDeployment($deploymentId)); + } + + public function promoteDeployment(int $deploymentId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $deployment = $this->getDeployment($deploymentId); + if (!self::deploymentCanBePromoted((string)($deployment['status'] ?? ''))) { + throw new RuntimeException(self::deploymentPromotionBlockedReason($deployment)); + } + $versionId = (int)($deployment['version_id'] ?? 0); + if ($versionId <= 0) { + throw new RuntimeException('Deployment has no release version to promote.'); + } + + $channelId = (int)$deployment['channel_id']; + $current = $this->currentChannelVersionRow($channelId); + $frontendVersionId = (int)($current['frontend_version_id'] ?? 0) ?: null; + $apiVersionId = (int)($current['api_version_id'] ?? 0) ?: null; + if ((string)$deployment['app'] === 'frontend') { + $frontendVersionId = $versionId; + } else { + $apiVersionId = $versionId; + } + + $this->execute('UPDATE release_channel_versions SET active = 0 WHERE channel_id = ?', 'i', [$channelId]); + $this->execute( + "INSERT INTO release_channel_versions (channel_id, frontend_version_id, api_version_id, deployment_id, actor_user_id, active) + VALUES (?, ?, ?, ?, ?, 1)", + 'iiiii', + [$channelId, $frontendVersionId, $apiVersionId, $deploymentId, $actorUserId] + ); + $this->execute("UPDATE release_deployments SET status = 'active', completed_at = COALESCE(completed_at, NOW()) WHERE id = ?", 'i', [$deploymentId]); + $this->execute("UPDATE release_versions SET status = 'active', deployed_at = COALESCE(deployed_at, NOW()) WHERE id = ?", 'i', [$versionId]); + + $this->audit($channelId, $deploymentId, 'deployment_promoted', $actorUserId, 'info', [ + 'app' => $deployment['app'], + 'version_id' => $versionId, + ]); + + return [ + 'channel' => $this->publicChannel($this->getChannel($channelId)), + 'versions' => $this->currentVersionsForChannel($channelId), + 'deployment' => $this->publicDeployment($this->getDeployment($deploymentId)), + ]; + } + + public function rollbackChannel(int $channelId, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $channel = $this->getChannel($channelId); + $previous = $this->selectOne( + "SELECT * FROM release_channel_versions + WHERE channel_id = ? AND active = 0 + ORDER BY activated_at DESC, id DESC + LIMIT 1", + 'i', + [$channelId] + ); + if ($previous === null) { + throw new RuntimeException('No previous release version exists for this channel.'); + } + + $this->execute('UPDATE release_channel_versions SET active = 0 WHERE channel_id = ?', 'i', [$channelId]); + $this->execute( + "INSERT INTO release_channel_versions (channel_id, frontend_version_id, api_version_id, deployment_id, actor_user_id, active) + VALUES (?, ?, ?, ?, ?, 1)", + 'iiiii', + [ + $channelId, + (int)($previous['frontend_version_id'] ?? 0) ?: null, + (int)($previous['api_version_id'] ?? 0) ?: null, + (int)($previous['deployment_id'] ?? 0) ?: null, + $actorUserId, + ] + ); + + $this->audit($channelId, (int)($previous['deployment_id'] ?? 0) ?: null, 'channel_rolled_back', $actorUserId, 'warning', [ + 'previous_channel_version_id' => $previous['id'] ?? null, + ]); + + return [ + 'channel' => $this->publicChannel($channel), + 'versions' => $this->currentVersionsForChannel($channelId), + ]; + } + + public function setReplayTarget(array $input, ?int $actorUserId = null): array + { + $this->ensureSchema(); + $targetType = strtolower(trim((string)($input['target_type'] ?? ''))); + if (!in_array($targetType, ['user', 'subuser', 'customer', 'channel'], true)) { + throw new RuntimeException('Invalid replay target type.'); + } + + $targetId = trim((string)($input['target_id'] ?? '')) ?: null; + $channel = null; + if ($targetType === 'channel' || isset($input['channel_id']) || isset($input['channel_slug'])) { + $channel = $this->channelFromInput($input); + $targetId = $targetId ?: (string)$channel['slug']; + } + + $captureLevel = $this->normalizeCaptureLevel((string)($input['capture_level'] ?? 'full_redacted')); + $enabled = $this->toBool($input['enabled'] ?? true) ? 1 : 0; + $expiresAt = $this->normalizeDateTime($input['expires_at'] ?? null); + + $this->execute( + "INSERT INTO release_replay_targets (target_type, target_id, channel_id, capture_level, enabled, expires_at, actor_user_id) + VALUES (?, ?, ?, ?, ?, ?, ?)", + 'ssisisi', + [$targetType, $targetId, $channel['id'] ?? null, $captureLevel, $enabled, $expiresAt, $actorUserId] + ); + + $id = $this->insertId(); + $this->audit($channel !== null ? (int)$channel['id'] : null, null, 'replay_target_created', $actorUserId, 'warning', [ + 'target_type' => $targetType, + 'target_id' => $targetId, + 'capture_level' => $captureLevel, + 'enabled' => (bool)$enabled, + ]); + + return $this->selectOne('SELECT * FROM release_replay_targets WHERE id = ?', 'i', [$id]) ?? []; + } + + public function ingestTimelineEvents(array $events, array $context = [], bool $ensureSchema = true): array + { + if ($ensureSchema) { + $this->ensureSchema(); + } + if ($events === [] || !isset($events[0])) { + $events = [$events]; + } + + $traceId = self::safeIdentifier((string)($context['trace_id'] ?? $this->requestTraceId()), 64); + if ($traceId === '') { + $traceId = $this->requestTraceId(); + } + + $principalContext = $this->currentPrincipalContext(); + $context = array_replace($principalContext, array_filter($context, static fn(mixed $value): bool => $value !== null && $value !== '')); + + $channelSlug = self::safeSlug((string)($context['channel_slug'] ?? $context['release_channel'] ?? '')); + $channel = $channelSlug !== '' ? $this->findChannelBySlug($channelSlug) : null; + if ($channel === null) { + $channel = $this->resolveChannel($context); + } + $sessionId = $this->timelineSessionId($traceId, $context, $channel); + $accepted = 0; + + foreach ($events as $event) { + if (!is_array($event)) { + continue; + } + $eventType = self::safeIdentifier((string)($event['type'] ?? $event['event_type'] ?? 'event'), 64) ?: 'event'; + $severity = self::safeIdentifier((string)($event['severity'] ?? 'info'), 16) ?: 'info'; + $payload = self::redactPayload($event['payload'] ?? $event); + $occurredAt = $this->normalizeDateTime($event['occurred_at'] ?? null) ?? date('Y-m-d H:i:s'); + + $this->execute( + "INSERT INTO release_timeline_events ( + timeline_session_id, trace_id, event_type, severity, module_key, + route_path, component, request_id, occurred_at, payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + 'isssssssss', + [ + $sessionId, + $traceId, + $eventType, + $severity, + self::safeIdentifier((string)($event['module_key'] ?? ''), 64) ?: null, + trim((string)($event['route'] ?? $event['route_path'] ?? '')) ?: null, + trim((string)($event['component'] ?? '')) ?: null, + trim((string)($event['request_id'] ?? '')) ?: null, + $occurredAt, + self::jsonEncode($payload), + ] + ); + $accepted++; + } + + return ['accepted' => $accepted, 'trace_id' => $traceId, 'timeline_session_id' => $sessionId]; + } + + public function searchTimeline(array $filters = []): array + { + $this->ensureSchema(); + $types = ''; + $params = []; + $where = ['1 = 1']; + + foreach ([ + 'trace_id' => 'e.trace_id', + 'event_type' => 'e.event_type', + 'severity' => 'e.severity', + 'module_key' => 'e.module_key', + 'channel_slug' => 's.channel_slug', + 'principal_type' => 's.principal_type', + 'principal_id' => 's.principal_id', + ] as $filterKey => $column) { + $value = trim((string)($filters[$filterKey] ?? '')); + if ($value === '') { + continue; + } + $where[] = "$column = ?"; + $types .= 's'; + $params[] = $value; + } + + if (isset($filters['customer_number']) && is_numeric($filters['customer_number'])) { + $where[] = 's.customer_number = ?'; + $types .= 'i'; + $params[] = (int)$filters['customer_number']; + } + + $limit = max(1, min(500, (int)($filters['limit'] ?? 100))); + $rows = $this->selectRows( + "SELECT e.*, s.principal_type, s.principal_id, s.customer_number, s.channel_slug + FROM release_timeline_events e + LEFT JOIN release_timeline_sessions s ON s.id = e.timeline_session_id + WHERE " . implode(' AND ', $where) . " + ORDER BY e.occurred_at DESC, e.id DESC + LIMIT $limit", + $types, + $params + ); + + return array_map(fn(array $row): array => $this->publicTimelineEvent($row), $rows); + } + + public function handleGithubWebhook(array $headers, string $rawBody): array + { + $this->ensureSchema(); + $secret = (string)$this->moduleConfigValue('ReleaseManager', 'github_webhook_secret', ''); + $signature = self::headerValue($headers, 'X-Hub-Signature-256'); + if (!self::verifyGithubSignature($secret, $rawBody, $signature)) { + throw new RuntimeException('Invalid GitHub webhook signature.'); + } + + $event = self::headerValue($headers, 'X-GitHub-Event') ?: 'unknown'; + $payload = json_decode($rawBody, true); + if (!is_array($payload)) { + throw new RuntimeException('Invalid GitHub webhook JSON payload.'); + } + + if ($event !== 'push') { + $this->audit(null, null, 'github_webhook_ignored', null, 'info', ['event' => $event]); + return ['event' => $event, 'deployments' => [], 'ignored' => true]; + } + + $repository = (string)($payload['repository']['full_name'] ?? $payload['repository']['name'] ?? ''); + $branch = preg_replace('#^refs/heads/#', '', (string)($payload['ref'] ?? '')); + $commitSha = (string)($payload['after'] ?? ''); + if ($repository === '' || $branch === '' || $commitSha === '') { + throw new RuntimeException('GitHub push payload is missing repository, branch, or commit.'); + } + + $targets = $this->selectRows( + "SELECT * FROM release_deployment_targets + WHERE deleted_at IS NULL AND auto_deploy = 1 AND repository = ? AND branch = ?", + 'ss', + [$repository, $branch] + ); + + $deployments = []; + foreach ($targets as $target) { + $deployments[] = $this->startDeployment([ + 'target_id' => (int)$target['id'], + 'channel_id' => (int)$target['channel_id'], + 'app' => (string)$target['app'], + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'version_label' => substr($commitSha, 0, 12), + 'build_url' => (string)($payload['compare'] ?? ''), + 'metadata' => [ + 'github_event' => $event, + 'head_commit' => self::redactPayload($payload['head_commit'] ?? []), + ], + ]); + } + + $this->audit(null, null, 'github_webhook_processed', null, 'info', [ + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'deployment_count' => count($deployments), + ]); + + return [ + 'event' => $event, + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'deployments' => $deployments, + ]; + } + + public function healthProbe(): array + { + $startedAt = microtime(true); + try { + $summary = $this->summary(); + $channels = $summary['channels'] ?? []; + $deployments = $summary['deployments'] ?? []; + $failedDeployments = array_values(array_filter($deployments, static fn(array $deployment): bool => ($deployment['status'] ?? '') === 'failed')); + $channelsWithoutVersions = array_values(array_filter($channels, static function (array $channel): bool { + if (($channel['enabled'] ?? false) !== true) { + return false; + } + $versions = $channel['versions'] ?? []; + return empty($versions['frontend']) && empty($versions['api']); + })); + + $status = 'ok'; + $reason = 'Release channels and deployment telemetry are available.'; + $reasonKey = 'release_manager_available'; + if ($failedDeployments !== []) { + $status = 'degraded'; + $reason = 'One or more recent release deployments failed.'; + $reasonKey = 'release_deployments_failed'; + } elseif ($channelsWithoutVersions !== []) { + $status = 'degraded'; + $reason = 'One or more enabled release channels have no active versions yet.'; + $reasonKey = 'release_channels_without_versions'; + } + + return [ + 'status' => $status, + 'status_reason' => $reason, + 'status_reason_key' => $reasonKey, + 'status_reason_params' => [ + 'channels' => count($channels), + 'deployment_targets' => count($summary['deployment_targets'] ?? []), + 'recent_failed_deployments' => count($failedDeployments), + ], + 'checked_at' => date('c'), + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + ]; + } catch (Throwable $throwable) { + return [ + 'status' => 'down', + 'status_reason' => 'Release manager probe failed: ' . $throwable->getMessage(), + 'status_reason_key' => 'release_manager_probe_failed', + 'status_reason_params' => ['error' => $throwable->getMessage()], + 'checked_at' => date('c'), + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + ]; + } + } + + private function ensureSchema(): void + { + if ($this->schemaEnsured) { + return; + } + release_manager_schema_bootstrap::ensureTables(); + $this->schemaEnsured = true; + } + + private function resolveChannel(array $context): array + { + $cacheKey = $this->assignmentCacheKey($context); + if ($cacheKey !== '' && defined('redis')) { + try { + $cached = redis->get($cacheKey); + if (is_string($cached) && $cached !== '') { + $decoded = json_decode($cached, true); + if (is_array($decoded) && !empty($decoded['id'])) { + return $decoded; + } + } + } catch (Throwable) { + } + } + + $candidates = []; + if (!empty($context['principal_type']) && !empty($context['principal_id'])) { + $candidates[] = [(string)$context['principal_type'], (string)$context['principal_id']]; + } + if (!empty($context['customer_number'])) { + $candidates[] = ['customer', (string)(int)$context['customer_number']]; + } + + foreach ($candidates as [$subjectType, $subjectId]) { + $row = $this->selectOne( + "SELECT c.* + FROM release_assignments a + INNER JOIN release_channels c ON c.id = a.channel_id + WHERE a.deleted_at IS NULL + AND c.deleted_at IS NULL + AND c.enabled = 1 + AND a.subject_type = ? + AND a.subject_id = ? + AND (a.expires_at IS NULL OR a.expires_at > NOW()) + ORDER BY a.created_at DESC, a.id DESC + LIMIT 1", + 'ss', + [$subjectType, $subjectId] + ); + if ($row !== null) { + $this->cacheResolvedChannel($cacheKey, $row); + return $row; + } + } + + $rolloutChannel = $this->rolloutChannelForContext($context); + if ($rolloutChannel !== null) { + $this->cacheResolvedChannel($cacheKey, $rolloutChannel); + return $rolloutChannel; + } + + $default = $this->defaultChannel(); + $this->cacheResolvedChannel($cacheKey, $default); + return $default; + } + + private function rolloutChannelForContext(array $context): ?array + { + $seed = (string)($context['principal_id'] ?? $context['customer_number'] ?? ''); + if ($seed === '') { + return null; + } + + $bucket = (hexdec(substr(hash('sha256', $seed), 0, 8)) % 10000) / 100; + $channels = $this->selectRows( + "SELECT * FROM release_channels + WHERE deleted_at IS NULL AND enabled = 1 AND default_channel = 0 AND rollout_percent > 0 + ORDER BY rollout_percent DESC, slug" + ); + + foreach ($channels as $channel) { + if ($bucket < (float)$channel['rollout_percent']) { + return $channel; + } + } + + return null; + } + + private function capturePolicyFor(array $context, array $channel): array + { + $enabled = (bool)((int)($channel['replay_enabled'] ?? 0)); + $captureLevel = $this->normalizeCaptureLevel((string)($channel['capture_level'] ?? 'metadata')); + $retentionDays = max(1, (int)($channel['retention_days'] ?? 14)); + + $targets = []; + if (!empty($context['principal_type']) && !empty($context['principal_id'])) { + $targets[] = [(string)$context['principal_type'], (string)$context['principal_id']]; + } + if (!empty($context['customer_number'])) { + $targets[] = ['customer', (string)(int)$context['customer_number']]; + } + $targets[] = ['channel', (string)$channel['slug']]; + + foreach ($targets as [$targetType, $targetId]) { + $row = $this->selectOne( + "SELECT capture_level + FROM release_replay_targets + WHERE deleted_at IS NULL + AND enabled = 1 + AND target_type = ? + AND (target_id = ? OR (target_type = 'channel' AND channel_id = ?)) + AND (expires_at IS NULL OR expires_at > NOW()) + ORDER BY created_at DESC, id DESC + LIMIT 1", + 'ssi', + [$targetType, $targetId, (int)$channel['id']] + ); + if ($row !== null) { + $enabled = true; + $captureLevel = $this->normalizeCaptureLevel((string)$row['capture_level']); + break; + } + } + + return [ + 'enabled' => $enabled, + 'capture_level' => $captureLevel, + 'all_failure_metadata' => true, + 'retention_days' => $retentionDays, + ]; + } + + private function currentPrincipalContext(): array + { + try { + $auth = new authentication(); + $subuser = $auth->get_subuser(); + if ($subuser !== false) { + return [ + 'principal_type' => 'subuser', + 'principal_id' => (string)$subuser->id, + 'customer_number' => $auth->get_subuser_customer_number_target() ?: null, + ]; + } + $user = $auth->get_user(); + if ($user !== false) { + return [ + 'principal_type' => 'user', + 'principal_id' => (string)$user->id, + 'customer_number' => isset($user->customer_number) ? (int)$user->customer_number->value() : null, + ]; + } + } catch (Throwable) { + } + + return [ + 'principal_type' => null, + 'principal_id' => null, + 'customer_number' => null, + ]; + } + + private function defaultChannel(): array + { + $channel = $this->selectOne( + "SELECT * FROM release_channels WHERE deleted_at IS NULL AND enabled = 1 AND default_channel = 1 ORDER BY id LIMIT 1" + ); + if ($channel !== null) { + return $channel; + } + + $channel = $this->selectOne( + "SELECT * FROM release_channels WHERE deleted_at IS NULL AND slug = 'stable' ORDER BY id LIMIT 1" + ); + if ($channel !== null) { + return $channel; + } + + throw new RuntimeException('No release channel is configured.'); + } + + private function currentVersionsForChannel(int $channelId): array + { + $current = $this->currentChannelVersionRow($channelId); + $frontend = null; + $api = null; + if (!empty($current['frontend_version_id'])) { + $frontend = $this->publicVersion($this->getVersion((int)$current['frontend_version_id'])); + } + if (!empty($current['api_version_id'])) { + $api = $this->publicVersion($this->getVersion((int)$current['api_version_id'])); + } + $serviceSet = null; + if (!empty($current['service_set_id'])) { + try { + $serviceSet = $this->publicServiceSet($this->getServiceSet((int)$current['service_set_id']), false); + } catch (Throwable) { + $serviceSet = null; + } + } + + return [ + 'frontend' => $frontend, + 'api' => $api, + 'service_set' => $serviceSet, + 'bundle_id' => isset($current['bundle_id']) ? (int)$current['bundle_id'] : null, + ]; + } + + private function channelAvailability(array $channel): array + { + $missing = []; + if (trim((string)($channel['frontend_base_url'] ?? '')) === '') { + $missing[] = 'frontend_base_url'; + } + if (trim((string)($channel['api_base_url'] ?? '')) === '') { + $missing[] = 'api_base_url'; + } + + return [ + 'configured' => count($missing) === 0, + 'missing' => $missing, + 'status' => count($missing) === 0 ? 'ready' : 'unconfigured', + ]; + } + + private function currentChannelVersionRow(int $channelId): ?array + { + return $this->selectOne( + "SELECT * FROM release_channel_versions + WHERE channel_id = ? AND active = 1 + ORDER BY activated_at DESC, id DESC + LIMIT 1", + 'i', + [$channelId] + ); + } + + private function createVersion(array $input): int + { + $this->execute( + "INSERT INTO release_versions ( + app, repository, branch, commit_sha, tag, version_label, + build_url, artifact_url, deployed_url, status, metadata_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + 'sssssssssss', + [ + $input['app'], + $input['repository'] ?? null, + $input['branch'] ?? null, + $input['commit_sha'] ?? null, + $input['tag'] ?? null, + $input['version_label'] ?? null, + $input['build_url'] ?? null, + $input['artifact_url'] ?? null, + $input['deployed_url'] ?? null, + $input['status'] ?? 'discovered', + self::jsonEncode($input['metadata'] ?? []), + ] + ); + return $this->insertId(); + } + + private function restartCoolifyService(int $instanceId, string $serviceUuid): array + { + $instance = $this->selectOne('SELECT * FROM coolify_instances WHERE id = ? AND deleted_at IS NULL', 'i', [$instanceId]); + if ($instance === null) { + throw new RuntimeException('Coolify instance for release deployment was not found.'); + } + $token = replication_secret_box::decrypt((string)$instance['api_token_secret']); + $client = new coolify_api_client((string)$instance['base_url'], $token, 20); + return $client->restartService($serviceUuid); + } + + private function deployCoolifyReleaseTarget(array $target): array + { + $instanceId = (int)($target['coolify_instance_id'] ?? 0); + $instance = $this->selectOne('SELECT * FROM coolify_instances WHERE id = ? AND deleted_at IS NULL', 'i', [$instanceId]); + if ($instance === null) { + throw new RuntimeException('Coolify instance for release deployment was not found.'); + } + + $token = replication_secret_box::decrypt((string)$instance['api_token_secret']); + $client = new coolify_api_client((string)$instance['base_url'], $token, 20); + $context = self::jsonDecode($target['deploy_context_json'] ?? null); + $serviceUuid = trim((string)($target['coolify_service_uuid'] ?? '')); + $created = null; + + if ($serviceUuid === '' && $this->toBool($context['coolify_auto_create'] ?? false)) { + $created = $client->createService($this->releaseCoolifyServicePayload($target, $context, $instance)); + $serviceUuid = trim((string)($created['uuid'] ?? '')); + if ($serviceUuid === '') { + throw new RuntimeException('Coolify did not return a service UUID for the release target.'); + } + $this->execute( + 'UPDATE release_deployment_targets SET coolify_service_uuid = ? WHERE id = ?', + 'si', + [$serviceUuid, (int)$target['id']] + ); + } + + if ($serviceUuid === '') { + throw new RuntimeException('Select an existing Coolify service or enable Coolify service creation before deployment.'); + } + + $update = null; + $publicUrl = $this->releaseCoolifyPublicUrl($target, $context); + if ($this->toBool($context['coolify_enable_ssl'] ?? false) && $publicUrl !== null) { + $update = $client->updateService($serviceUuid, [ + 'urls' => [ + [ + 'name' => (string)($target['app'] ?? 'release'), + 'url' => $publicUrl, + ], + ], + 'force_domain_override' => true, + ]); + } + + $restart = $client->restartService($serviceUuid); + return [ + 'service_uuid' => $serviceUuid, + 'ssl_enabled' => $this->toBool($context['coolify_enable_ssl'] ?? false), + 'public_url' => $publicUrl, + 'created' => self::redactPayload($created ?? []), + 'updated' => self::redactPayload($update ?? []), + 'restart' => self::redactPayload($restart), + ]; + } + + private function releaseCoolifyServicePayload(array $target, array $context, array $instance): array + { + $publicUrl = $this->releaseCoolifyPublicUrl($target, $context); + $projectUuid = trim((string)($context['coolify_project_uuid'] ?? $context['project_uuid'] ?? $instance['default_project_uuid'] ?? '')); + if ($projectUuid === '') { + throw new RuntimeException('Select a Coolify project for this release target before creating a service.'); + } + $environmentUuid = trim((string)($context['coolify_environment_uuid'] ?? $context['environment_uuid'] ?? $instance['default_environment_uuid'] ?? '')) ?: null; + $environmentName = trim((string)($context['coolify_environment_name'] ?? $context['environment_name'] ?? $instance['default_environment_name'] ?? 'production')) ?: 'production'; + $serverUuid = $this->releaseCoolifyServerUuid($context, $instance); + if ($serverUuid === '') { + throw new RuntimeException('Release Manager could not resolve a Coolify server UUID automatically for this instance.'); + } + + $requestedName = trim((string)($context['coolify_service_name'] ?? $context['service_name'] ?? '')); + $name = self::safeIdentifier( + $requestedName !== '' + ? $requestedName + : 'release-' . (string)($target['channel_slug'] ?? $target['channel_id'] ?? 'channel') . '-' . (string)($target['app'] ?? 'app'), + 64 + ); + $compose = trim((string)($context['docker_compose_raw'] ?? '')); + if ($compose === '') { + $repository = strtolower(trim((string)($target['repository'] ?? ''))); + $branch = self::safeIdentifier((string)($target['branch'] ?? 'main'), 64); + $image = trim((string)($context['image'] ?? '')); + if ($image === '' && $repository !== '') { + $image = 'ghcr.io/' . $repository . ':' . ($branch !== '' ? $branch : 'main'); + } + if ($image === '') { + throw new RuntimeException('Coolify service creation needs either docker_compose_raw or an image/repository.'); + } + $compose = "services:\n app:\n image: " . $image . "\n restart: unless-stopped\n"; + } + + $payload = [ + 'type' => 'docker-compose-empty', + 'name' => $name, + 'description' => 'Truckwash release manager target for ' . (string)($target['repository'] ?? ''), + 'project_uuid' => $projectUuid, + 'environment_name' => $environmentName, + 'environment_uuid' => $environmentUuid, + 'server_uuid' => $serverUuid, + 'instant_deploy' => $this->toBool($context['coolify_deploy_now'] ?? true), + 'docker_compose_raw' => base64_encode($compose), + 'force_domain_override' => true, + ]; + + if ($publicUrl !== null) { + $payload['urls'] = [ + [ + 'name' => (string)($target['app'] ?? 'release'), + 'url' => $publicUrl, + ], + ]; + } + + return array_filter($payload, static fn(mixed $value): bool => $value !== null && $value !== ''); + } + + private function releaseCoolifyServerUuid(array $context, array $instance): string + { + $explicit = trim((string)($context['coolify_server_uuid'] ?? $context['server_uuid'] ?? '')); + if ($explicit !== '') { + return $explicit; + } + + $default = trim((string)($instance['default_server_uuid'] ?? '')); + if ($default !== '') { + return $default; + } + + $tokenSecret = trim((string)($instance['api_token_secret'] ?? '')); + if ($tokenSecret === '') { + return ''; + } + + try { + $token = replication_secret_box::decrypt($tokenSecret); + $servers = (new coolify_api_client((string)($instance['base_url'] ?? ''), $token, 4))->listServers(); + } catch (Throwable) { + return ''; + } + + $firstServerUuid = ''; + foreach ($this->payloadRows($servers) as $server) { + if (!is_array($server)) { + continue; + } + $uuid = trim((string)($server['uuid'] ?? '')); + if ($uuid === '') { + continue; + } + if ($firstServerUuid === '') { + $firstServerUuid = $uuid; + } + $settings = is_array($server['settings'] ?? null) ? $server['settings'] : []; + if (($settings['is_reachable'] ?? true) !== false && ($settings['is_usable'] ?? true) !== false) { + return $uuid; + } + } + + return $firstServerUuid; + } + + private function releaseCoolifyPublicUrl(array $target, array $context): ?string + { + $raw = trim((string)($context['coolify_public_url'] ?? $context['coolify_domain'] ?? '')); + if ($raw === '' && !$this->toBool($context['coolify_enable_ssl'] ?? false)) { + $raw = trim((string)($target['health_url'] ?? '')); + } + if ($raw === '') { + return null; + } + if ($this->toBool($context['coolify_enable_ssl'] ?? false)) { + $domain = self::domainSuggestionHost($raw); + if ($domain === null) { + throw new RuntimeException('Coolify SSL requires a DNS domain routed to the load balancer.'); + } + return 'https://' . $domain; + } + $raw = preg_replace('#/(health|ping)$#i', '', rtrim($raw, '/')) ?: $raw; + if (preg_match('#^https?://#i', $raw) !== 1) { + $raw = ($this->toBool($context['coolify_enable_ssl'] ?? false) ? 'https://' : 'http://') . $raw; + } + return rtrim($raw, '/'); + } + + private function timelineSessionId(string $traceId, array $context, ?array $channel): int + { + $existing = $this->selectOne('SELECT id FROM release_timeline_sessions WHERE trace_id = ? LIMIT 1', 's', [$traceId]); + $userAgent = substr((string)($_SERVER['HTTP_USER_AGENT'] ?? ''), 0, 512); + $principalType = trim((string)($context['principal_type'] ?? '')) ?: null; + $principalId = trim((string)($context['principal_id'] ?? '')) ?: null; + $customerNumber = isset($context['customer_number']) && is_numeric($context['customer_number']) + ? (int)$context['customer_number'] + : null; + if ($existing !== null) { + $this->execute( + "UPDATE release_timeline_sessions + SET last_seen_at = NOW(), channel_id = COALESCE(?, channel_id), channel_slug = COALESCE(?, channel_slug), + principal_type = COALESCE(?, principal_type), principal_id = COALESCE(?, principal_id), + customer_number = COALESCE(?, customer_number), user_agent = COALESCE(?, user_agent) + WHERE id = ?", + 'isssisi', + [ + $channel['id'] ?? null, + $channel['slug'] ?? null, + $principalType, + $principalId, + $customerNumber, + $userAgent !== '' ? $userAgent : null, + (int)$existing['id'], + ] + ); + return (int)$existing['id']; + } + + $this->execute( + "INSERT INTO release_timeline_sessions ( + trace_id, session_hash, principal_type, principal_id, customer_number, + channel_id, channel_slug, user_agent + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + 'ssssiiss', + [ + $traceId, + $this->sessionHash(), + $principalType, + $principalId, + $customerNumber, + $channel['id'] ?? null, + $channel['slug'] ?? null, + $userAgent !== '' ? $userAgent : null, + ] + ); + + return $this->insertId(); + } + + private function sessionHash(): ?string + { + $authorization = (string)($_SERVER['HTTP_AUTHORIZATION'] ?? ''); + if ($authorization === '') { + return null; + } + return hash('sha256', str_replace('Bearer ', '', $authorization)); + } + + private function inferModuleKeyFromUri(string $uri): ?string + { + $path = strtolower(explode('?', $uri)[0] ?? ''); + $map = [ + '/auth' => 'auth', + '/superuser/releases' => 'releasemanager', + '/release' => 'releasemanager', + '/superuser/coolify' => 'coolify', + '/coolify' => 'coolify', + '/failover' => 'failover', + '/edge-gateway' => 'edgegateway', + '/edgegateway' => 'edgegateway', + '/modules/action-logs' => 'moduleactionlogs', + '/worker' => 'worker', + '/economic' => 'economic', + '/stripe' => 'stripe', + '/selfserve' => 'selfserve', + '/bird' => 'bird', + '/xlvask' => 'xlvask', + ]; + + foreach ($map as $prefix => $moduleKey) { + if (str_starts_with($path, $prefix)) { + return $moduleKey; + } + } + + return null; + } + + private function timelineSummary(): array + { + $events = $this->selectOne( + "SELECT COUNT(*) AS total, + SUM(CASE WHEN severity = 'error' THEN 1 ELSE 0 END) AS errors, + MAX(created_at) AS last_event_at + FROM release_timeline_events" + ) ?? []; + $sessions = $this->selectOne('SELECT COUNT(*) AS total FROM release_timeline_sessions') ?? []; + + return [ + 'sessions' => (int)($sessions['total'] ?? 0), + 'events' => (int)($events['total'] ?? 0), + 'errors' => (int)($events['errors'] ?? 0), + 'last_event_at' => $events['last_event_at'] ?? null, + ]; + } + + private function latestModuleHealth(): array + { + return $this->selectRows( + "SELECT h.* + FROM release_module_health_snapshots h + INNER JOIN ( + SELECT module_key, MAX(checked_at) AS checked_at + FROM release_module_health_snapshots + GROUP BY module_key + ) latest ON latest.module_key = h.module_key AND latest.checked_at = h.checked_at + ORDER BY h.module_key" + ); + } + + private function githubRepositoryAccess(array $input): array + { + $tokenConfigured = $this->hasGithubApiToken(); + $repository = self::normalizeGithubRepositoryName((string)($input['repository'] ?? '')); + $branch = trim((string)($input['branch'] ?? '')); + $rawCommitSha = trim((string)($input['commit_sha'] ?? $input['commit'] ?? '')); + $commitMode = $this->normalizeCommitMode((string)($input['commit_mode'] ?? ''), $rawCommitSha); + + if ($repository === '') { + return [ + 'ok' => false, + 'status' => 'invalid_repository', + 'token_configured' => $tokenConfigured, + 'message' => 'GitHub repository must use owner/repo format.', + 'repository' => trim((string)($input['repository'] ?? '')), + 'branch' => $branch, + 'commit_mode' => $commitMode, + ]; + } + if ($commitMode === 'specific' && $rawCommitSha === '') { + return [ + 'ok' => false, + 'status' => 'commit_required', + 'token_configured' => $tokenConfigured, + 'message' => 'Specific commit deployment requires a commit SHA.', + 'repository' => $repository, + 'branch' => $branch, + 'commit_mode' => $commitMode, + ]; + } + + if (!$tokenConfigured) { + $response = $this->githubTokenMissingResponse($repository, $branch); + $response['commit_mode'] = $commitMode; + return $response; + } + + try { + $repo = $this->githubRequest('GET', '/repos/' . $this->githubRepositoryPath($repository)); + $defaultBranch = trim((string)($repo['default_branch'] ?? 'main')) ?: 'main'; + $branch = $branch !== '' ? $branch : $defaultBranch; + $branchRow = $this->githubRequest( + 'GET', + '/repos/' . $this->githubRepositoryPath($repository) . '/branches/' . rawurlencode($branch) + ); + $latestCommitSha = trim((string)($branchRow['commit']['sha'] ?? '')); + $commitSha = $latestCommitSha; + $commitUrl = (string)($branchRow['commit']['url'] ?? ''); + if ($commitMode === 'specific') { + $commitRow = $this->githubRequest( + 'GET', + '/repos/' . $this->githubRepositoryPath($repository) . '/commits/' . rawurlencode($rawCommitSha) + ); + $commitSha = trim((string)($commitRow['sha'] ?? $rawCommitSha)); + $commitUrl = (string)($commitRow['html_url'] ?? $commitUrl); + } + + return [ + 'ok' => true, + 'status' => 'accessible', + 'token_configured' => true, + 'message' => $commitMode === 'specific' + ? 'Repository, branch, and commit are accessible with the configured GitHub token.' + : 'Repository and branch are accessible with the configured GitHub token.', + 'repository' => $repository, + 'branch' => $branch, + 'default_branch' => $defaultBranch, + 'private' => (bool)($repo['private'] ?? false), + 'html_url' => $repo['html_url'] ?? null, + 'commit_mode' => $commitMode, + 'commit_sha' => $commitSha !== '' ? $commitSha : null, + 'latest_commit_sha' => $latestCommitSha !== '' ? $latestCommitSha : null, + 'commit_url' => $commitUrl !== '' ? $commitUrl : null, + ]; + } catch (Throwable $throwable) { + return [ + 'ok' => false, + 'status' => 'inaccessible', + 'token_configured' => true, + 'message' => $throwable->getMessage(), + 'repository' => $repository, + 'branch' => $branch, + 'commit_mode' => $commitMode, + ]; + } + } + + private function normalizeCommitMode(string $commitMode, string $commitSha): string + { + $mode = strtolower(trim($commitMode)); + $commit = strtolower(trim($commitSha)); + if ($mode === 'specific') { + return 'specific'; + } + if ($mode === 'latest' || $mode === 'head' || $commit === '' || in_array($commit, ['latest', 'head'], true)) { + return 'latest'; + } + return 'specific'; + } + + private function githubTokenMissingResponse(?string $repository = null, ?string $branch = null): array + { + return [ + 'ok' => false, + 'status' => 'not_configured', + 'token_configured' => false, + 'message' => 'Configure ReleaseManager github_token or RELEASE_MANAGER_GITHUB_TOKEN before using private GitHub repositories.', + 'repository' => $repository, + 'branch' => $branch, + 'repositories' => [], + 'branches' => [], + 'commits' => [], + ]; + } + + private function hasGithubApiToken(): bool + { + return $this->githubApiToken() !== ''; + } + + private function githubEnvToken(): string + { + foreach (['RELEASE_MANAGER_GITHUB_TOKEN', 'GITHUB_TOKEN', 'GH_TOKEN'] as $key) { + $value = trim((string)(getenv($key) ?: ($_SERVER[$key] ?? ''))); + if ($value !== '') { + return $value; + } + } + + return ''; + } + + private function githubApiToken(): string + { + $envToken = $this->githubEnvToken(); + if ($envToken !== '') { + return $envToken; + } + + $token = trim((string)$this->moduleConfigValue('ReleaseManager', 'github_token', '')); + if ($token !== '' && str_starts_with($token, 'twsec:v1:') && class_exists(replication_secret_box::class)) { + try { + $token = replication_secret_box::decrypt($token); + } catch (Throwable) { + $token = ''; + } + } + return trim($token); + } + + private function githubApiBaseUrl(): string + { + $value = trim((string)(getenv('RELEASE_MANAGER_GITHUB_API_URL') ?: ($_SERVER['RELEASE_MANAGER_GITHUB_API_URL'] ?? ''))); + if ($value === '') { + $value = trim((string)$this->moduleConfigValue('ReleaseManager', 'github_api_url', 'https://api.github.com')); + } + $value = rtrim($value, '/'); + return preg_match('#^https?://#i', $value) === 1 ? $value : 'https://api.github.com'; + } + + private function githubRepositoryPath(string $repository): string + { + [$owner, $name] = explode('/', $repository, 2); + return rawurlencode($owner) . '/' . rawurlencode($name); + } + + private function githubRequest(string $method, string $path, array $query = []): array + { + $token = $this->githubApiToken(); + if ($token === '') { + throw new RuntimeException('GitHub token is not configured.'); + } + + $url = $this->githubApiBaseUrl() . '/' . ltrim($path, '/'); + if ($query !== []) { + $url .= '?' . http_build_query($query); + } + + $curl = curl_init($url); + if ($curl === false) { + throw new RuntimeException('Could not initialize GitHub API request.'); + } + + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_CUSTOMREQUEST, strtoupper($method)); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 3); + curl_setopt($curl, CURLOPT_TIMEOUT, 10); + curl_setopt($curl, CURLOPT_NOSIGNAL, true); + curl_setopt($curl, CURLOPT_HTTPHEADER, [ + 'Accept: application/vnd.github+json', + 'Authorization: Bearer ' . $token, + 'User-Agent: Truckwash-Release-Manager', + 'X-GitHub-Api-Version: 2022-11-28', + ]); + + $raw = curl_exec($curl); + $error = curl_error($curl); + $status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE); + curl_close($curl); + + if ($raw === false) { + throw new RuntimeException('GitHub API request failed: ' . $error); + } + + $decoded = []; + if (trim((string)$raw) !== '') { + $decodedJson = json_decode((string)$raw, true); + $decoded = is_array($decodedJson) ? $decodedJson : ['raw' => (string)$raw]; + } + + if ($status < 200 || $status >= 300) { + $message = is_array($decoded) + ? (string)($decoded['message'] ?? $decoded['error'] ?? ('HTTP ' . $status)) + : ('HTTP ' . $status); + throw new RuntimeException('GitHub API request failed: ' . $message); + } + + return $decoded; + } + + private function publicGithubRepository(array $row): array + { + $fullName = self::normalizeGithubRepositoryName((string)($row['full_name'] ?? '')); + return [ + 'id' => isset($row['id']) ? (int)$row['id'] : null, + 'name' => (string)($row['name'] ?? ''), + 'full_name' => $fullName, + 'private' => (bool)($row['private'] ?? false), + 'default_branch' => (string)($row['default_branch'] ?? 'main'), + 'description' => $row['description'] ?? null, + 'html_url' => $row['html_url'] ?? null, + 'clone_url' => $row['clone_url'] ?? null, + 'ssh_url' => $row['ssh_url'] ?? null, + 'pushed_at' => $row['pushed_at'] ?? null, + 'updated_at' => $row['updated_at'] ?? null, + ]; + } + + private function publicGithubBranch(array $row): array + { + return [ + 'name' => (string)($row['name'] ?? ''), + 'commit_sha' => $row['commit']['sha'] ?? null, + 'protected' => (bool)($row['protected'] ?? false), + ]; + } + + private function publicGithubCommit(array $row): array + { + $sha = (string)($row['sha'] ?? ''); + $message = (string)($row['commit']['message'] ?? ''); + $title = trim(strtok($message, "\n") ?: $message); + return [ + 'sha' => $sha, + 'short_sha' => substr($sha, 0, 12), + 'message' => $title, + 'author_name' => $row['commit']['author']['name'] ?? $row['author']['login'] ?? null, + 'authored_at' => $row['commit']['author']['date'] ?? null, + 'html_url' => $row['html_url'] ?? null, + ]; + } + + private function releaseSuggestions(): array + { + $channels = $this->listChannels(); + $targets = $this->listDeploymentTargets(); + $deployments = $this->listDeployments(50); + $versions = release_manager_schema_bootstrap::tablesExist() + ? $this->selectRows( + "SELECT app, repository, branch, deployed_url, build_url + FROM release_versions + WHERE repository IS NOT NULL OR branch IS NOT NULL OR deployed_url IS NOT NULL + ORDER BY created_at DESC + LIMIT 100" + ) + : []; + + $repositories = []; + $branches = ['main', 'master', 'develop', 'staging']; + $frontendUrls = []; + $apiUrls = []; + $healthUrls = []; + $loadBalancerDomains = []; + $serviceUuids = []; + + foreach ([ + $this->moduleConfigValue('Coolify', 'public_gateway_host', 'api-v2.truckwash.io'), + getenv('COOLIFY_PUBLIC_GATEWAY_HOST') ?: ($_SERVER['COOLIFY_PUBLIC_GATEWAY_HOST'] ?? null), + getenv('PUBLIC_GATEWAY_HOST') ?: ($_SERVER['PUBLIC_GATEWAY_HOST'] ?? null), + getenv('RELEASE_LOAD_BALANCER_DOMAIN') ?: ($_SERVER['RELEASE_LOAD_BALANCER_DOMAIN'] ?? null), + getenv('RELEASE_LOAD_BALANCER_DOMAINS') ?: ($_SERVER['RELEASE_LOAD_BALANCER_DOMAINS'] ?? null), + ] as $domain) { + $this->appendDomainSuggestion($loadBalancerDomains, $domain); + } + + foreach (array_merge($targets, $deployments, $versions) as $row) { + $this->appendSuggestion($repositories, $row['repository'] ?? null); + $this->appendSuggestion($branches, $row['branch'] ?? null); + $this->appendSuggestion($healthUrls, $row['health_url'] ?? null); + $this->appendSuggestion($healthUrls, $row['deployment_url'] ?? null); + $this->appendSuggestion($healthUrls, $row['deployed_url'] ?? null); + $this->appendSuggestion($serviceUuids, $row['coolify_service_uuid'] ?? null); + } + + foreach ($channels as $channel) { + $slug = (string)($channel['slug'] ?? ''); + $this->appendSuggestion($branches, $slug !== '' ? 'release/' . $slug : null); + $this->appendSuggestion($frontendUrls, $channel['frontend_base_url'] ?? null); + $this->appendSuggestion($apiUrls, $channel['api_base_url'] ?? null); + if (!empty($channel['frontend_base_url'])) { + $this->appendSuggestion($healthUrls, rtrim((string)$channel['frontend_base_url'], '/') . '/health'); + } + if (!empty($channel['api_base_url'])) { + $this->appendSuggestion($healthUrls, rtrim((string)$channel['api_base_url'], '/') . '/ping'); + } + } + + foreach ([ + 'GITHUB_REPOSITORY', + 'RELEASE_FRONTEND_REPOSITORY', + 'RELEASE_API_REPOSITORY', + 'FRONTEND_GITHUB_REPOSITORY', + 'API_GITHUB_REPOSITORY', + ] as $key) { + $this->appendSuggestion($repositories, getenv($key) ?: ($_SERVER[$key] ?? null)); + } + + foreach (['GITHUB_REF_NAME', 'RELEASE_BRANCH', 'FRONTEND_BRANCH', 'API_BRANCH'] as $key) { + $this->appendSuggestion($branches, getenv($key) ?: ($_SERVER[$key] ?? null)); + } + + foreach (['FRONTEND_URL', 'APP_URL', 'VITE_APP_URL'] as $key) { + $this->appendSuggestion($frontendUrls, getenv($key) ?: ($_SERVER[$key] ?? null)); + } + foreach (['API_URL', 'BACKEND_URL', 'PUBLIC_API_URL'] as $key) { + $this->appendSuggestion($apiUrls, getenv($key) ?: ($_SERVER[$key] ?? null)); + } + + $origin = trim((string)($_SERVER['HTTP_ORIGIN'] ?? '')); + if ($origin !== '') { + $this->appendSuggestion($frontendUrls, $origin); + } + $host = trim((string)($_SERVER['HTTP_HOST'] ?? '')); + if ($host !== '') { + $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http'; + $this->appendSuggestion($apiUrls, $scheme . '://' . $host); + } + + $coolifyInstances = []; + $coolifyProjects = []; + $coolifyServices = []; + if ($this->tableExists('coolify_instances')) { + $instanceRows = $this->selectRows( + 'SELECT id, label, base_url, api_token_secret, status, default_project_uuid, default_environment_uuid, default_environment_name, default_server_uuid + FROM coolify_instances + WHERE deleted_at IS NULL + ORDER BY status = \'ok\' DESC, label' + ); + $coolifyInstances = array_map(static function (array $row): array { + return [ + 'id' => (int)($row['id'] ?? 0), + 'label' => (string)($row['label'] ?? ''), + 'base_url' => (string)($row['base_url'] ?? ''), + 'status' => (string)($row['status'] ?? 'unknown'), + 'default_project_uuid' => $row['default_project_uuid'] ?? null, + 'default_environment_uuid' => $row['default_environment_uuid'] ?? null, + 'default_environment_name' => $row['default_environment_name'] ?? null, + 'default_server_uuid' => $row['default_server_uuid'] ?? null, + ]; + }, $instanceRows); + + foreach ($instanceRows as $instanceRow) { + foreach ($this->coolifyProjectSuggestions($instanceRow) as $project) { + $coolifyProjects[] = $project; + } + foreach ($this->coolifyServiceSuggestions($instanceRow) as $service) { + $coolifyServices[] = $service; + $this->appendSuggestion($serviceUuids, $service['uuid'] ?? null); + foreach (($service['urls'] ?? []) as $url) { + $this->appendSuggestion($healthUrls, $url); + } + } + } + } + + $channelPresets = [ + [ + 'slug' => 'stable', + 'name' => 'Stable', + 'description' => 'Default production release channel.', + 'rollout_percent' => 100, + 'default_channel' => true, + 'replay_enabled' => false, + 'capture_level' => 'metadata', + 'retention_days' => 14, + ], + [ + 'slug' => 'canary', + 'name' => 'Canary', + 'description' => 'Small early-access channel for validating a release before broad rollout.', + 'rollout_percent' => 5, + 'default_channel' => false, + 'replay_enabled' => true, + 'capture_level' => 'full_redacted', + 'retention_days' => 7, + ], + [ + 'slug' => 'beta', + 'name' => 'Beta', + 'description' => 'Customer or staff opt-in channel for release candidate validation.', + 'rollout_percent' => 0, + 'default_channel' => false, + 'replay_enabled' => true, + 'capture_level' => 'metadata', + 'retention_days' => 14, + ], + [ + 'slug' => 'internal', + 'name' => 'Internal', + 'description' => 'Staff-only channel for internal verification and support replay.', + 'rollout_percent' => 0, + 'default_channel' => false, + 'replay_enabled' => true, + 'capture_level' => 'full_redacted', + 'retention_days' => 14, + ], + ]; + + $frontendRepository = $this->firstSuggestion($repositories, ['front-end', 'frontend', 'vue']) ?? ($repositories[0] ?? ''); + $apiRepository = $this->firstSuggestion($repositories, ['backend', 'api', 'php']) ?? ($repositories[1] ?? $repositories[0] ?? ''); + + return [ + 'github_token_configured' => $this->hasGithubApiToken(), + 'github_api_url' => $this->githubApiBaseUrl(), + 'repositories' => array_values($repositories), + 'branches' => array_values($branches), + 'frontend_base_urls' => array_values($frontendUrls), + 'api_base_urls' => array_values($apiUrls), + 'health_urls' => array_values($healthUrls), + 'load_balancer_domains' => array_values($loadBalancerDomains), + 'coolify_instances' => $coolifyInstances, + 'coolify_projects' => $coolifyProjects, + 'coolify_services' => $coolifyServices, + 'coolify_service_uuids' => array_values($serviceUuids), + 'channel_presets' => $channelPresets, + 'target_presets' => [ + [ + 'label' => 'Frontend target', + 'app' => 'frontend', + 'repository' => $frontendRepository, + 'branch' => $branches[0] ?? 'main', + 'health_url' => $healthUrls[0] ?? '', + 'auto_deploy' => true, + ], + [ + 'label' => 'API target', + 'app' => 'api', + 'repository' => $apiRepository, + 'branch' => $branches[0] ?? 'main', + 'health_url' => $this->firstSuggestion($healthUrls, ['/ping']) + ?? $this->firstSuggestion($healthUrls, ['/health']) + ?? '', + 'auto_deploy' => true, + ], + ], + 'setup_steps' => [ + ['key' => 'channels', 'done' => count($channels) > 0], + ['key' => 'targets', 'done' => count($targets) > 0], + ['key' => 'deployments', 'done' => count($deployments) > 0], + ['key' => 'timeline', 'done' => (int)($this->timelineSummary()['events'] ?? 0) > 0], + ], + ]; + } + + private function appendSuggestion(array &$values, mixed $value): void + { + $value = trim((string)($value ?? '')); + if ($value === '') { + return; + } + foreach (preg_split('/\s*,\s*/', $value) ?: [] as $part) { + $part = trim($part); + if ($part !== '' && !in_array($part, $values, true)) { + $values[] = $part; + } + } + } + + private function appendDomainSuggestion(array &$values, mixed $value): void + { + $value = trim((string)($value ?? '')); + if ($value === '') { + return; + } + + foreach (preg_split('/\s*,\s*/', $value) ?: [] as $part) { + $domain = self::domainSuggestionHost($part); + if ($domain !== null && !in_array($domain, $values, true)) { + $values[] = $domain; + } + } + } + + private static function domainSuggestionHost(mixed $value): ?string + { + $raw = trim((string)($value ?? '')); + if ($raw === '') { + return null; + } + + $candidate = preg_match('#^https?://#i', $raw) === 1 ? $raw : 'https://' . $raw; + $host = parse_url($candidate, PHP_URL_HOST); + $port = parse_url($candidate, PHP_URL_PORT); + $host = strtolower(trim((string)$host, "[] \t\n\r\0\x0B.")); + + if ( + $host === '' + || $port !== null + || $host === 'localhost' + || str_ends_with($host, '.localhost') + || str_contains($host, '/') + || filter_var($host, FILTER_VALIDATE_IP) !== false + ) { + return null; + } + + return $host; + } + + private function firstSuggestion(array $values, array $needles): ?string + { + foreach ($values as $value) { + foreach ($needles as $needle) { + if (stripos((string)$value, $needle) !== false) { + return (string)$value; + } + } + } + return null; + } + + private function coolifyProjectSuggestions(array $instance): array + { + $tokenSecret = trim((string)($instance['api_token_secret'] ?? '')); + if ($tokenSecret === '') { + return []; + } + + try { + $token = replication_secret_box::decrypt($tokenSecret); + $projects = (new coolify_api_client((string)($instance['base_url'] ?? ''), $token, 4))->listProjects(); + } catch (Throwable) { + return []; + } + + $suggestions = []; + foreach ($this->payloadRows($projects) as $row) { + if (!is_array($row)) { + continue; + } + $uuid = trim((string)($row['uuid'] ?? '')); + if ($uuid === '') { + continue; + } + $suggestions[] = [ + 'instance_id' => (int)($instance['id'] ?? 0), + 'instance_label' => (string)($instance['label'] ?? ''), + 'uuid' => $uuid, + 'name' => (string)($row['name'] ?? $uuid), + 'description' => (string)($row['description'] ?? ''), + 'default' => $uuid === trim((string)($instance['default_project_uuid'] ?? '')), + ]; + } + + return array_slice($suggestions, 0, 50); + } + + private function coolifyServiceSuggestions(array $instance): array + { + $tokenSecret = trim((string)($instance['api_token_secret'] ?? '')); + if ($tokenSecret === '') { + return []; + } + + try { + $token = replication_secret_box::decrypt($tokenSecret); + $services = (new coolify_api_client((string)($instance['base_url'] ?? ''), $token, 4))->listServices(); + } catch (Throwable) { + return []; + } + + $rows = $this->payloadRows($services); + $suggestions = []; + foreach ($rows as $row) { + if (!is_array($row)) { + continue; + } + $uuid = trim((string)($row['uuid'] ?? $row['id'] ?? '')); + if ($uuid === '') { + continue; + } + $urls = []; + foreach (['fqdn', 'domain', 'url'] as $key) { + $this->appendSuggestion($urls, $row[$key] ?? null); + } + foreach (['urls', 'domains'] as $key) { + if (!is_array($row[$key] ?? null)) { + continue; + } + foreach ($row[$key] as $url) { + if (is_array($url)) { + $this->appendSuggestion($urls, $url['url'] ?? $url['domain'] ?? $url['fqdn'] ?? null); + } else { + $this->appendSuggestion($urls, $url); + } + } + } + + $suggestions[] = [ + 'instance_id' => (int)($instance['id'] ?? 0), + 'instance_label' => (string)($instance['label'] ?? ''), + 'uuid' => $uuid, + 'name' => (string)($row['name'] ?? $row['service_name'] ?? $uuid), + 'status' => (string)($row['status'] ?? $row['deployment_status'] ?? 'unknown'), + 'urls' => array_values($urls), + ]; + } + + return array_slice($suggestions, 0, 50); + } + + private function payloadRows(array $payload): array + { + if ($payload === []) { + return []; + } + if (array_keys($payload) === range(0, count($payload) - 1)) { + return $payload; + } + foreach (['data', 'services', 'projects', 'servers', 'results'] as $key) { + if (is_array($payload[$key] ?? null)) { + return $this->payloadRows($payload[$key]); + } + } + return []; + } + + private function normalizeChannelInput(array $input, bool $creating): array + { + $slug = self::safeSlug((string)($input['slug'] ?? '')); + if ($slug === '') { + throw new RuntimeException('Release channel slug is required.'); + } + + $name = trim((string)($input['name'] ?? ($creating ? '' : $slug))); + if ($name === '') { + throw new RuntimeException('Release channel name is required.'); + } + + $retention = (int)($input['retention_days'] ?? 14); + return [ + 'slug' => $slug, + 'name' => substr($name, 0, 128), + 'description' => trim((string)($input['description'] ?? '')) ?: null, + 'enabled' => $this->toBool($input['enabled'] ?? true) ? 1 : 0, + 'default_channel' => $this->toBool($input['default_channel'] ?? false) ? 1 : 0, + 'rollout_percent' => max(0, min(100, (float)($input['rollout_percent'] ?? 0))), + 'frontend_base_url' => trim((string)($input['frontend_base_url'] ?? '')) ?: null, + 'api_base_url' => trim((string)($input['api_base_url'] ?? '')) ?: null, + 'replay_enabled' => $this->toBool($input['replay_enabled'] ?? false) ? 1 : 0, + 'capture_level' => $this->normalizeCaptureLevel((string)($input['capture_level'] ?? 'metadata')), + 'retention_days' => max(1, min(365, $retention > 0 ? $retention : 14)), + 'metadata' => is_array($input['metadata'] ?? null) ? $input['metadata'] : self::jsonDecode($input['metadata_json'] ?? null), + ]; + } + + private function channelFromInput(array $input): array + { + $id = $this->nullablePositiveInt($input['channel_id'] ?? null); + if ($id !== null) { + return $this->getChannel($id); + } + + $slug = self::safeSlug((string)($input['channel_slug'] ?? $input['channel'] ?? '')); + if ($slug !== '') { + $channel = $this->findChannelBySlug($slug); + if ($channel !== null) { + return $channel; + } + } + + throw new RuntimeException('Release channel is required.'); + } + + private function deploymentTargetFromInput(array $input, int $channelId, string $app): ?array + { + $targetId = $this->nullablePositiveInt($input['target_id'] ?? null); + if ($targetId !== null) { + return $this->getDeploymentTarget($targetId); + } + + return $this->selectOne( + "SELECT * FROM release_deployment_targets + WHERE deleted_at IS NULL AND channel_id = ? AND app = ? + ORDER BY auto_deploy DESC, id DESC + LIMIT 1", + 'is', + [$channelId, $app] + ); + } + + private function normalizeServiceSetMode(string $value): string + { + $mode = strtolower(trim($value)); + if (!in_array($mode, self::SERVICE_SET_MODES, true)) { + throw new RuntimeException('Release service set mode must be attach_existing, clone_existing, fresh_empty, or isolated_stack.'); + } + return $mode; + } + + private function channelFromInputOrDefault(array $input, ?array $source = null): array + { + foreach (['channel_id', 'channel_slug', 'channel'] as $key) { + if (array_key_exists($key, $input) && trim((string)$input[$key]) !== '') { + return $this->channelFromInput($input); + } + } + + if ($source !== null && !empty($source['channel_id'])) { + return $this->getChannel((int)$source['channel_id']); + } + + return $this->defaultChannel(); + } + + private function serviceSetTargetIdFromInput(array $input, string $app, ?array $source): ?int + { + $aliases = $app === 'api' + ? ['api_target_id', 'php_target_id', 'backend_target_id'] + : ['frontend_target_id']; + $targets = is_array($input['targets'] ?? null) ? $input['targets'] : []; + if (is_array($targets[$app] ?? null)) { + foreach (['target_id', 'id'] as $key) { + $aliases[] = $app . '.' . $key; + } + } + + foreach ($aliases as $key) { + $value = str_contains($key, '.') + ? ($targets[$app][substr($key, strpos($key, '.') + 1)] ?? null) + : ($input[$key] ?? null); + $id = $this->nullablePositiveInt($value); + if ($id === null) { + continue; + } + $target = $this->getDeploymentTarget($id); + if ((string)($target['app'] ?? '') !== $app) { + throw new RuntimeException(sprintf('Selected %s target does not match the requested app.', $app)); + } + return $id; + } + + $sourceKey = $app . '_target_id'; + return $source !== null ? $this->nullablePositiveInt($source[$sourceKey] ?? null) : null; + } + + private function serviceSetDataTargetIdFromInput(array $input, string $kind, ?array $source): ?int + { + $dataTargets = is_array($input['data_targets'] ?? null) ? $input['data_targets'] : []; + $value = $input[$kind . '_coolify_target_id'] + ?? $input[$kind . '_target_id'] + ?? $dataTargets[$kind . '_coolify_target_id'] + ?? $dataTargets[$kind . '_target_id'] + ?? $dataTargets[$kind] + ?? null; + $id = $this->nullablePositiveInt($value); + if ($id !== null) { + $target = $this->nullableCoolifyTarget($id); + if ($target !== null && (string)($target['kind'] ?? '') !== $kind) { + throw new RuntimeException(sprintf('Selected %s data service target has the wrong replica kind.', $kind)); + } + return $id; + } + + $sourceKey = $kind . '_coolify_target_id'; + return $source !== null ? $this->nullablePositiveInt($source[$sourceKey] ?? null) : null; + } + + private function assertIsolatedStackTarget(?int $targetId, string $app): void + { + if ($targetId === null) { + throw new RuntimeException(sprintf('Isolated stack deployments require a new %s Coolify target.', $app)); + } + + $target = $this->getDeploymentTarget($targetId); + if ((string)($target['app'] ?? '') !== $app) { + throw new RuntimeException(sprintf('Isolated stack %s target does not match the requested app.', $app)); + } + if ($this->nullablePositiveInt($target['coolify_instance_id'] ?? null) === null) { + throw new RuntimeException(sprintf('Isolated stack %s target must select a Coolify instance.', $app)); + } + if (trim((string)($target['coolify_service_uuid'] ?? '')) !== '') { + throw new RuntimeException(sprintf('Isolated stack %s target must not point at an existing Coolify service.', $app)); + } + + $context = self::jsonDecode($target['deploy_context_json'] ?? null); + if (!$this->toBool($context['coolify_auto_create'] ?? false)) { + throw new RuntimeException(sprintf('Isolated stack %s target must create a new Coolify service.', $app)); + } + } + + private function uniqueServiceSetSlug(string $slug): string + { + $base = $slug !== '' ? $slug : 'service-set-' . date('Ymd-His'); + $candidate = substr($base, 0, 64); + $suffix = 2; + while ($this->selectOne('SELECT id FROM release_service_sets WHERE slug = ? LIMIT 1', 's', [$candidate]) !== null) { + $tail = '-' . $suffix; + $candidate = substr($base, 0, 64 - strlen($tail)) . $tail; + $suffix++; + } + return $candidate; + } + + private function serviceSetStatus(string $mode, ?int $frontendTargetId, ?int $apiTargetId, array $dataTargets): string + { + $hasCode = $frontendTargetId !== null && $apiTargetId !== null; + $hasData = !in_array(null, $dataTargets, true); + if ($mode === 'isolated_stack') { + return $hasCode ? 'isolated_stack' : 'needs_isolated_targets'; + } + if ($hasCode && $hasData) { + return $mode === 'clone_existing' ? 'provisioning' : 'ready'; + } + if ($mode === 'fresh_empty') { + return 'isolated_empty'; + } + return $mode === 'clone_existing' ? 'needs_clone_targets' : 'needs_configuration'; + } + + private function replicaProvisioningPlan(string $mode, ?array $source, array $dataTargets): array + { + $plan = []; + foreach (self::STACK_DATA_KINDS as $kind) { + $sourceTargetId = $source !== null ? $this->nullablePositiveInt($source[$kind . '_coolify_target_id'] ?? null) : null; + $sourceTarget = $this->nullableCoolifyTarget($sourceTargetId); + $target = $this->nullableCoolifyTarget($dataTargets[$kind] ?? null); + $plan[$kind] = [ + 'action' => match ($mode) { + 'clone_existing' => 'clone_replica_from_source', + 'isolated_stack' => 'create_isolated_empty_stack_service', + 'fresh_empty' => 'register_isolated_empty_service', + default => 'attach_existing_service', + }, + 'source_coolify_target_id' => $sourceTargetId, + 'source_replication_host_id' => $sourceTarget['replication']['id'] ?? null, + 'target_coolify_target_id' => $dataTargets[$kind] ?? null, + 'target_replication_host_id' => $target['replication']['id'] ?? null, + 'production_replication_attached' => $mode === 'attach_existing', + ]; + } + + return $plan; + } + + private function bundleAppInput(array $input, string $app, ?array $target, string $versionLabel): array + { + $appPayload = is_array($input[$app] ?? null) ? $input[$app] : []; + if ($app === 'api') { + $appPayload = array_replace( + is_array($input['php'] ?? null) ? $input['php'] : [], + is_array($input['backend'] ?? null) ? $input['backend'] : [], + $appPayload + ); + } + + $repository = trim((string)($appPayload['repository'] ?? $input[$app . '_repository'] ?? $target['repository'] ?? '')); + $normalizedRepository = self::normalizeGithubRepositoryName($repository); + if ($normalizedRepository !== '') { + $repository = $normalizedRepository; + } + if ($repository === '') { + throw new RuntimeException(sprintf('%s repository is required for bundle releases.', $app === 'api' ? 'PHP backend' : 'Frontend')); + } + + $branch = trim((string)($appPayload['branch'] ?? $input[$app . '_branch'] ?? $target['branch'] ?? 'main')) ?: 'main'; + $rawCommitSha = trim((string)($appPayload['commit_sha'] ?? $appPayload['commit'] ?? $input[$app . '_commit_sha'] ?? '')); + $commitMode = $this->normalizeCommitMode((string)($appPayload['commit_mode'] ?? $input[$app . '_commit_mode'] ?? ''), $rawCommitSha); + $commitSha = $commitMode === 'specific' && $rawCommitSha !== '' ? $rawCommitSha : null; + $githubAccess = $this->githubRepositoryAccess([ + 'repository' => $repository, + 'branch' => $branch, + 'commit_sha' => $commitSha, + 'commit_mode' => $commitMode, + ]); + if (($githubAccess['token_configured'] ?? false) && !($githubAccess['ok'] ?? false)) { + throw new RuntimeException('GitHub repository access test failed: ' . (string)($githubAccess['message'] ?? 'Repository is not accessible.')); + } + if (($githubAccess['ok'] ?? false) && !empty($githubAccess['commit_sha'])) { + $commitSha = (string)$githubAccess['commit_sha']; + $branch = (string)($githubAccess['branch'] ?? $branch); + } + + return [ + 'app' => $app, + 'repository' => $repository, + 'branch' => $branch, + 'commit_mode' => $commitMode, + 'commit_sha' => $commitSha, + 'version_label' => sprintf('%s-%s', $versionLabel, $app === 'api' ? 'php' : 'frontend'), + 'deployed_url' => $target['health_url'] ?? null, + 'github_access' => $githubAccess, + ]; + } + + private function createBundleVersion(array $input, string $app): int + { + return $this->createVersion([ + 'app' => $app, + 'repository' => $input['repository'], + 'branch' => $input['branch'], + 'commit_sha' => $input['commit_sha'], + 'version_label' => $input['version_label'], + 'deployed_url' => $input['deployed_url'] ?? null, + 'status' => 'draft', + 'metadata' => [ + 'commit_mode' => $input['commit_mode'], + 'github_access' => $input['github_access'], + 'bundle_member' => true, + ], + ]); + } + + private function getChannel(int $id): array + { + $row = $this->selectOne('SELECT * FROM release_channels WHERE id = ? AND deleted_at IS NULL', 'i', [$id]); + if ($row === null) { + throw new RuntimeException('Release channel not found.'); + } + return $row; + } + + private function findChannelBySlug(string $slug): ?array + { + return $this->selectOne( + 'SELECT * FROM release_channels WHERE slug = ? AND deleted_at IS NULL LIMIT 1', + 's', + [$slug] + ); + } + + private function getVersion(int $id): array + { + return $this->selectOne('SELECT * FROM release_versions WHERE id = ?', 'i', [$id]) ?? []; + } + + private function getDeployment(int $id): array + { + $row = $this->selectOne( + "SELECT d.*, c.slug AS channel_slug, c.name AS channel_name, v.version_label, v.deployed_url + FROM release_deployments d + INNER JOIN release_channels c ON c.id = d.channel_id + LEFT JOIN release_versions v ON v.id = d.version_id + WHERE d.id = ?", + 'i', + [$id] + ); + if ($row === null) { + throw new RuntimeException('Release deployment not found.'); + } + return $row; + } + + private function getServiceSet(int $id): array + { + $row = $this->selectOne( + "SELECT s.*, c.slug AS channel_slug, c.name AS channel_name + FROM release_service_sets s + LEFT JOIN release_channels c ON c.id = s.channel_id + WHERE s.id = ? AND s.deleted_at IS NULL", + 'i', + [$id] + ); + if ($row === null) { + throw new RuntimeException('Release service set not found.'); + } + return $row; + } + + private function getBundle(int $id): array + { + $row = $this->selectOne( + "SELECT b.*, c.slug AS channel_slug, c.name AS channel_name, s.name AS service_set_name, s.slug AS service_set_slug + FROM release_bundles b + INNER JOIN release_channels c ON c.id = b.channel_id + INNER JOIN release_service_sets s ON s.id = b.service_set_id + WHERE b.id = ? AND b.deleted_at IS NULL", + 'i', + [$id] + ); + if ($row === null) { + throw new RuntimeException('Release bundle not found.'); + } + return $row; + } + + private function getDeploymentTarget(int $id): array + { + $row = $this->selectOne( + "SELECT t.*, c.slug AS channel_slug, c.name AS channel_name, i.label AS coolify_instance_label + FROM release_deployment_targets t + INNER JOIN release_channels c ON c.id = t.channel_id + LEFT JOIN coolify_instances i ON i.id = t.coolify_instance_id + WHERE t.id = ? AND t.deleted_at IS NULL", + 'i', + [$id] + ); + if ($row === null) { + throw new RuntimeException('Release deployment target not found.'); + } + return $row; + } + + private function publicChannel(array $channel): array + { + return [ + 'id' => (int)($channel['id'] ?? 0), + 'slug' => (string)($channel['slug'] ?? ''), + 'name' => (string)($channel['name'] ?? ''), + 'description' => $channel['description'] ?? null, + 'enabled' => (bool)((int)($channel['enabled'] ?? 0)), + 'default_channel' => (bool)((int)($channel['default_channel'] ?? 0)), + 'rollout_percent' => (float)($channel['rollout_percent'] ?? 0), + 'frontend_base_url' => $channel['frontend_base_url'] ?? null, + 'api_base_url' => $channel['api_base_url'] ?? null, + 'replay_enabled' => (bool)((int)($channel['replay_enabled'] ?? 0)), + 'capture_level' => (string)($channel['capture_level'] ?? 'metadata'), + 'retention_days' => (int)($channel['retention_days'] ?? 14), + 'metadata' => self::jsonDecode($channel['metadata_json'] ?? null), + 'created_at' => $channel['created_at'] ?? null, + 'updated_at' => $channel['updated_at'] ?? null, + ]; + } + + private function publicVersion(?array $version): ?array + { + if (!$version || empty($version['id'])) { + return null; + } + + return [ + 'id' => (int)$version['id'], + 'app' => (string)$version['app'], + 'repository' => $version['repository'] ?? null, + 'branch' => $version['branch'] ?? null, + 'commit_sha' => $version['commit_sha'] ?? null, + 'tag' => $version['tag'] ?? null, + 'version_label' => $version['version_label'] ?? null, + 'build_url' => $version['build_url'] ?? null, + 'artifact_url' => $version['artifact_url'] ?? null, + 'deployed_url' => $version['deployed_url'] ?? null, + 'status' => (string)($version['status'] ?? 'unknown'), + 'metadata' => self::jsonDecode($version['metadata_json'] ?? null), + 'created_at' => $version['created_at'] ?? null, + 'deployed_at' => $version['deployed_at'] ?? null, + ]; + } + + private function publicAssignment(array $assignment): array + { + return [ + 'id' => (int)($assignment['id'] ?? 0), + 'subject_type' => (string)($assignment['subject_type'] ?? ''), + 'subject_id' => (string)($assignment['subject_id'] ?? ''), + 'channel_id' => (int)($assignment['channel_id'] ?? 0), + 'channel_slug' => (string)($assignment['channel_slug'] ?? ''), + 'channel_name' => (string)($assignment['channel_name'] ?? ''), + 'reason' => $assignment['reason'] ?? null, + 'expires_at' => $assignment['expires_at'] ?? null, + 'actor_user_id' => isset($assignment['actor_user_id']) ? (int)$assignment['actor_user_id'] : null, + 'created_at' => $assignment['created_at'] ?? null, + ]; + } + + private function publicDeploymentTarget(array $target): array + { + return [ + 'id' => (int)($target['id'] ?? 0), + 'channel_id' => (int)($target['channel_id'] ?? 0), + 'channel_slug' => (string)($target['channel_slug'] ?? ''), + 'channel_name' => (string)($target['channel_name'] ?? ''), + 'app' => (string)($target['app'] ?? ''), + 'coolify_instance_id' => isset($target['coolify_instance_id']) ? (int)$target['coolify_instance_id'] : null, + 'coolify_instance_label' => $target['coolify_instance_label'] ?? null, + 'coolify_service_uuid' => $target['coolify_service_uuid'] ?? null, + 'repository' => (string)($target['repository'] ?? ''), + 'branch' => (string)($target['branch'] ?? ''), + 'auto_deploy' => (bool)((int)($target['auto_deploy'] ?? 0)), + 'health_url' => $target['health_url'] ?? null, + 'deploy_context' => self::jsonDecode($target['deploy_context_json'] ?? null), + 'created_at' => $target['created_at'] ?? null, + 'updated_at' => $target['updated_at'] ?? null, + ]; + } + + private function publicServiceSet(array $serviceSet, bool $includeBundles = true): array + { + $dataServices = []; + foreach (self::STACK_DATA_KINDS as $kind) { + $dataServices[$kind] = $this->nullableCoolifyTarget( + $this->nullablePositiveInt($serviceSet[$kind . '_coolify_target_id'] ?? null) + ); + } + + $attachedBundles = $includeBundles ? $this->serviceSetBundles((int)($serviceSet['id'] ?? 0)) : []; + + return [ + 'id' => (int)($serviceSet['id'] ?? 0), + 'channel_id' => isset($serviceSet['channel_id']) ? (int)$serviceSet['channel_id'] : null, + 'channel_slug' => $serviceSet['channel_slug'] ?? null, + 'channel_name' => $serviceSet['channel_name'] ?? null, + 'name' => (string)($serviceSet['name'] ?? ''), + 'slug' => (string)($serviceSet['slug'] ?? ''), + 'mode' => (string)($serviceSet['mode'] ?? 'attach_existing'), + 'source_service_set_id' => isset($serviceSet['source_service_set_id']) ? (int)$serviceSet['source_service_set_id'] : null, + 'status' => (string)($serviceSet['status'] ?? 'unknown'), + 'targets' => [ + 'frontend' => $this->nullableDeploymentTarget($this->nullablePositiveInt($serviceSet['frontend_target_id'] ?? null)), + 'api' => $this->nullableDeploymentTarget($this->nullablePositiveInt($serviceSet['api_target_id'] ?? null)), + ], + 'data_services' => $dataServices, + 'stack' => [ + 'frontend' => $this->nullableDeploymentTarget($this->nullablePositiveInt($serviceSet['frontend_target_id'] ?? null)), + 'api' => $this->nullableDeploymentTarget($this->nullablePositiveInt($serviceSet['api_target_id'] ?? null)), + 'database' => $dataServices['database'], + 'redis' => $dataServices['redis'], + 'minio' => $dataServices['minio'], + ], + 'health' => self::jsonDecode($serviceSet['health_json'] ?? null), + 'metadata' => self::jsonDecode($serviceSet['metadata_json'] ?? null), + 'attached_bundle_count' => count($attachedBundles), + 'attached_bundles' => $attachedBundles, + 'created_at' => $serviceSet['created_at'] ?? null, + 'updated_at' => $serviceSet['updated_at'] ?? null, + ]; + } + + private function publicBundle(array $bundle, bool $includeServiceSet = true): array + { + $frontendVersionId = $this->nullablePositiveInt($bundle['frontend_version_id'] ?? null); + $apiVersionId = $this->nullablePositiveInt($bundle['api_version_id'] ?? null); + $frontendDeploymentId = $this->nullablePositiveInt($bundle['frontend_deployment_id'] ?? null); + $apiDeploymentId = $this->nullablePositiveInt($bundle['api_deployment_id'] ?? null); + + return [ + 'id' => (int)($bundle['id'] ?? 0), + 'channel_id' => (int)($bundle['channel_id'] ?? 0), + 'channel_slug' => (string)($bundle['channel_slug'] ?? ''), + 'channel_name' => (string)($bundle['channel_name'] ?? ''), + 'service_set_id' => (int)($bundle['service_set_id'] ?? 0), + 'service_set_name' => $bundle['service_set_name'] ?? null, + 'service_set_slug' => $bundle['service_set_slug'] ?? null, + 'service_set' => $includeServiceSet ? $this->publicServiceSet($this->getServiceSet((int)$bundle['service_set_id']), false) : null, + 'version_label' => $bundle['version_label'] ?? null, + 'status' => (string)($bundle['status'] ?? 'draft'), + 'apps' => [ + 'frontend' => [ + 'version_id' => $frontendVersionId, + 'deployment_id' => $frontendDeploymentId, + 'repository' => $bundle['frontend_repository'] ?? null, + 'branch' => $bundle['frontend_branch'] ?? null, + 'commit_sha' => $bundle['frontend_commit_sha'] ?? null, + 'version' => $frontendVersionId !== null ? $this->publicVersion($this->getVersion($frontendVersionId)) : null, + 'deployment' => $this->nullableDeployment($frontendDeploymentId), + ], + 'api' => [ + 'version_id' => $apiVersionId, + 'deployment_id' => $apiDeploymentId, + 'repository' => $bundle['api_repository'] ?? null, + 'branch' => $bundle['api_branch'] ?? null, + 'commit_sha' => $bundle['api_commit_sha'] ?? null, + 'version' => $apiVersionId !== null ? $this->publicVersion($this->getVersion($apiVersionId)) : null, + 'deployment' => $this->nullableDeployment($apiDeploymentId), + ], + ], + 'deployment_result' => self::jsonDecode($bundle['deployment_result_json'] ?? null), + 'metadata' => self::jsonDecode($bundle['metadata_json'] ?? null), + 'actor_user_id' => isset($bundle['actor_user_id']) ? (int)$bundle['actor_user_id'] : null, + 'deployed_at' => $bundle['deployed_at'] ?? null, + 'promoted_at' => $bundle['promoted_at'] ?? null, + 'created_at' => $bundle['created_at'] ?? null, + 'updated_at' => $bundle['updated_at'] ?? null, + ]; + } + + private function serviceSetBundles(int $serviceSetId): array + { + if ($serviceSetId <= 0 || !release_manager_schema_bootstrap::tablesExist()) { + return []; + } + + return array_map( + fn(array $row): array => $this->publicBundle($row, false), + $this->selectRows( + "SELECT b.*, c.slug AS channel_slug, c.name AS channel_name, s.name AS service_set_name, s.slug AS service_set_slug + FROM release_bundles b + INNER JOIN release_channels c ON c.id = b.channel_id + INNER JOIN release_service_sets s ON s.id = b.service_set_id + WHERE b.deleted_at IS NULL AND b.service_set_id = ? + ORDER BY b.created_at DESC, b.id DESC + LIMIT 10", + 'i', + [$serviceSetId] + ) + ); + } + + private function nullableDeploymentTarget(?int $id): ?array + { + if ($id === null || $id <= 0) { + return null; + } + + try { + return $this->publicDeploymentTarget($this->getDeploymentTarget($id)); + } catch (Throwable) { + return null; + } + } + + private function nullableDeployment(?int $id): ?array + { + if ($id === null || $id <= 0) { + return null; + } + + try { + return $this->publicDeployment($this->getDeployment($id)); + } catch (Throwable) { + return null; + } + } + + private function nullableCoolifyTarget(?int $id): ?array + { + if ($id === null || $id <= 0 || !$this->tableExists('coolify_targets')) { + return null; + } + + $hasReplicationHosts = $this->tableExists('replication_hosts'); + $replicationColumns = $hasReplicationHosts + ? "h.id AS host_id, h.kind AS host_kind, h.label AS host_label, h.host AS host_host, + h.port AS host_port, h.role AS host_role, h.status AS host_status, + h.replication_source_id AS host_replication_source_id, + h.last_status_json AS host_last_status_json, h.last_checked_at AS host_last_checked_at" + : "NULL AS host_id, NULL AS host_kind, NULL AS host_label, NULL AS host_host, + NULL AS host_port, NULL AS host_role, NULL AS host_status, + NULL AS host_replication_source_id, + NULL AS host_last_status_json, NULL AS host_last_checked_at"; + $replicationJoin = $hasReplicationHosts ? 'LEFT JOIN replication_hosts h ON h.id = t.replication_host_id' : ''; + + $row = $this->selectOne( + "SELECT t.*, i.label AS instance_label, i.base_url AS instance_base_url, + $replicationColumns + FROM coolify_targets t + LEFT JOIN coolify_instances i ON i.id = t.instance_id + $replicationJoin + WHERE t.id = ? AND t.deleted_at IS NULL", + 'i', + [$id] + ); + if ($row === null) { + return null; + } + + return [ + 'id' => (int)($row['id'] ?? 0), + 'kind' => (string)($row['kind'] ?? ''), + 'label' => (string)($row['label'] ?? ''), + 'role' => (string)($row['role'] ?? ''), + 'instance_id' => isset($row['instance_id']) ? (int)$row['instance_id'] : null, + 'instance_label' => $row['instance_label'] ?? null, + 'resource_uuid' => $row['resource_uuid'] ?? null, + 'resource_name' => $row['resource_name'] ?? null, + 'deployment_status' => (string)($row['deployment_status'] ?? 'unknown'), + 'availability_state' => (string)($row['availability_state'] ?? 'unknown'), + 'last_reconcile_status' => $row['last_reconcile_status'] ?? null, + 'last_reconciled_at' => $row['last_reconciled_at'] ?? null, + 'replication' => !empty($row['host_id']) ? [ + 'id' => (int)$row['host_id'], + 'kind' => (string)($row['host_kind'] ?? $row['kind'] ?? ''), + 'label' => (string)($row['host_label'] ?? ''), + 'host' => $row['host_host'] ?? null, + 'port' => isset($row['host_port']) ? (int)$row['host_port'] : null, + 'role' => (string)($row['host_role'] ?? 'unknown'), + 'status' => (string)($row['host_status'] ?? 'unknown'), + 'source_host_id' => isset($row['host_replication_source_id']) ? (int)$row['host_replication_source_id'] : null, + 'last_status' => self::jsonDecode($row['host_last_status_json'] ?? null), + 'last_checked_at' => $row['host_last_checked_at'] ?? null, + ] : null, + 'options' => self::jsonDecode($row['options_json'] ?? null), + ]; + } + + private function publicDeployment(array $deployment): array + { + $status = (string)($deployment['status'] ?? 'unknown'); + $result = self::jsonDecode($deployment['result_json'] ?? null); + $failureSummary = is_array($result['failure_summary'] ?? null) ? $result['failure_summary'] : null; + $promotable = self::deploymentCanBePromoted($status); + + return [ + 'id' => (int)($deployment['id'] ?? 0), + 'channel_id' => (int)($deployment['channel_id'] ?? 0), + 'channel_slug' => (string)($deployment['channel_slug'] ?? ''), + 'channel_name' => (string)($deployment['channel_name'] ?? ''), + 'target_id' => isset($deployment['target_id']) ? (int)$deployment['target_id'] : null, + 'version_id' => isset($deployment['version_id']) ? (int)$deployment['version_id'] : null, + 'service_set_id' => isset($deployment['service_set_id']) ? (int)$deployment['service_set_id'] : null, + 'bundle_id' => isset($deployment['bundle_id']) ? (int)$deployment['bundle_id'] : null, + 'deployment_kind' => (string)($deployment['deployment_kind'] ?? 'single_app'), + 'version_label' => $deployment['version_label'] ?? null, + 'app' => (string)($deployment['app'] ?? ''), + 'provider' => (string)($deployment['provider'] ?? 'coolify'), + 'repository' => $deployment['repository'] ?? null, + 'branch' => $deployment['branch'] ?? null, + 'commit_sha' => $deployment['commit_sha'] ?? null, + 'status' => $status, + 'deployment_url' => $deployment['deployment_url'] ?? $deployment['deployed_url'] ?? null, + 'actor_user_id' => isset($deployment['actor_user_id']) ? (int)$deployment['actor_user_id'] : null, + 'result' => $result, + 'failure_summary' => $failureSummary, + 'error_message' => $deployment['error_message'] ?? null, + 'promotable' => $promotable, + 'promotion_blocked_reason' => $promotable ? null : self::deploymentPromotionBlockedReason($deployment), + 'started_at' => $deployment['started_at'] ?? null, + 'completed_at' => $deployment['completed_at'] ?? null, + 'created_at' => $deployment['created_at'] ?? null, + 'updated_at' => $deployment['updated_at'] ?? null, + ]; + } + + private function publicTimelineEvent(array $event): array + { + return [ + 'id' => (int)($event['id'] ?? 0), + 'timeline_session_id' => isset($event['timeline_session_id']) ? (int)$event['timeline_session_id'] : null, + 'trace_id' => (string)($event['trace_id'] ?? ''), + 'event_type' => (string)($event['event_type'] ?? ''), + 'severity' => (string)($event['severity'] ?? 'info'), + 'module_key' => $event['module_key'] ?? null, + 'route_path' => $event['route_path'] ?? null, + 'component' => $event['component'] ?? null, + 'request_id' => $event['request_id'] ?? null, + 'occurred_at' => $event['occurred_at'] ?? null, + 'payload' => self::jsonDecode($event['payload_json'] ?? null), + 'principal_type' => $event['principal_type'] ?? null, + 'principal_id' => $event['principal_id'] ?? null, + 'customer_number' => isset($event['customer_number']) ? (int)$event['customer_number'] : null, + 'channel_slug' => $event['channel_slug'] ?? null, + ]; + } + + private function clearOtherDefaultChannels(int $channelId): void + { + $this->execute('UPDATE release_channels SET default_channel = 0 WHERE id <> ?', 'i', [$channelId]); + } + + private function normalizeApp(string $value): string + { + $app = strtolower(trim($value)); + if (!in_array($app, self::APPS, true)) { + throw new RuntimeException('Release app must be frontend or api.'); + } + return $app; + } + + private function normalizeCaptureLevel(string $value): string + { + $level = strtolower(trim($value)); + return in_array($level, self::CAPTURE_LEVELS, true) ? $level : 'metadata'; + } + + private function normalizeDateTime(mixed $value): ?string + { + if (!is_string($value) || trim($value) === '') { + return null; + } + $timestamp = strtotime($value); + return $timestamp === false ? null : date('Y-m-d H:i:s', $timestamp); + } + + private function nullablePositiveInt(mixed $value): ?int + { + if ($value === null || $value === '') { + return null; + } + $int = (int)$value; + return $int > 0 ? $int : null; + } + + private function toBool(mixed $value): bool + { + if (is_bool($value)) { + return $value; + } + return in_array(strtolower(trim((string)$value)), ['1', 'true', 'yes', 'on'], true); + } + + private function requestTraceId(): string + { + $context = is_array($GLOBALS['RELEASE_REQUEST_CONTEXT'] ?? null) + ? $GLOBALS['RELEASE_REQUEST_CONTEXT'] + : self::initializeRequestContext(); + return (string)($context['trace_id'] ?? ''); + } + + private function assignmentCacheKey(array $context): string + { + if (!empty($context['principal_type']) && !empty($context['principal_id'])) { + return 'release_manager:assignment:' . $context['principal_type'] . ':' . $context['principal_id']; + } + if (!empty($context['customer_number'])) { + return 'release_manager:assignment:customer:' . (int)$context['customer_number']; + } + return ''; + } + + private function cacheResolvedChannel(string $cacheKey, array $channel): void + { + if ($cacheKey === '' || !defined('redis')) { + return; + } + try { + redis->setEx($cacheKey, self::jsonEncode($channel), 60); + } catch (Throwable) { + } + } + + private function clearAssignmentCache(string $subjectType, string $subjectId): void + { + if (!defined('redis')) { + return; + } + try { + redis->delete('release_manager:assignment:' . $subjectType . ':' . $subjectId); + } catch (Throwable) { + } + } + + private function moduleConfigValue(string $module, string $variable, mixed $default = null): mixed + { + $row = $this->selectOne( + 'SELECT value FROM module_config WHERE module = ? AND variable = ? LIMIT 1', + 'ss', + [$module, $variable] + ); + return $row['value'] ?? $default; + } + + private function upsertModuleConfigValue(string $module, string $variable, string $value, string $type): void + { + $existing = $this->selectOne( + 'SELECT value FROM module_config WHERE module = ? AND variable = ? LIMIT 1', + 'ss', + [$module, $variable] + ); + + if ($existing === null) { + $this->execute( + 'INSERT INTO module_config (module, variable, value, type) VALUES (?, ?, ?, ?)', + 'ssss', + [$module, $variable, $value, $type] + ); + return; + } + + $this->execute( + 'UPDATE module_config SET value = ?, type = ? WHERE module = ? AND variable = ?', + 'ssss', + [$value, $type, $module, $variable] + ); + } + + private function audit(?int $channelId, ?int $deploymentId, string $action, ?int $actorUserId, string $severity, array $context): void + { + $this->execute( + "INSERT INTO release_audit_logs (channel_id, deployment_id, action, actor_user_id, severity, context_json) + VALUES (?, ?, ?, ?, ?, ?)", + 'iisiss', + [$channelId, $deploymentId, $action, $actorUserId, $severity, self::jsonEncode(self::redactPayload($context))] + ); + } + + private function selectOne(string $sql, string $types = '', array $params = []): ?array + { + $rows = $this->selectRows($sql, $types, $params); + return $rows[0] ?? null; + } + + private function selectRows(string $sql, string $types = '', array $params = []): array + { + global $db; + if ($types === '') { + $result = $db->query($sql); + return $result ? $result->fetch_all(MYSQLI_ASSOC) : []; + } + + $stmt = $db->prepare($sql); + if ($stmt === false) { + throw new RuntimeException('Could not prepare release manager query.'); + } + $stmt->bind_param($types, ...$params); + $stmt->execute(); + $result = $stmt->get_result(); + return $result ? $result->fetch_all(MYSQLI_ASSOC) : []; + } + + private function tableExists(string $table): bool + { + $table = preg_replace('/[^a-zA-Z0-9_]/', '', $table) ?? ''; + if ($table === '') { + return false; + } + + try { + return $this->selectOne( + 'SELECT 1 FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ? LIMIT 1', + 's', + [$table] + ) !== null; + } catch (Throwable) { + return false; + } + } + + private function execute(string $sql, string $types = '', array $params = []): void + { + global $db; + if ($types === '') { + $db->query($sql); + return; + } + + $stmt = $db->prepare($sql); + if ($stmt === false) { + throw new RuntimeException('Could not prepare release manager statement.'); + } + $stmt->bind_param($types, ...$params); + $stmt->execute(); + } + + private function insertId(): int + { + global $db; + return (int)$db->insert_id(); + } + + private static function headerValue(array $headers, string $name): string + { + foreach ($headers as $key => $value) { + if (strcasecmp((string)$key, $name) === 0) { + return trim((string)$value); + } + } + $serverKey = 'HTTP_' . strtoupper(str_replace('-', '_', $name)); + return trim((string)($_SERVER[$serverKey] ?? '')); + } + + private static function safeSlug(string $value): string + { + $slug = strtolower(trim($value)); + $slug = preg_replace('/[^a-z0-9_-]/', '-', $slug) ?? ''; + $slug = trim(preg_replace('/-+/', '-', $slug) ?? '', '-'); + return substr($slug, 0, 64); + } + + private static function safeIdentifier(string $value, int $maxLength): string + { + $value = trim($value); + $value = preg_replace('/[^a-zA-Z0-9_.:-]/', '', $value) ?? ''; + return substr($value, 0, max(1, $maxLength)); + } + + private static function isSensitiveKey(string $key): bool + { + return preg_match('/authorization|cookie|password|passwd|secret|token|api[_-]?key|session|credential|card|cpr|ssn/i', $key) === 1; + } + + private static function jsonEncode(mixed $value): string + { + $json = json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($json === false) { + throw new RuntimeException('Could not encode release manager JSON payload.'); + } + return $json; + } + + private static function jsonDecode(mixed $value): array + { + if (!is_string($value) || trim($value) === '') { + return []; + } + $decoded = json_decode($value, true); + return is_array($decoded) ? $decoded : []; + } +} diff --git a/services/nginx/app/classes/release_manager_schema_bootstrap.php b/services/nginx/app/classes/release_manager_schema_bootstrap.php new file mode 100644 index 00000000..2494ec3f --- /dev/null +++ b/services/nginx/app/classes/release_manager_schema_bootstrap.php @@ -0,0 +1,411 @@ +query($sql); + } + + self::ensureColumn('release_channel_versions', 'service_set_id', 'BIGINT UNSIGNED NULL AFTER deployment_id'); + self::ensureColumn('release_channel_versions', 'bundle_id', 'BIGINT UNSIGNED NULL AFTER service_set_id'); + self::ensureColumn('release_deployments', 'service_set_id', 'BIGINT UNSIGNED NULL AFTER version_id'); + self::ensureColumn('release_deployments', 'bundle_id', 'BIGINT UNSIGNED NULL AFTER service_set_id'); + self::ensureColumn('release_deployments', 'deployment_kind', "VARCHAR(32) NOT NULL DEFAULT 'single_app' AFTER bundle_id"); + + self::ensureModuleConfigDefault('ReleaseManager', 'enabled', 'true', 'bool'); + self::ensureModuleConfigDefault('ReleaseManager', 'github_webhook_secret', '', 'string'); + self::ensureModuleConfigDefault('ReleaseManager', 'github_token', '', 'string'); + self::ensureModuleConfigDefault('ReleaseManager', 'github_api_url', 'https://api.github.com', 'string'); + self::ensureModuleConfigDefault('ReleaseManager', 'default_retention_days', '14', 'int'); + + self::ensureDefaultChannels(); + + self::$initialized = true; + self::$tablesExist = true; + } + + public static function tablesExist(): bool + { + if (self::$tablesExist !== null) { + return self::$tablesExist; + } + + global $db; + + foreach ([ + 'release_channels', + 'release_versions', + 'release_channel_versions', + 'release_assignments', + 'release_service_sets', + 'release_deployments', + 'release_bundles', + 'release_timeline_sessions', + 'release_timeline_events', + ] as $table) { + $tableSql = $db->escape_string($table); + $result = $db->query("SHOW TABLES LIKE '$tableSql'"); + if ($result === false || $result->num_rows === 0) { + self::$tablesExist = false; + return false; + } + } + + self::$tablesExist = true; + return true; + } + + private static function ensureDefaultChannels(): void + { + global $db; + + $channels = [ + ['stable', 'Stable', 'Default production channel.', 1, 1, 'metadata'], + ['canary', 'Canary', 'Earliest production validation channel.', 1, 0, 'metadata'], + ['beta', 'Beta', 'Broader pre-stable rollout channel.', 1, 0, 'metadata'], + ['internal', 'Internal', 'Internal staff and superuser validation channel.', 1, 0, 'metadata'], + ]; + + foreach ($channels as [$slug, $name, $description, $enabled, $default, $captureLevel]) { + $db->query(sprintf( + "INSERT IGNORE INTO release_channels (slug, name, description, enabled, default_channel, capture_level) + VALUES ('%s', '%s', '%s', %d, %d, '%s')", + $db->escape_string($slug), + $db->escape_string($name), + $db->escape_string($description), + (int)$enabled, + (int)$default, + $db->escape_string($captureLevel) + )); + } + } + + private static function ensureModuleConfigDefault(string $module, string $variable, string $value, string $type): void + { + global $db; + + $moduleSql = $db->escape_string($module); + $variableSql = $db->escape_string($variable); + $result = $db->query("SELECT value FROM module_config WHERE module = '$moduleSql' AND variable = '$variableSql' LIMIT 1"); + if ($result !== false && $result->num_rows > 0) { + return; + } + + $valueSql = $db->escape_string($value); + $typeSql = $db->escape_string($type); + $db->query("INSERT INTO module_config (module, variable, value, type) VALUES ('$moduleSql', '$variableSql', '$valueSql', '$typeSql')"); + } + + private static function ensureColumn(string $table, string $column, string $definition): void + { + global $db; + + $table = preg_replace('/[^a-zA-Z0-9_]/', '', $table); + $column = preg_replace('/[^a-zA-Z0-9_]/', '', $column); + if ($table === '' || $column === '') { + return; + } + + $result = $db->query("SHOW COLUMNS FROM `$table` LIKE '$column'"); + if ($result !== false && $result->num_rows > 0) { + return; + } + + $db->query("ALTER TABLE `$table` ADD COLUMN `$column` $definition"); + } +} diff --git a/services/nginx/app/classes/releasemanager.php b/services/nginx/app/classes/releasemanager.php new file mode 100644 index 00000000..a4450dfe --- /dev/null +++ b/services/nginx/app/classes/releasemanager.php @@ -0,0 +1,19 @@ +query("SELECT value FROM module_config WHERE module = 'ReleaseManager' AND variable = 'enabled' LIMIT 1"); + $row = $result ? $result->fetch_assoc() : null; + return in_array(strtolower(trim((string)($row['value'] ?? 'true'))), ['1', 'true', 'yes', 'on'], true); + } catch (\Throwable) { + return true; + } + } +} diff --git a/services/nginx/app/classes/replica_failover_manager.php b/services/nginx/app/classes/replica_failover_manager.php new file mode 100644 index 00000000..5aaa03d8 --- /dev/null +++ b/services/nginx/app/classes/replica_failover_manager.php @@ -0,0 +1,521 @@ + false, + 'database_enabled' => false, + 'redis_enabled' => false, + 'minio_enabled' => false, + 'max_status_age_seconds' => self::DEFAULT_MAX_STATUS_AGE_SECONDS, + ]; + } + + public static function normalizeConfig(array $config): array + { + $normalized = self::configDefaults(); + foreach (['enabled', 'database_enabled', 'redis_enabled', 'minio_enabled'] as $key) { + if (array_key_exists($key, $config)) { + $normalized[$key] = self::boolValue($config[$key]); + } + } + + if (array_key_exists('max_status_age_seconds', $config)) { + $normalized['max_status_age_seconds'] = max(1, (int)$config['max_status_age_seconds']); + } + + return $normalized; + } + + public static function kindEnabled(array $config, string $kind): bool + { + $config = self::normalizeConfig($config); + return $config['enabled'] && !empty($config[$kind . '_enabled']); + } + + public static function snapshotHostIsStrictlyFresh(array $host, int $maxAgeSeconds, ?int $now = null): bool + { + if (($host['role'] ?? '') !== 'replica') { + return false; + } + + if (!empty($host['deleted_at'])) { + return false; + } + + $status = self::hostStatus($host); + if (($status['status'] ?? '') !== 'ok') { + return false; + } + + if (round((float)($status['replication_percent'] ?? 0), 2) < 100.0) { + return false; + } + + $blockers = $status['blockers'] ?? []; + if (is_array($blockers) && $blockers !== []) { + return false; + } + + $checkedAt = self::hostCheckedAt($host, $status); + if ($checkedAt === null) { + return false; + } + + return (($now ?? time()) - $checkedAt) <= max(1, $maxAgeSeconds); + } + + public static function snapshotFailoverCandidate(array $hosts, string $kind, int $maxAgeSeconds, ?int $now = null): ?array + { + $eligible = array_values(array_filter( + $hosts, + static fn(array $host): bool => ($host['kind'] ?? '') === $kind + && self::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds, $now) + )); + + if ($eligible === []) { + return null; + } + + usort($eligible, static function (array $a, array $b) use ($now): int { + $aChecked = self::hostCheckedAt($a, self::hostStatus($a)) ?? 0; + $bChecked = self::hostCheckedAt($b, self::hostStatus($b)) ?? 0; + if ($aChecked === $bChecked) { + return (int)($a['id'] ?? 0) <=> (int)($b['id'] ?? 0); + } + return $bChecked <=> $aChecked; + }); + + return $eligible[0]; + } + + public static function activeConfigFromHost(string $kind, array $host): ?array + { + if ($kind === self::KIND_DATABASE) { + $database = trim((string)($host['database_name'] ?? $host['database'] ?? '')); + $user = trim((string)($host['username'] ?? $host['user'] ?? '')); + if ($database === '' || $user === '') { + return null; + } + + return [ + 'id' => isset($host['id']) ? (int)$host['id'] : null, + 'host' => (string)($host['host'] ?? ''), + 'port' => (int)($host['port'] ?? 3306) ?: 3306, + 'database' => $database, + 'user' => $user, + 'password_secret' => (string)($host['password_secret'] ?? ''), + 'ssl_mode' => (string)($host['ssl_mode'] ?? 'DISABLED'), + ]; + } + + if ($kind === self::KIND_REDIS) { + return [ + 'id' => isset($host['id']) ? (int)$host['id'] : null, + 'host' => (string)($host['host'] ?? ''), + 'port' => (int)($host['port'] ?? 6379) ?: 6379, + 'database' => (int)($host['database_index'] ?? $host['database'] ?? 0), + 'user' => (string)($host['username'] ?? $host['user'] ?? ''), + 'password_secret' => (string)($host['password_secret'] ?? ''), + ]; + } + + if ($kind === self::KIND_MINIO) { + $options = self::jsonDecode($host['options_json'] ?? null); + return [ + 'id' => isset($host['id']) ? (int)$host['id'] : null, + 'endpoint' => self::minioEndpoint($host, $options), + 'access_key' => (string)($host['username'] ?? $host['access_key'] ?? ''), + 'secret_key_secret' => (string)($host['password_secret'] ?? ''), + 'buckets' => is_array($options['buckets'] ?? null) ? array_values($options['buckets']) : [], + ]; + } + + return null; + } + + public static function applyStartupFailoverFromSnapshot(?string $path = null, array $probes = []): array + { + $snapshot = replication_bootstrap_config::loadSnapshot($path); + $failover = is_array($snapshot['failover'] ?? null) ? $snapshot['failover'] : []; + $config = self::normalizeConfig(is_array($failover['config'] ?? null) ? $failover['config'] : $failover); + $active = is_array($snapshot['active'] ?? null) ? $snapshot['active'] : []; + $hostGroups = is_array($failover['hosts'] ?? null) ? $failover['hosts'] : []; + $maxAgeSeconds = (int)$config['max_status_age_seconds']; + $summary = []; + $changed = false; + + $primaryDown = $probes['primary_down'] ?? [self::class, 'activePrimaryIsDown']; + $candidateReachable = $probes['candidate_reachable'] ?? [self::class, 'candidateReachable']; + $promoteCandidate = $probes['promote_candidate'] ?? [self::class, 'promoteCandidate']; + + foreach ([self::KIND_DATABASE, self::KIND_REDIS, self::KIND_MINIO] as $kind) { + if (!self::kindEnabled($config, $kind)) { + $summary[$kind] = ['status' => 'skipped', 'reason' => 'disabled']; + continue; + } + + if (!is_array($active[$kind] ?? null)) { + $summary[$kind] = ['status' => 'skipped', 'reason' => 'missing_active_primary']; + continue; + } + + try { + if (!call_user_func($primaryDown, $kind, $active[$kind], $snapshot)) { + $summary[$kind] = ['status' => 'skipped', 'reason' => 'primary_healthy']; + continue; + } + + $hosts = is_array($hostGroups[$kind] ?? null) ? $hostGroups[$kind] : []; + $candidate = self::snapshotFailoverCandidate($hosts, $kind, $maxAgeSeconds); + if ($candidate === null) { + $summary[$kind] = ['status' => 'skipped', 'reason' => 'no_fresh_caught_up_replica']; + continue; + } + + if (!call_user_func($candidateReachable, $kind, $candidate)) { + $summary[$kind] = [ + 'status' => 'skipped', + 'reason' => 'candidate_unreachable', + 'candidate_id' => (int)($candidate['id'] ?? 0), + ]; + continue; + } + + call_user_func($promoteCandidate, $kind, $candidate); + $candidateActive = self::activeConfigFromHost($kind, $candidate); + if ($candidateActive === null) { + $summary[$kind] = [ + 'status' => 'skipped', + 'reason' => 'candidate_missing_active_config', + 'candidate_id' => (int)($candidate['id'] ?? 0), + ]; + continue; + } + + $snapshot['active'][$kind] = $candidateActive; + $pending = is_array($snapshot['pending_failovers'] ?? null) ? $snapshot['pending_failovers'] : []; + $pending[] = [ + 'kind' => $kind, + 'host_id' => (int)($candidate['id'] ?? 0), + 'label' => (string)($candidate['label'] ?? ''), + 'source' => 'startup_snapshot', + 'promoted_at' => date('c'), + ]; + $snapshot['pending_failovers'] = $pending; + $summary[$kind] = [ + 'status' => 'promoted', + 'candidate_id' => (int)($candidate['id'] ?? 0), + ]; + $changed = true; + } catch (Throwable $throwable) { + $summary[$kind] = [ + 'status' => 'failed', + 'reason' => $throwable->getMessage(), + ]; + } + } + + if ($changed) { + $snapshot['generated_at'] = date('c'); + replication_bootstrap_config::writeSnapshot($snapshot, $path); + if ($path === null) { + replication_bootstrap_config::applyToGlobals($snapshot); + } + } + + return [ + 'changed' => $changed, + 'results' => $summary, + ]; + } + + public static function activePrimaryIsDown(string $kind, array $activeConfig, array $snapshot = []): bool + { + try { + match ($kind) { + self::KIND_DATABASE => self::probeActiveDatabase($activeConfig), + self::KIND_REDIS => self::probeActiveRedis($activeConfig), + self::KIND_MINIO => self::probeActiveMinio($activeConfig), + default => null, + }; + return false; + } catch (Throwable) { + return true; + } + } + + public static function candidateReachable(string $kind, array $host): bool + { + try { + match ($kind) { + self::KIND_DATABASE => self::probeHostDatabase($host), + self::KIND_REDIS => self::probeHostRedis($host), + self::KIND_MINIO => self::probeHostMinio($host), + default => null, + }; + return true; + } catch (Throwable) { + return false; + } + } + + public static function promoteCandidate(string $kind, array $host): void + { + match ($kind) { + self::KIND_DATABASE => self::promoteDatabaseCandidate($host), + self::KIND_REDIS => self::promoteRedisCandidate($host), + self::KIND_MINIO => self::probeHostMinio($host), + default => null, + }; + } + + private static function probeActiveDatabase(array $config): void + { + $host = (string)($config['host'] ?? ''); + $user = (string)($config['user'] ?? ''); + $database = (string)($config['database'] ?? ''); + $password = self::activePassword($config, 'password_secret', 'password'); + self::connectMysqli($host, $user, $password, $database, (int)($config['port'] ?? 3306))->close(); + } + + private static function probeHostDatabase(array $host): void + { + $credentials = self::hostCredentials($host); + $connection = self::connectMysqli( + (string)($host['host'] ?? ''), + $credentials['username'], + $credentials['password'], + (string)($host['database_name'] ?? ''), + (int)($host['port'] ?? 3306) + ); + $connection->close(); + } + + private static function promoteDatabaseCandidate(array $host): void + { + $credentials = self::hostCredentials($host); + $user = $credentials['admin_username'] !== '' ? $credentials['admin_username'] : $credentials['username']; + $password = $credentials['admin_password'] !== '' ? $credentials['admin_password'] : $credentials['password']; + $connection = self::connectMysqli( + (string)($host['host'] ?? ''), + $user, + $password, + (string)($host['database_name'] ?? ''), + (int)($host['port'] ?? 3306) + ); + + try { + foreach (['STOP REPLICA', 'STOP SLAVE'] as $statement) { + try { + $connection->query($statement); + break; + } catch (Throwable) { + } + } + foreach (['SET GLOBAL super_read_only = OFF', 'SET GLOBAL read_only = OFF'] as $statement) { + try { + $connection->query($statement); + } catch (Throwable) { + } + } + } finally { + $connection->close(); + } + } + + private static function probeActiveRedis(array $config): void + { + self::redisClientFromConfig([ + 'host' => (string)($config['host'] ?? ''), + 'port' => (int)($config['port'] ?? 6379), + 'database' => (int)($config['database'] ?? 0), + 'user' => (string)($config['user'] ?? ''), + 'password' => self::activePassword($config, 'password_secret', 'password'), + ])->ping(); + } + + private static function probeHostRedis(array $host): void + { + self::redisClientFromHost($host)->ping(); + } + + private static function promoteRedisCandidate(array $host): void + { + $client = self::redisClientFromHost($host); + $client->executeRaw(['REPLICAOF', 'NO', 'ONE']); + try { + $client->executeRaw(['CONFIG', 'REWRITE']); + } catch (Throwable) { + } + } + + private static function probeActiveMinio(array $config): void + { + self::minioClientFromConfig([ + 'endpoint' => (string)($config['endpoint'] ?? ''), + 'access_key' => (string)($config['access_key'] ?? $config['user'] ?? ''), + 'secret_key' => self::activePassword($config, 'secret_key_secret', 'secret_key'), + ])->listBuckets(); + } + + private static function probeHostMinio(array $host): void + { + self::minioClientFromHost($host)->listBuckets(); + } + + private static function connectMysqli(string $host, string $user, string $password, string $database, int $port): mysqli + { + mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); + $connection = mysqli_init(); + $connection->options(MYSQLI_OPT_CONNECT_TIMEOUT, 2); + $connection->real_connect($host, $user, $password, $database, $port ?: 3306); + $connection->set_charset('utf8mb4'); + return $connection; + } + + private static function redisClientFromHost(array $host): PredisClient + { + $credentials = self::hostCredentials($host); + return self::redisClientFromConfig([ + 'host' => (string)($host['host'] ?? ''), + 'port' => (int)($host['port'] ?? 6379), + 'database' => (int)($host['database_index'] ?? 0), + 'user' => $credentials['username'], + 'password' => $credentials['password'], + ]); + } + + private static function redisClientFromConfig(array $config): PredisClient + { + $params = [ + 'scheme' => 'tcp', + 'host' => (string)$config['host'], + 'port' => (int)$config['port'], + 'database' => (int)$config['database'], + 'password' => (string)$config['password'], + 'timeout' => 2.0, + 'read_write_timeout' => 2.0, + ]; + if (($config['user'] ?? '') !== '' && $config['user'] !== 'default') { + $params['username'] = (string)$config['user']; + } + return new PredisClient($params); + } + + private static function minioClientFromHost(array $host): S3Client + { + $credentials = self::hostCredentials($host); + $options = self::jsonDecode($host['options_json'] ?? null); + return self::minioClientFromConfig([ + 'endpoint' => self::minioEndpoint($host, $options), + 'access_key' => $credentials['username'], + 'secret_key' => $credentials['password'], + ]); + } + + private static function minioClientFromConfig(array $config): S3Client + { + return new S3Client([ + 'version' => 'latest', + 'region' => 'us-east-1', + 'endpoint' => (string)$config['endpoint'], + 'use_path_style_endpoint' => true, + 'credentials' => [ + 'key' => (string)$config['access_key'], + 'secret' => (string)$config['secret_key'], + ], + 'http' => [ + 'connect_timeout' => 2, + 'timeout' => 2, + ], + ]); + } + + private static function minioEndpoint(array $host, array $options): string + { + $endpoint = trim((string)($options['endpoint'] ?? '')); + if ($endpoint !== '') { + return $endpoint; + } + + $scheme = strtolower(trim((string)($options['scheme'] ?? 'http'))); + if ($scheme !== 'https') { + $scheme = 'http'; + } + + return $scheme . '://' . (string)($host['host'] ?? '') . ':' . ((int)($host['port'] ?? 9000) ?: 9000); + } + + private static function hostCredentials(array $host): array + { + return [ + 'username' => (string)($host['username'] ?? ''), + 'password' => replication_secret_box::decrypt($host['password_secret'] ?? ''), + 'admin_username' => (string)($host['admin_username'] ?? ''), + 'admin_password' => replication_secret_box::decrypt($host['admin_password_secret'] ?? ''), + ]; + } + + private static function activePassword(array $config, string $secretKey, string $plainKey): string + { + if (!empty($config[$secretKey])) { + return replication_secret_box::decrypt((string)$config[$secretKey]); + } + + return (string)($config[$plainKey] ?? ''); + } + + private static function hostStatus(array $host): array + { + if (isset($host['last_status']) && is_array($host['last_status'])) { + return $host['last_status']; + } + + return self::jsonDecode($host['last_status_json'] ?? null); + } + + private static function hostCheckedAt(array $host, array $status): ?int + { + $raw = $host['last_checked_at'] ?? $status['checked_at'] ?? null; + if (!is_string($raw) || trim($raw) === '') { + return null; + } + + $timestamp = strtotime($raw); + return $timestamp === false ? null : $timestamp; + } + + private static function boolValue(mixed $value): bool + { + if (is_bool($value)) { + return $value; + } + + return in_array(strtolower(trim((string)$value)), ['1', 'true', 'yes', 'on'], true); + } + + private static function jsonDecode(mixed $value): array + { + if (!is_string($value) || trim($value) === '') { + return []; + } + + $decoded = json_decode($value, true); + return is_array($decoded) ? $decoded : []; + } +} diff --git a/services/nginx/app/classes/replication_manager.php b/services/nginx/app/classes/replication_manager.php index 24908432..e2aa189c 100644 --- a/services/nginx/app/classes/replication_manager.php +++ b/services/nginx/app/classes/replication_manager.php @@ -14,7 +14,21 @@ class replication_manager private const KIND_REDIS = 'redis'; private const KIND_MINIO = 'minio'; private const MINIO_DEFAULT_BUCKETS = ['attachments', 'backups', 'invoices', 'pdfs', 'uploads', 'truckwashdev']; + private const MINIO_BACKUP_BUCKET = 'backups'; + private const MINIO_BACKUP_REPLICA_RETENTION_DAYS = 30; + private const MINIO_BACKUP_REPLICA_RETENTION_RULE_ID = 'truckwash-replica-backup-retention-30-days'; + private const MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT = '25Mi'; + private const MINIO_PROGRESS_SCAN_INTERVAL_SECONDS = 30; + private const MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER = 'MinIO replica has not caught up.'; + private const MINIO_CATCH_UP_PENDING_BYTES_TOLERANCE = 26214400; + private const MINIO_CATCH_UP_PENDING_OBJECTS_TOLERANCE = 100; + private const MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT = 99.9; + private const MINIO_S3_CONNECT_TIMEOUT_SECONDS = 2; + private const MINIO_S3_REQUEST_TIMEOUT_SECONDS = 5; + private const MINIO_MC_COMMAND_TIMEOUT_SECONDS = 8; + private const MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS = 8; private const MINIO_SPACE_HEADROOM_PERCENT = 20.0; + private const MINIO_MC_DOWNLOAD_BASE_URL = 'https://dl.min.io/client/mc/release'; private const MARIADB_SEED_BATCH_ROWS = 500; private const MARIADB_SEED_STEP_SECONDS = 3; private const MARIADB_SEED_FREEZE_TTL_SECONDS = 900; @@ -26,6 +40,7 @@ class replication_manager 'replication_status_snapshots', 'replication_operations', 'replication_audit_logs', + 'system_search_documents', ]; public function __construct() @@ -125,6 +140,7 @@ class replication_manager }; $this->storeStatus($host, $status); + $this->writeBootstrapSnapshot(); $this->audit($kind, $id, 'host_tested', $actorUserId, $status['blockers'] === [] ? 'info' : 'warning', [ 'status' => $status['status'], 'replication_percent' => $status['replication_percent'], @@ -148,7 +164,10 @@ class replication_manager $status = match ($kind) { self::KIND_DATABASE => $this->testDatabaseHost($host), self::KIND_REDIS => $this->testRedisHost($host), - self::KIND_MINIO => $this->testMinioHost($host), + self::KIND_MINIO => $this->testMinioHost(array_merge($host, [ + 'test_connectivity_only' => true, + 'skip_storage_scan' => true, + ])), }; if ($kind === self::KIND_DATABASE @@ -182,13 +201,22 @@ class replication_manager ]; } - public function provisionHost(string $kind, int $id, ?int $actorUserId = null): array + public function provisionHost( + string $kind, + int $id, + ?int $actorUserId = null, + bool $deferCoolifyManagedMinio = false + ): array { $kind = self::normalizeKind($kind); $host = $this->getHost($kind, $id); $operationId = $this->activeOperationId($kind, $id, 'provision') ?? $this->startOperation($kind, $id, 'provision', $actorUserId); + if ($deferCoolifyManagedMinio && $this->shouldDeferCoolifyManagedMinioProvision($kind, $host)) { + return $this->deferCoolifyManagedMinioProvision($host, $operationId); + } + try { $result = match ($kind) { self::KIND_DATABASE => $this->provisionDatabaseHost($host, $operationId), @@ -212,6 +240,69 @@ class replication_manager } } + private function shouldDeferCoolifyManagedMinioProvision(string $kind, array $host): bool + { + if ($kind !== self::KIND_MINIO) { + return false; + } + + $options = $this->decodeOptions($host); + return (string)($options['deployment_provider'] ?? '') === 'coolify' + || isset($options['coolify_target_id']) + || isset($options['coolify_instance_id']); + } + + private function deferCoolifyManagedMinioProvision(array $host, int $operationId): array + { + $lastStatus = self::sanitizePublicLastStatus($host, self::jsonDecode($host['last_status_json'] ?? null)); + $progress = 45.0; + if (is_array($lastStatus) && is_numeric($lastStatus['replication_percent'] ?? null)) { + $progress = max($progress, self::minioIncompleteProgress((float)$lastStatus['replication_percent'])); + } + + $message = 'MinIO provisioning was queued for Coolify background maintenance.'; + $this->updateOperationProgress($operationId, $progress, $message, [ + 'phase' => 'coolify_deferred', + 'queued_at' => date('c'), + ]); + $this->execute( + "UPDATE replication_hosts SET status = 'provisioning' WHERE id = ? AND kind = ?", + 'is', + [(int)$host['id'], self::KIND_MINIO] + ); + + $status = [ + 'status' => 'provisioning', + 'replication_percent' => $progress, + 'lag_seconds' => null, + 'blockers' => [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER], + 'raw' => [ + 'progress_source' => 'coolify_deferred', + 'previous_status' => is_array($lastStatus) ? [ + 'status' => $lastStatus['status'] ?? null, + 'replication_percent' => $lastStatus['replication_percent'] ?? null, + 'checked_at' => $lastStatus['checked_at'] ?? null, + ] : null, + ], + 'checked_at' => date('c'), + ]; + $this->storeStatus($this->getHost(self::KIND_MINIO, (int)$host['id']), $status); + + return [ + 'ok' => true, + 'message' => $message, + 'blockers' => $status['blockers'], + 'replication_percent' => $progress, + 'operation' => [ + 'id' => $operationId, + 'status' => 'running', + 'progress_percent' => $progress, + 'message' => $message, + ], + 'host' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])), + ]; + } + public function promoteHost(string $kind, int $id, ?int $actorUserId = null): array { $kind = self::normalizeKind($kind); @@ -250,11 +341,87 @@ class replication_manager } } - public function removeHost(string $kind, int $id, ?int $actorUserId = null): array + public function runAutomaticFailoverMonitor(?int $actorUserId = null): array + { + $this->ensureEnvironmentPrimaryRows(); + $config = $this->failoverConfigForSnapshot(); + $results = []; + + foreach ([self::KIND_DATABASE, self::KIND_REDIS, self::KIND_MINIO] as $kind) { + $results[$kind] = $this->runAutomaticFailoverForKind($kind, $config, $actorUserId); + } + + try { + $this->refreshStatuses(); + } catch (Throwable $throwable) { + $this->audit(self::KIND_DATABASE, null, 'automatic_failover_status_refresh_failed', $actorUserId, 'warning', [ + 'error' => $throwable->getMessage(), + ]); + } + + $this->writeBootstrapSnapshot(); + + return [ + 'ok' => true, + 'config' => $config, + 'results' => $results, + ]; + } + + public function syncStartupFailoversFromSnapshot(?int $actorUserId = null): array + { + $snapshot = replication_bootstrap_config::loadSnapshot(); + $pending = is_array($snapshot['pending_failovers'] ?? null) ? $snapshot['pending_failovers'] : []; + $synced = []; + + foreach ($pending as $entry) { + if (!is_array($entry)) { + continue; + } + + try { + $kind = self::normalizeKind((string)($entry['kind'] ?? '')); + $hostId = (int)($entry['host_id'] ?? 0); + if ($hostId <= 0) { + continue; + } + + $currentPrimary = $this->primaryHost($kind); + if ($currentPrimary !== null && (int)$currentPrimary['id'] !== $hostId) { + $this->switchPrimary($kind, $hostId, (int)$currentPrimary['id']); + } + + $this->audit($kind, $hostId, 'startup_failover_synced', $actorUserId, 'critical', $entry); + $synced[] = [ + 'kind' => $kind, + 'host_id' => $hostId, + ]; + } catch (Throwable $throwable) { + $this->audit((string)($entry['kind'] ?? self::KIND_DATABASE), null, 'startup_failover_sync_failed', $actorUserId, 'error', [ + 'entry' => $entry, + 'error' => $throwable->getMessage(), + ]); + } + } + + if ($pending !== []) { + $snapshot['pending_failovers'] = []; + replication_bootstrap_config::writeSnapshot($snapshot); + $this->writeBootstrapSnapshot(); + } + + return $synced; + } + + public function removeHost(string $kind, int $id, ?int $actorUserId = null, bool $removeLinkedCoolifyTargets = true): array { $kind = self::normalizeKind($kind); $host = $this->getHost($kind, $id); - if (!self::replicationHostCanBeRemoved($host)) { + $canRemove = self::replicationHostCanBeRemoved($host); + if (!$canRemove && class_exists(coolify_manager::class)) { + $canRemove = coolify_manager::replicationHostCanBeRemoved($host); + } + if (!$canRemove) { if (($host['role'] ?? '') === 'primary') { throw new RuntimeException('Primary hosts cannot be removed. Promote a healthy replica first.'); } @@ -269,6 +436,9 @@ class replication_manager 'label' => $host['label'] ?? '', 'host' => $host['host'] ?? '', ]); + if ($removeLinkedCoolifyTargets && class_exists(coolify_manager::class)) { + coolify_manager::markTargetsRemovedForReplicationHost($id, $actorUserId); + } $this->writeBootstrapSnapshot(); return [ @@ -279,6 +449,38 @@ class replication_manager ]; } + public function renameHost(string $kind, int $id, array $input, ?int $actorUserId = null): array + { + $kind = self::normalizeKind($kind); + $host = $this->getHost($kind, $id); + $label = trim((string)($input['label'] ?? $input['name'] ?? '')); + if ($label === '') { + throw new RuntimeException('Replication host label is required.'); + } + if (mb_strlen($label) > 128) { + throw new RuntimeException('Replication host label must be 128 characters or fewer.'); + } + + $oldLabel = (string)($host['label'] ?? ''); + if ($label !== $oldLabel) { + $this->execute( + "UPDATE replication_hosts SET label = ? WHERE id = ? AND kind = ? AND deleted_at IS NULL", + 'sis', + [$label, $id, $kind] + ); + if (class_exists(coolify_manager::class)) { + coolify_manager::syncLabelForReplicationHost($id, $label); + } + $this->audit($kind, $id, 'host_renamed', $actorUserId, 'info', [ + 'old_label' => $oldLabel, + 'new_label' => $label, + ]); + $this->writeBootstrapSnapshot(); + } + + return $this->publicHost($this->getHost($kind, $id)); + } + public static function replicationHostCanBeRemoved(array $host): bool { $role = (string)($host['role'] ?? ''); @@ -335,6 +537,8 @@ class replication_manager $replicationPassword = self::composePassword($input['replication_password'] ?? null); $primaryHost = self::composeScalar($input['primary_host'] ?? null, ''); $primaryPort = self::boundedInt($input['primary_port'] ?? null, 3306, 1, 65535); + $primaryAdminUsername = self::composeScalar($input['primary_admin_username'] ?? null, 'root'); + $primaryAdminPassword = trim((string)($input['primary_admin_password'] ?? '')); $command = [ 'mariadbd', @@ -386,18 +590,18 @@ class replication_manager $seedServiceName = self::composeIdentifier($serviceName . '-seed', 'mariadb-replica-seed'); $seedScript = [ 'marker="/var/lib/mysql/.truckwash-replica-seeded"', - 'if [ -f "$marker" ]; then', + 'if [ -f "$${marker}" ]; then', ' echo "Replica already seeded."', ' exit 0', 'fi', 'echo "Waiting for local replica..."', - 'until mariadb-admin ping -h ' . self::shellArg($serviceName) . ' -uroot -p"$MARIADB_ROOT_PASSWORD" --silent; do sleep 2; done', + 'until mariadb-admin ping -h ' . self::shellArg($serviceName) . ' -uroot -p"$${MARIADB_ROOT_PASSWORD}" --silent; do sleep 2; done', 'echo "Importing seed from primary..."', - 'mariadb-dump --host="$MARIADB_PRIMARY_HOST" --port="$MARIADB_PRIMARY_PORT" --user="$MARIADB_PRIMARY_ADMIN_USER" --password="$MARIADB_PRIMARY_ADMIN_PASSWORD" --single-transaction --quick --routines --triggers --events --gtid --master-data=2 ' . self::mariaDbSchemaOnlyDumpIgnoreArgs('$MARIADB_SEED_DATABASE') . ' --databases "$MARIADB_SEED_DATABASE" | mariadb --host=' . self::shellArg($serviceName) . ' --user=root --password="$MARIADB_ROOT_PASSWORD"', + 'mariadb-dump --host="$${MARIADB_PRIMARY_HOST}" --port="$${MARIADB_PRIMARY_PORT}" --user="$${MARIADB_PRIMARY_ADMIN_USER}" --password="$${MARIADB_PRIMARY_ADMIN_PASSWORD}" --single-transaction --quick --routines --triggers --events --gtid --master-data=2 ' . self::mariaDbSchemaOnlyDumpIgnoreArgs('$${MARIADB_SEED_DATABASE}') . ' --databases "$${MARIADB_SEED_DATABASE}" | mariadb --host=' . self::shellArg($serviceName) . ' --user=root --password="$${MARIADB_ROOT_PASSWORD}"', 'for table in ' . implode(' ', self::MARIADB_SCHEMA_ONLY_TABLES) . '; do', - ' mariadb-dump --host="$MARIADB_PRIMARY_HOST" --port="$MARIADB_PRIMARY_PORT" --user="$MARIADB_PRIMARY_ADMIN_USER" --password="$MARIADB_PRIMARY_ADMIN_PASSWORD" --single-transaction --quick --no-data "$MARIADB_SEED_DATABASE" "$table" | mariadb --host=' . self::shellArg($serviceName) . ' --user=root --password="$MARIADB_ROOT_PASSWORD" "$MARIADB_SEED_DATABASE" || true', + ' mariadb-dump --host="$${MARIADB_PRIMARY_HOST}" --port="$${MARIADB_PRIMARY_PORT}" --user="$${MARIADB_PRIMARY_ADMIN_USER}" --password="$${MARIADB_PRIMARY_ADMIN_PASSWORD}" --single-transaction --quick --no-data "$${MARIADB_SEED_DATABASE}" "$${table}" | mariadb --host=' . self::shellArg($serviceName) . ' --user=root --password="$${MARIADB_ROOT_PASSWORD}" "$${MARIADB_SEED_DATABASE}" || true', 'done', - 'touch "$marker"', + 'touch "$${marker}"', 'echo "Replica seed completed."', ]; @@ -475,8 +679,8 @@ class replication_manager if ($role === 'replica') { $envLines[] = 'MARIADB_PRIMARY_HOST=' . $primaryHost; $envLines[] = 'MARIADB_PRIMARY_PORT=' . $primaryPort; - $envLines[] = 'MARIADB_PRIMARY_ADMIN_USER=root'; - $envLines[] = 'MARIADB_PRIMARY_ADMIN_PASSWORD='; + $envLines[] = 'MARIADB_PRIMARY_ADMIN_USER=' . $primaryAdminUsername; + $envLines[] = 'MARIADB_PRIMARY_ADMIN_PASSWORD=' . $primaryAdminPassword; } return [ @@ -622,17 +826,28 @@ class replication_manager $mcImage = self::composeImage($input['mc_image'] ?? null, 'minio/mc:latest'); $hostPort = self::boundedInt($input['host_port'] ?? null, $role === 'primary' ? 9000 : 9010, 1, 65535); $consolePort = self::boundedInt($input['console_port'] ?? null, $role === 'primary' ? 9001 : 9011, 1, 65535); - $rootUser = self::composeScalar($input['username'] ?? $input['access_key'] ?? null, 'truckwash-minio'); + $rootUser = self::composeAccessKey($input['username'] ?? $input['access_key'] ?? null); $rootPassword = self::composePassword($input['password'] ?? $input['secret_key'] ?? null); $buckets = self::normalizeMinioBuckets($input['buckets'] ?? self::MINIO_DEFAULT_BUCKETS); + $transferLimit = self::normalizeMinioTransferLimit( + $input['replication_transfer_limit'] ?? self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT, + true + ); + $primaryEndpoint = self::minioPrimaryComposeValue($input, 'endpoint'); + $primaryAccessKey = self::minioPrimaryComposeValue($input, 'access_key'); + $primarySecretKey = self::minioPrimaryComposeValue($input, 'secret_key'); + [$serverUrl, $browserRedirectUrl] = self::minioComposePublicUrls($input, $hostPort, $consolePort); $setupScript = [ 'until mc alias set local http://' . self::shellArg($serviceName) . ':9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"; do sleep 2; done', ]; foreach ($buckets as $bucket) { $bucketArg = self::shellArg('local/' . $bucket); - $setupScript[] = 'mc mb --ignore-existing ' . $bucketArg; + $setupScript[] = 'mc mb --with-lock --ignore-existing ' . $bucketArg; $setupScript[] = 'mc version enable ' . $bucketArg . ' || true'; + if ($role === 'replica' && self::minioBucketUsesBoundedReplicaRetention($bucket)) { + $setupScript[] = 'mc ilm rule add --expire-days "' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . '" --noncurrent-expire-days "' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . '" ' . $bucketArg . ' || true'; + } } $lines = [ @@ -648,6 +863,8 @@ class replication_manager ' environment:', ' MINIO_ROOT_USER: "${MINIO_ROOT_USER:?set MINIO_ROOT_USER}"', ' MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD}"', + ' MINIO_SERVER_URL: "${MINIO_SERVER_URL:-}"', + ' MINIO_BROWSER_REDIRECT_URL: "${MINIO_BROWSER_REDIRECT_URL:-}"', ' volumes:', ' - ' . $volumeName . ':/data', ' ports:', @@ -694,12 +911,16 @@ class replication_manager $envLines = [ 'MINIO_ROOT_USER=' . $rootUser, 'MINIO_ROOT_PASSWORD=' . $rootPassword, + 'MINIO_SERVER_URL=' . $serverUrl, + 'MINIO_BROWSER_REDIRECT_URL=' . $browserRedirectUrl, 'MINIO_BUCKETS=' . implode(',', $buckets), ]; if ($role === 'replica') { - $envLines[] = 'MINIO_PRIMARY_ENDPOINT='; - $envLines[] = 'MINIO_PRIMARY_ACCESS_KEY='; - $envLines[] = 'MINIO_PRIMARY_SECRET_KEY='; + $envLines[] = 'MINIO_BACKUP_REPLICA_RETENTION_DAYS=' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS; + $envLines[] = 'MINIO_REPLICATION_TRANSFER_LIMIT=' . $transferLimit; + $envLines[] = 'MINIO_PRIMARY_ENDPOINT=' . $primaryEndpoint; + $envLines[] = 'MINIO_PRIMARY_ACCESS_KEY=' . $primaryAccessKey; + $envLines[] = 'MINIO_PRIMARY_SECRET_KEY=' . $primarySecretKey; } $steps = [ @@ -709,6 +930,8 @@ class replication_manager ]; if ($role === 'replica') { $steps[] = 'Fill the MINIO_PRIMARY_* .env values for reference; managed bucket replication is configured from the superuser UI.'; + $steps[] = 'The backups bucket is retained on replicas for ' . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . ' days; other buckets are fully replicated.'; + $steps[] = 'Replica seeding and bucket replication are bandwidth-limited to ' . ($transferLimit !== '' ? $transferLimit : 'unlimited') . '.'; $steps[] = 'Test the connection, then save and provision the replica.'; } @@ -731,6 +954,7 @@ class replication_manager 'console_port' => $consolePort, 'username' => $rootUser, 'password' => $rootPassword, + 'replication_transfer_limit' => $transferLimit, 'space_headroom_percent' => self::MINIO_SPACE_HEADROOM_PERCENT, ], 'steps' => $steps, @@ -770,6 +994,82 @@ class replication_manager return self::generateSecret(24); } + private static function composeAccessKey(mixed $value): string + { + $accessKey = trim((string)$value); + if ($accessKey !== '') { + return $accessKey; + } + + return 'twminio' . bin2hex(random_bytes(12)); + } + + private static function minioPrimaryComposeValue(array $input, string $field): string + { + if ($field === 'endpoint') { + foreach (['primary_endpoint', 'minio_primary_endpoint'] as $key) { + $value = trim((string)($input[$key] ?? '')); + if ($value !== '') { + return $value; + } + } + + $primaryHost = trim((string)($input['primary_host'] ?? '')); + if ($primaryHost !== '') { + if (preg_match('/^https?:\/\//i', $primaryHost) === 1) { + return $primaryHost; + } + + $primaryScheme = trim((string)($input['primary_scheme'] ?? 'http')) ?: 'http'; + $primaryPort = self::boundedInt($input['primary_port'] ?? null, 9000, 1, 65535); + return self::minioEndpointFromParts($primaryScheme, $primaryHost, $primaryPort); + } + } + + $inputKeys = match ($field) { + 'endpoint' => [], + 'access_key' => ['primary_access_key', 'minio_primary_access_key', 'primary_username'], + 'secret_key' => ['primary_secret_key', 'minio_primary_secret_key', 'primary_password'], + default => [], + }; + + foreach ($inputKeys as $key) { + $value = trim((string)($input[$key] ?? '')); + if ($value !== '') { + return $value; + } + } + + $minioConfig = $GLOBALS['MINIO'] ?? null; + if (!is_array($minioConfig)) { + return ''; + } + + return trim((string)($minioConfig[$field] ?? '')); + } + + /** + * Public MinIO URLs keep browser redirects on the externally mapped ports. + */ + private static function minioComposePublicUrls(array $input, int $hostPort, int $consolePort): array + { + $rawHost = trim((string)($input['public_host'] ?? $input['host'] ?? $input['endpoint'] ?? '')); + if ($rawHost === '') { + return ['', '']; + } + + try { + [$host, , $scheme] = self::normalizeMinioAddress($rawHost, null, $input['scheme'] ?? null); + } catch (Throwable) { + return ['', '']; + } + + return [ + self::minioEndpointFromParts($scheme, $host, $hostPort), + self::minioEndpointFromParts($scheme, $host, $consolePort), + ]; + } + private static function generateSecret(int $bytes): string { return rtrim(strtr(base64_encode(random_bytes($bytes)), '+/', '-_'), '='); @@ -876,6 +1176,56 @@ class replication_manager return round(min(100, max(0, ($replicaOffset / $primaryOffset) * 100)), 2); } + public static function redisReplicationPercentFromInfo(array $primaryInfo, array $replicaInfo): float + { + $syncInProgress = (string)($replicaInfo['master_sync_in_progress'] ?? '0') === '1'; + if ($syncInProgress) { + $totalBytes = (int)($replicaInfo['master_sync_total_bytes'] ?? 0); + $leftBytes = (int)($replicaInfo['master_sync_left_bytes'] ?? 0); + if ($totalBytes <= 0) { + return 5.0; + } + + $copiedBytes = max(0, $totalBytes - max(0, $leftBytes)); + return round(min(99.99, max(5.0, ($copiedBytes / $totalBytes) * 100)), 2); + } + + $primaryOffset = (int)($primaryInfo['master_repl_offset'] ?? 0); + $replicaOffset = (int)($replicaInfo['slave_repl_offset'] ?? $replicaInfo['master_repl_offset'] ?? 0); + return self::redisOffsetPercent($primaryOffset, $replicaOffset); + } + + public static function redisProvisionProgress(float $replicationPercent, array $syncBlockers = []): float + { + if ($replicationPercent >= 100.0 && $syncBlockers === []) { + return 100.0; + } + + return round(min(99.99, max(5.0, $replicationPercent)), 2); + } + + public static function replicationHealthStatus( + bool $reachable, + string $role, + float $replicationPercent, + array $blockers, + bool $replicationChecked = true + ): string { + if (!$reachable) { + return 'down'; + } + + if ($blockers !== []) { + return 'degraded'; + } + + if ($replicationChecked && $role !== 'primary' && $replicationPercent < 100.0) { + return 'degraded'; + } + + return 'ok'; + } + public static function minioRequiredFreeBytes(int $sourceBytes, float $headroomPercent = self::MINIO_SPACE_HEADROOM_PERCENT): int { return (int)ceil(max(0, $sourceBytes) * (1 + max(0.0, $headroomPercent) / 100)); @@ -890,10 +1240,132 @@ class replication_manager return round(min(100, max(0, ($replicaBytes / $sourceBytes) * 100)), 2); } + public static function minioProvisionProgress(array $status): float + { + $percent = round((float)($status['replication_percent'] ?? 0), 2); + $blockers = array_values(array_filter($status['blockers'] ?? [])); + if ($percent >= 100.0 && $blockers === []) { + return 100.0; + } + + $measured = !empty($status['raw']['storage']['measured']) + || (string)($status['raw']['progress_source'] ?? '') === 'minio_replicate_status'; + if ($measured) { + return min(self::MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT, max(0.0, $percent)); + } + + return self::minioIncompleteProgress($percent); + } + + private static function minioIncompleteProgress(float $percent, float $minimum = 5.0): float + { + return round(min(self::MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT, max($minimum, $percent)), 2); + } + + public static function minioReplicationProgressFromStatusOutput(mixed $value): ?array + { + if (is_string($value)) { + $textProgress = self::minioReplicationProgressFromText($value); + if ($textProgress !== null) { + return $textProgress; + } + + $decoded = self::decodeMinioJsonOutput($value); + if ($decoded !== null && $decoded !== $value) { + return self::minioReplicationProgressFromStatusOutput($decoded); + } + + return null; + } + + $stats = [ + 'completed_bytes' => 0.0, + 'pending_bytes' => 0.0, + 'failed_bytes' => 0.0, + 'total_bytes' => 0.0, + 'completed_count' => 0.0, + 'pending_count' => 0.0, + 'failed_count' => 0.0, + 'total_count' => 0.0, + 'complete_signals' => 0, + 'incomplete_signals' => 0, + ]; + self::collectMinioReplicationProgress($value, $stats); + + $completedBytes = (float)$stats['completed_bytes']; + $remainingBytes = (float)$stats['pending_bytes'] + (float)$stats['failed_bytes']; + $totalBytes = (float)$stats['total_bytes']; + $completedCount = (float)$stats['completed_count']; + $remainingCount = (float)$stats['pending_count'] + (float)$stats['failed_count']; + $totalCount = (float)$stats['total_count']; + $basis = null; + $percent = null; + + if ($totalBytes > 0.0) { + $percent = ($completedBytes / $totalBytes) * 100; + $basis = 'total_bytes'; + } elseif (($completedBytes + $remainingBytes) > 0.0) { + $percent = ($completedBytes / ($completedBytes + $remainingBytes)) * 100; + $basis = 'byte_balance'; + } elseif ($totalCount > 0.0) { + $percent = ($completedCount / $totalCount) * 100; + $basis = 'total_count'; + } elseif (($completedCount + $remainingCount) > 0.0) { + $percent = ($completedCount / ($completedCount + $remainingCount)) * 100; + $basis = 'count_balance'; + } elseif ((int)$stats['complete_signals'] > 0 && (int)$stats['incomplete_signals'] === 0) { + $percent = 100.0; + $basis = 'status_signal'; + } elseif ((int)$stats['incomplete_signals'] > 0) { + $percent = 5.0; + $basis = 'status_signal'; + } + + if ($percent === null) { + return null; + } + + $percent = round(min(100.0, max(0.0, $percent)), 2); + + return [ + 'replication_percent' => $percent, + 'blockers' => $percent < 100.0 ? [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER] : [], + 'basis' => $basis, + 'stats' => $stats, + ]; + } + + public static function minioBackupReplicaRetentionDays(): int + { + return self::MINIO_BACKUP_REPLICA_RETENTION_DAYS; + } + + public static function minioBackupRetentionBlockers(array $stats): array + { + foreach ($stats['buckets'] ?? [] as $bucket) { + if (!is_array($bucket) || (string)($bucket['name'] ?? '') !== self::MINIO_BACKUP_BUCKET) { + continue; + } + + $expiredObjects = (int)($bucket['expired_objects'] ?? 0); + if ($expiredObjects <= 0) { + return []; + } + + return [ + 'MinIO backup replica contains ' . $expiredObjects . ' backup object' + . ($expiredObjects === 1 ? '' : 's') . ' older than ' + . self::MINIO_BACKUP_REPLICA_RETENTION_DAYS . ' days. Run provisioning to prune retained backups.', + ]; + } + + return []; + } + public static function minioSpaceBlockers(?int $availableBytes, int $requiredBytes): array { if ($availableBytes === null) { - return ['MinIO target free space could not be determined.']; + return []; } if ($availableBytes < $requiredBytes) { return ['MinIO target does not have enough free space. Required ' . $requiredBytes . ' bytes, available ' . $availableBytes . ' bytes.']; @@ -924,6 +1396,266 @@ class replication_manager return $buckets !== [] ? $buckets : self::MINIO_DEFAULT_BUCKETS; } + private static function minioBucketUsesBoundedReplicaRetention(string $bucket): bool + { + return strtolower(trim($bucket)) === self::MINIO_BACKUP_BUCKET; + } + + public static function minioBucketCountsTowardCatchUp(string $bucket): bool + { + return !self::minioBucketUsesBoundedReplicaRetention($bucket); + } + + private static function minioReplicaRetentionDaysByBucket(array $buckets): array + { + $retention = []; + foreach ($buckets as $bucket) { + if (self::minioBucketUsesBoundedReplicaRetention((string)$bucket)) { + $retention[(string)$bucket] = self::MINIO_BACKUP_REPLICA_RETENTION_DAYS; + } + } + + return $retention; + } + + public static function minioDefaultReplicationTransferLimit(): string + { + return self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT; + } + + public static function normalizeMinioTransferLimit(mixed $value, bool $defaultWhenEmpty = true): string + { + $raw = trim((string)$value); + if ($raw === '') { + return $defaultWhenEmpty ? self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT : ''; + } + + $normalized = preg_replace('/\s+/', '', $raw) ?? $raw; + $lower = strtolower($normalized); + if (in_array($lower, ['0', 'none', 'off', 'unlimited', 'disabled'], true)) { + return ''; + } + + $normalized = preg_replace('/\/s$/i', '', $normalized) ?? $normalized; + if (preg_match('/^(\d+(?:\.\d+)?)([a-zA-Z]*)$/', $normalized, $matches) !== 1) { + throw new RuntimeException('MinIO transfer limit must be empty, 0, or a rate like 25Mi, 100M, or 1G.'); + } + + $amount = $matches[1]; + if (str_contains($amount, '.')) { + $amount = rtrim(rtrim($amount, '0'), '.'); + } + if ($amount === '' || (float)$amount <= 0) { + return ''; + } + + $unit = $matches[2]; + $unitMap = [ + '' => '', + 'b' => 'B', + 'k' => 'K', + 'kb' => 'K', + 'm' => 'M', + 'mb' => 'M', + 'g' => 'G', + 'gb' => 'G', + 't' => 'T', + 'tb' => 'T', + 'ki' => 'Ki', + 'kib' => 'Ki', + 'mi' => 'Mi', + 'mib' => 'Mi', + 'gi' => 'Gi', + 'gib' => 'Gi', + 'ti' => 'Ti', + 'tib' => 'Ti', + ]; + $unitKey = strtolower($unit); + if (!array_key_exists($unitKey, $unitMap)) { + throw new RuntimeException('MinIO transfer limit must use B, K, M, G, T, Ki, Mi, Gi, or Ti units.'); + } + + return $amount . $unitMap[$unitKey]; + } + + private static function minioReplicationTransferLimitFromOptions(array $options): string + { + foreach (['replication_transfer_limit', 'transfer_limit', 'bandwidth_limit'] as $key) { + if (array_key_exists($key, $options)) { + return self::normalizeMinioTransferLimit($options[$key], false); + } + } + + return self::MINIO_DEFAULT_REPLICATION_TRANSFER_LIMIT; + } + + private static function minioReplicationTransferLimitArgs(string $transferLimit): array + { + $transferLimit = self::normalizeMinioTransferLimit($transferLimit, false); + if ($transferLimit === '') { + return []; + } + + return ['--limit-upload', $transferLimit, '--limit-download', $transferLimit]; + } + + private static function minioReplicationProgressFromText(string $output): ?array + { + if (preg_match_all('/(? $percent > 0.0)); + $percent = $nonZero !== [] ? min($nonZero) : 0.0; + $percent = round(min(100.0, max(0.0, $percent)), 2); + + return [ + 'replication_percent' => $percent, + 'blockers' => $percent < 100.0 ? [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER] : [], + 'basis' => 'text_percent', + 'stats' => [ + 'percent_values' => $percentages, + ], + ]; + } + + private static function collectMinioReplicationProgress(mixed $value, array &$stats, array $path = []): void + { + if (is_object($value)) { + $value = get_object_vars($value); + } + if (!is_array($value)) { + return; + } + + foreach ($value as $key => $entry) { + $normalizedKey = self::normalizeMinioProgressKey((string)$key); + $nextPath = array_values(array_filter(array_merge($path, [$normalizedKey]), static fn(string $part): bool => $part !== '')); + + if (is_numeric($entry)) { + self::collectMinioReplicationProgressNumber($nextPath, (float)$entry, $stats); + continue; + } + + if (is_string($entry)) { + self::collectMinioReplicationProgressString($entry, $stats); + $textProgress = self::minioReplicationProgressFromText($entry); + if ($textProgress !== null) { + $stats['completed_count'] += (float)$textProgress['replication_percent']; + $stats['total_count'] += 100.0; + } + continue; + } + + self::collectMinioReplicationProgress($entry, $stats, $nextPath); + } + } + + private static function collectMinioReplicationProgressNumber(array $path, float $value, array &$stats): void + { + if ($value < 0.0) { + return; + } + + $pathText = implode('', $path); + foreach ([ + 'percent', + 'percentage', + 'duration', + 'elapsed', + 'timestamp', + 'time', + 'priority', + 'port', + 'versionid', + 'avg', + 'average', + 'peak', + 'rate', + 'latency', + 'uptime', + 'downtime', + 'lastminute', + 'lasthour', + 'last1hr', + 'last1m', + 'last5min', + 'sinceuptime', + ] as $ignored) { + if (str_contains($pathText, $ignored)) { + return; + } + } + + $category = null; + foreach (['failed', 'failure', 'failures', 'error', 'errors'] as $needle) { + if (str_contains($pathText, $needle)) { + $category = 'failed'; + break; + } + } + if ($category === null) { + foreach (['pending', 'queued', 'queue', 'backlog', 'remaining', 'unreplicated', 'inprogress', 'missing'] as $needle) { + if (str_contains($pathText, $needle)) { + $category = 'pending'; + break; + } + } + } + if ($category === null) { + foreach (['completed', 'complete', 'replicated', 'replicate', 'replica', 'success', 'synced'] as $needle) { + if (str_contains($pathText, $needle)) { + $category = 'completed'; + break; + } + } + } + if ($category === null && str_contains($pathText, 'total')) { + $category = 'total'; + } + if ($category === null) { + return; + } + + $isBytes = str_contains($pathText, 'byte') + || str_contains($pathText, 'bytes') + || str_contains($pathText, 'size'); + $suffix = $isBytes ? 'bytes' : 'count'; + $stats[$category . '_' . $suffix] += $value; + } + + private static function collectMinioReplicationProgressString(string $value, array &$stats): void + { + $normalized = self::normalizeMinioProgressKey($value); + if ($normalized === '') { + return; + } + + foreach (['pending', 'queued', 'backlog', 'replicating', 'syncing', 'inprogress', 'failed', 'failure', 'error'] as $needle) { + if (str_contains($normalized, $needle)) { + $stats['incomplete_signals']++; + return; + } + } + + foreach (['completed', 'complete', 'replicated', 'synced', 'success', 'healthy', 'ok'] as $needle) { + if (str_contains($normalized, $needle)) { + $stats['complete_signals']++; + return; + } + } + } + + private static function normalizeMinioProgressKey(string $value): string + { + return strtolower((string)preg_replace('/[^a-zA-Z0-9]+/', '', $value)); + } + public static function mariadbGtidCoveragePercent(string $sourceSet, string $replicaSet): float { $source = self::parseMariaDbGtidSet($sourceSet); @@ -1031,22 +1763,24 @@ class replication_manager $blockers = array_merge($targetStatus['blockers'], $primaryStatus['blockers']); $targetEngine = self::databaseEngine($targetStatus['raw'] ?? []); $primaryEngine = self::databaseEngine($primaryStatus['raw'] ?? []); + $targetEngineKnown = self::databaseEngineKnown($targetStatus['raw'] ?? []); + $primaryEngineKnown = self::databaseEngineKnown($primaryStatus['raw'] ?? []); if (($targetStatus['raw']['server_id'] ?? null) !== null && ($primaryStatus['raw']['server_id'] ?? null) !== null && (int)$targetStatus['raw']['server_id'] === (int)$primaryStatus['raw']['server_id']) { $blockers[] = 'Database replica must have a unique server_id.'; } - if ($targetEngine !== $primaryEngine) { + if ($targetEngineKnown && $primaryEngineKnown && $targetEngine !== $primaryEngine) { $blockers[] = 'Database primary and replica must use the same engine family.'; } $cloneReady = (bool)($targetStatus['raw']['clone_plugin_active'] ?? false); $primaryCloneReady = (bool)($primaryStatus['raw']['clone_plugin_active'] ?? false); - if ($targetEngine === 'mariadb' && !$usePreseededReplica) { + if ($targetEngineKnown && $targetEngine === 'mariadb' && !$usePreseededReplica) { $blockers[] = 'MariaDB replicas must be safely seeded before managed replication can be configured.'; } - if ($targetEngine === 'mysql' && !$usePreseededReplica) { + if ($targetEngineKnown && $targetEngine === 'mysql' && !$usePreseededReplica) { if (!$cloneReady) { $blockers[] = 'MySQL Clone plugin is not active on the target. Set allow_preseeded_replica only after the target has been safely seeded.'; } @@ -1122,7 +1856,8 @@ class replication_manager } if ($shouldManageReplicationUser) { - $this->ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword); + $grantHosts = $this->databaseReplicationGrantHosts($host, $targetStatus); + $this->ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword, $grantHosts); } if ($targetEngine === 'mysql' && !$usePreseededReplica) { @@ -1161,7 +1896,116 @@ class replication_manager ]; } - private function ensureDatabaseReplicationUser(array $primary, string $replicationUser, string $replicationPassword): void + private function databaseReplicationGrantHosts(array $host, array $targetStatus = []): array + { + $grantHosts = ['%']; + if (isset($host['host'])) { + $grantHosts[] = (string)$host['host']; + } + + $lastStatus = self::jsonDecode($host['last_status_json'] ?? null); + foreach ([$targetStatus, $lastStatus] as $status) { + if (!is_array($status)) { + continue; + } + + foreach ($this->databaseDeniedAccountHostsFromStatus($status) as $deniedHost) { + $grantHosts[] = $deniedHost; + } + } + + $normalized = []; + foreach ($grantHosts as $grantHost) { + foreach (self::databaseAccountHostGrantCandidates((string)$grantHost) as $candidate) { + if (!in_array($candidate, $normalized, true)) { + $normalized[] = $candidate; + } + } + } + + return $normalized === [] ? ['%'] : $normalized; + } + + private function databaseDeniedAccountHostsFromStatus(array $status): array + { + $hosts = []; + + foreach ($status['blockers'] ?? [] as $blocker) { + if (is_scalar($blocker)) { + $hosts = array_merge($hosts, self::databaseDeniedAccountHostsFromText((string)$blocker)); + } + } + + $replicaStatus = $status['raw']['replica_status'] ?? []; + if (is_array($replicaStatus)) { + foreach (['Last_IO_Error', 'Last_SQL_Error', 'Last_Error'] as $errorKey) { + $error = trim((string)($replicaStatus[$errorKey] ?? '')); + if ($error !== '') { + $hosts = array_merge($hosts, self::databaseDeniedAccountHostsFromText($error)); + } + } + } + + return array_values(array_unique($hosts)); + } + + private static function databaseDeniedAccountHostsFromText(string $text): array + { + preg_match_all('/Access denied for user\s+[\'"][^\'"]+[\'"]@[\'"]([^\'"]+)[\'"]/i', $text, $matches); + return array_values(array_unique(array_filter($matches[1] ?? []))); + } + + private static function normalizeDatabaseAccountHost(string $host): ?string + { + $host = trim($host); + if ($host === '') { + return null; + } + + if ($host !== '%') { + $host = trim($host, '[]'); + } + + if ($host === '' || strlen($host) > 255) { + return null; + } + + if (preg_match('/[\s\'"`;\\\\]/', $host)) { + return null; + } + + return preg_match('/^[A-Za-z0-9_.:%-]+$/', $host) === 1 ? $host : null; + } + + private static function databaseAccountHostGrantCandidates(string $host): array + { + $host = self::normalizeDatabaseAccountHost($host); + if ($host === null) { + return []; + } + + $candidates = [$host]; + if ($host !== '%' && !str_contains($host, '%')) { + if (filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false) { + $lastDot = strrpos($host, '.'); + if ($lastDot !== false) { + $candidates[] = substr($host, 0, $lastDot + 1) . '%'; + } + } elseif (str_contains($host, ':')) { + $lastColon = strrpos($host, ':'); + if ($lastColon !== false) { + $candidates[] = substr($host, 0, $lastColon + 1) . '%'; + } + } + } + + return array_values(array_unique(array_filter(array_map( + static fn(string $candidate): ?string => self::normalizeDatabaseAccountHost($candidate), + $candidates + )))); + } + + private function ensureDatabaseReplicationUser(array $primary, string $replicationUser, string $replicationPassword, array $grantHosts = []): void { if (trim($replicationUser) === '' || trim($replicationPassword) === '') { throw new RuntimeException('Replication username and password are required.'); @@ -1169,15 +2013,26 @@ class replication_manager $connection = $this->databaseConnection($primary, true); try { - $account = sprintf( - "'%s'@'%%'", - $connection->real_escape_string($replicationUser) - ); + $grantHosts = $grantHosts === [] ? ['%'] : $grantHosts; + $user = $connection->real_escape_string($replicationUser); $password = $connection->real_escape_string($replicationPassword); - $this->mysqliExec($connection, "CREATE USER IF NOT EXISTS " . $account . " IDENTIFIED BY '" . $password . "'"); - $this->mysqliExec($connection, "ALTER USER " . $account . " IDENTIFIED BY '" . $password . "'"); - $this->mysqliExec($connection, "GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " . $account); + foreach ($grantHosts as $grantHost) { + $grantHost = self::normalizeDatabaseAccountHost((string)$grantHost); + if ($grantHost === null) { + continue; + } + + $account = sprintf( + "'%s'@'%s'", + $user, + $connection->real_escape_string($grantHost) + ); + + $this->mysqliExec($connection, "CREATE USER IF NOT EXISTS " . $account . " IDENTIFIED BY '" . $password . "'"); + $this->mysqliExec($connection, "ALTER USER " . $account . " IDENTIFIED BY '" . $password . "'"); + $this->mysqliExec($connection, "GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " . $account); + } $this->mysqliExec($connection, 'FLUSH PRIVILEGES'); } catch (Throwable $throwable) { throw new RuntimeException( @@ -1370,35 +2225,31 @@ class replication_manager $host = $this->getHost(self::KIND_REDIS, (int)$host['id']); $status = $this->testRedisHost($host); - $this->storeStatus($host, $status); - $progress = min(99.99, max(5.0, (float)$status['replication_percent'])); $syncBlockers = array_values(array_intersect($status['blockers'], [ 'Redis host is not currently a replica.', 'Redis replica link to primary is not up.', ])); $onlySyncBlockers = $status['blockers'] === [] || ($syncBlockers !== [] && count($syncBlockers) === count($status['blockers'])); + $progress = self::redisProvisionProgress((float)$status['replication_percent'], $syncBlockers); if ($onlySyncBlockers && ((float)$status['replication_percent'] < 100.0 || $syncBlockers !== [])) { $message = $syncBlockers !== [] - ? 'Redis replica is waiting for the primary link.' - : 'Redis replica is syncing from the primary.'; + ? 'Redis replication is configured, but the replica is waiting for the primary link.' + : 'Redis replication is configured and syncing in the background.'; + $this->storeStatus($host, array_replace($status, ['replication_percent' => $progress])); $this->updateOperationProgress($operationId, $progress, $message, $context); return [ 'ok' => true, + 'healthy' => false, 'message' => $message, 'blockers' => $status['blockers'], 'replication_percent' => $progress, - 'operation' => [ - 'id' => $operationId, - 'status' => 'running', - 'progress_percent' => $progress, - 'message' => $message, - ], - 'host' => $this->publicHost($host), + 'host' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$host['id'])), ]; } + $this->storeStatus($host, $status); if ($status['blockers'] !== []) { return [ 'ok' => false, @@ -1426,8 +2277,14 @@ class replication_manager throw new RuntimeException('No MinIO primary is registered.'); } - $targetStatus = $this->testMinioHost(array_merge($host, ['test_connectivity_only' => true])); - $primaryStatus = $this->testMinioHost($primary); + $targetStatus = $this->testMinioHost(array_merge($host, [ + 'test_connectivity_only' => true, + 'skip_storage_scan' => true, + ])); + $primaryStatus = $this->testMinioHost(array_merge($primary, [ + 'test_connectivity_only' => true, + 'skip_storage_scan' => true, + ])); $blockers = array_values(array_unique(array_merge($targetStatus['blockers'], $primaryStatus['blockers']))); if ($blockers !== []) { $this->storeStatus($host, array_replace($targetStatus, ['blockers' => $blockers])); @@ -1441,7 +2298,8 @@ class replication_manager } $context = $this->operationContext($operationId); - if (($context['phase'] ?? '') !== 'configured') { + $replicationConfigured = $this->minioReplicationConfiguredForHosts($primary, $host); + if (!$replicationConfigured) { try { $this->configureMinioReplication($primary, $host); } catch (Throwable $throwable) { @@ -1480,18 +2338,14 @@ class replication_manager ); $host = $this->getHost(self::KIND_MINIO, (int)$host['id']); - $status = $this->testMinioHost($host); + $status = $this->minioProvisionStatus($primary, $host); $this->storeStatus($host, $status); - $progress = min(99.99, max(5.0, (float)$status['replication_percent'])); - $syncBlockers = array_values(array_intersect($status['blockers'], [ - 'MinIO replica has not caught up.', - 'MinIO bucket replication is not configured.', - ])); - $onlySyncBlockers = $status['blockers'] === [] - || ($syncBlockers !== [] && count($syncBlockers) === count($status['blockers'])); - if ($onlySyncBlockers && ((float)$status['replication_percent'] < 100.0 || $syncBlockers !== [])) { - $message = 'MinIO replica is syncing from the primary.'; + $progress = self::minioProvisionProgress($status); + $syncInProgress = ((float)$status['replication_percent'] < 100.0 || $status['blockers'] !== []) + && self::minioOnlyProgressBlockers($status['blockers']); + if ($syncInProgress) { + $message = self::minioProvisionProgressMessage($status); $this->updateOperationProgress($operationId, $progress, $message, $context); return [ 'ok' => true, @@ -1504,7 +2358,7 @@ class replication_manager 'progress_percent' => $progress, 'message' => $message, ], - 'host' => $this->publicHost($host), + 'host' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])), ]; } @@ -1660,6 +2514,178 @@ class replication_manager ]; } + private function runAutomaticFailoverForKind(string $kind, array $config, ?int $actorUserId): array + { + if (!replica_failover_manager::kindEnabled($config, $kind)) { + return [ + 'ok' => true, + 'status' => 'skipped', + 'reason' => 'disabled', + ]; + } + + $primary = $this->primaryHost($kind); + if ($primary === null) { + return [ + 'ok' => false, + 'status' => 'skipped', + 'reason' => 'missing_primary', + ]; + } + + if (!$this->primaryHostDown($kind, $primary)) { + return [ + 'ok' => true, + 'status' => 'skipped', + 'reason' => 'primary_healthy', + 'primary' => $this->publicHost($primary), + ]; + } + + $maxAgeSeconds = (int)$config['max_status_age_seconds']; + $candidate = replica_failover_manager::snapshotFailoverCandidate($this->listHosts($kind), $kind, $maxAgeSeconds); + if ($candidate === null) { + $result = [ + 'ok' => false, + 'status' => 'blocked', + 'reason' => 'no_fresh_caught_up_replica', + 'primary' => $this->publicHost($primary), + ]; + $this->audit($kind, (int)$primary['id'], 'automatic_failover_blocked', $actorUserId, 'warning', $result); + return $result; + } + + if (!replica_failover_manager::candidateReachable($kind, $candidate)) { + $result = [ + 'ok' => false, + 'status' => 'blocked', + 'reason' => 'candidate_unreachable', + 'primary' => $this->publicHost($primary), + 'candidate' => $this->publicHost($candidate), + ]; + $this->audit($kind, (int)$candidate['id'], 'automatic_failover_blocked', $actorUserId, 'warning', $result); + return $result; + } + + $operationId = $this->startOperation($kind, (int)$candidate['id'], 'automatic_failover', $actorUserId); + $owner = 'replication-auto-failover-' . $kind . '-' . (int)$candidate['id'] . '-' . bin2hex(random_bytes(4)); + $lockHandle = $this->acquirePromotionLock(); + + try { + application_write_freeze::freeze('Automatic replica failover in progress.', $owner, 600); + $result = match ($kind) { + self::KIND_DATABASE => $this->promoteDatabaseHostForFailover($candidate, $primary, $maxAgeSeconds), + self::KIND_REDIS => $this->promoteRedisHostForFailover($candidate, $primary, $maxAgeSeconds), + self::KIND_MINIO => $this->promoteMinioHostForFailover($candidate, $primary, $maxAgeSeconds), + }; + + $this->finishOperation($operationId, 'completed', 100, $result['message'] ?? null, []); + $this->audit($kind, (int)$candidate['id'], 'automatic_failover_promoted', $actorUserId, 'critical', $result); + return array_merge($result, [ + 'status' => 'promoted', + 'candidate' => $this->publicHost($this->getHost($kind, (int)$candidate['id'])), + ]); + } catch (Throwable $throwable) { + $this->finishOperation($operationId, 'failed', 0, null, [$throwable->getMessage()]); + $this->audit($kind, (int)$candidate['id'], 'automatic_failover_failed', $actorUserId, 'error', [ + 'error' => $throwable->getMessage(), + ]); + return [ + 'ok' => false, + 'status' => 'failed', + 'reason' => $throwable->getMessage(), + 'candidate' => $this->publicHost($candidate), + ]; + } finally { + application_write_freeze::unfreeze($owner); + $this->releasePromotionLock($lockHandle); + } + } + + private function primaryHostDown(string $kind, array $primary): bool + { + $activeConfig = replica_failover_manager::activeConfigFromHost($kind, $primary); + if ($activeConfig === null) { + return false; + } + + return replica_failover_manager::activePrimaryIsDown($kind, $activeConfig); + } + + private function promoteDatabaseHostForFailover(array $host, array $oldPrimary, int $maxAgeSeconds): array + { + if (!replica_failover_manager::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds)) { + throw new RuntimeException('Database failover blocked: replica status is not fresh and caught up.'); + } + + $targetConn = $this->databaseConnection($host, true); + try { + $targetStatus = $this->databaseServerStatus($targetConn); + $this->stopDatabaseReplication($targetConn, $targetStatus); + $this->setDatabaseReadOnly($targetConn, $targetStatus, false); + $this->switchPrimary(self::KIND_DATABASE, (int)$host['id'], (int)$oldPrimary['id']); + $this->writeBootstrapSnapshot(); + replication_bootstrap_config::applyToGlobals(replication_bootstrap_config::loadSnapshot()); + } finally { + $targetConn->close(); + } + + return [ + 'ok' => true, + 'message' => 'Database replica promoted to primary after primary health check failed.', + 'primary' => $this->publicHost($this->getHost(self::KIND_DATABASE, (int)$host['id'])), + 'prior_primary' => $this->publicHost($this->getHost(self::KIND_DATABASE, (int)$oldPrimary['id'], true)), + 'blockers' => [], + ]; + } + + private function promoteRedisHostForFailover(array $host, array $oldPrimary, int $maxAgeSeconds): array + { + if (!replica_failover_manager::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds)) { + throw new RuntimeException('Redis failover blocked: replica status is not fresh and caught up.'); + } + + $client = $this->redisClient($host); + $client->ping(); + $client->executeRaw(['REPLICAOF', 'NO', 'ONE']); + try { + $client->executeRaw(['CONFIG', 'REWRITE']); + } catch (Throwable) { + } + + $this->switchPrimary(self::KIND_REDIS, (int)$host['id'], (int)$oldPrimary['id']); + $this->writeBootstrapSnapshot(); + replication_bootstrap_config::applyToGlobals(replication_bootstrap_config::loadSnapshot()); + + return [ + 'ok' => true, + 'message' => 'Redis replica promoted to primary after primary health check failed.', + 'primary' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$host['id'])), + 'prior_primary' => $this->publicHost($this->getHost(self::KIND_REDIS, (int)$oldPrimary['id'], true)), + 'blockers' => [], + ]; + } + + private function promoteMinioHostForFailover(array $host, array $oldPrimary, int $maxAgeSeconds): array + { + if (!replica_failover_manager::snapshotHostIsStrictlyFresh($host, $maxAgeSeconds)) { + throw new RuntimeException('MinIO failover blocked: replica status is not fresh and caught up.'); + } + + $this->minioS3Client($host)->listBuckets(); + $this->switchPrimary(self::KIND_MINIO, (int)$host['id'], (int)$oldPrimary['id']); + $this->writeBootstrapSnapshot(); + replication_bootstrap_config::applyToGlobals(replication_bootstrap_config::loadSnapshot()); + + return [ + 'ok' => true, + 'message' => 'MinIO replica endpoint selected after primary health check failed.', + 'primary' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$host['id'])), + 'prior_primary' => $this->publicHost($this->getHost(self::KIND_MINIO, (int)$oldPrimary['id'], true)), + 'blockers' => [], + ]; + } + private function testDatabaseHost(array $host): array { $blockers = []; @@ -1747,14 +2773,277 @@ class replication_manager } $blockers = array_values(array_unique(array_filter($blockers))); - return [ - 'status' => !$reachable ? 'down' : ($blockers === [] ? 'ok' : 'degraded'), + $status = [ + 'status' => self::replicationHealthStatus( + $reachable, + (string)($host['role'] ?? ''), + (float)$percent, + $blockers, + empty($host['test_connectivity_only']) + ), 'replication_percent' => round($percent, 2), 'lag_seconds' => $lagSeconds, 'blockers' => $blockers, 'raw' => $raw, 'checked_at' => date('c'), ]; + + if ($this->shouldRepairDatabaseReplicationAccess($host, $status)) { + $repair = $this->repairDatabaseReplicationAccess($host, $status); + if (($repair['ok'] ?? false) === true) { + $retested = $this->testDatabaseHost(array_merge($host, [ + 'skip_replication_access_repair' => true, + 'skip_replication_thread_repair' => true, + ])); + $retested['raw']['replication_access_repair'] = $repair; + return $retested; + } + + if (!empty($repair['message'])) { + $status['blockers'][] = 'Database replication access repair failed: ' . $repair['message']; + $status['blockers'] = array_values(array_unique(array_filter($status['blockers']))); + $status['status'] = self::replicationHealthStatus( + $reachable, + (string)($host['role'] ?? ''), + (float)$status['replication_percent'], + $status['blockers'] + ); + } + $status['raw']['replication_access_repair'] = $repair; + } + + if ($this->shouldRepairDatabaseReplicationThreads($host, $status)) { + $repair = $this->repairDatabaseReplicationThreads($host); + if (($repair['ok'] ?? false) === true) { + $retested = $this->testDatabaseHost(array_merge($host, [ + 'skip_replication_access_repair' => true, + 'skip_replication_thread_repair' => true, + ])); + $retested['raw']['replication_thread_repair'] = $repair; + return $retested; + } + + if (!empty($repair['message'])) { + $status['blockers'][] = 'Database replication thread restart failed: ' . $repair['message']; + $status['blockers'] = array_values(array_unique(array_filter($status['blockers']))); + $status['status'] = self::replicationHealthStatus( + $reachable, + (string)($host['role'] ?? ''), + (float)$status['replication_percent'], + $status['blockers'] + ); + } + $status['raw']['replication_thread_repair'] = $repair; + } + + return $status; + } + + private function shouldRepairDatabaseReplicationAccess(array $host, array $status): bool + { + if (!empty($host['skip_replication_access_repair']) + || !empty($host['test_connectivity_only']) + || (string)($host['role'] ?? '') === 'primary') { + return false; + } + + if ($this->databaseDeniedAccountHostsFromStatus($status) === []) { + return false; + } + + $hostCredentials = $this->credentials($host); + $primary = $this->primaryHost(self::KIND_DATABASE); + $primaryCredentials = $primary !== null ? $this->credentials($primary) : []; + + return (($hostCredentials['replication_username'] ?? '') !== '' && ($hostCredentials['replication_password'] ?? '') !== '') + || (($primaryCredentials['replication_username'] ?? '') !== '' && ($primaryCredentials['replication_password'] ?? '') !== ''); + } + + private function repairDatabaseReplicationAccess(array $host, array $status): array + { + $deniedHosts = $this->databaseDeniedAccountHostsFromStatus($status); + if ($deniedHosts === []) { + return [ + 'ok' => false, + 'skipped' => true, + 'message' => 'No denied replication account host was detected.', + ]; + } + + $primary = $this->primaryHost(self::KIND_DATABASE); + if ($primary === null) { + return [ + 'ok' => false, + 'message' => 'No database primary is registered.', + ]; + } + + $primaryCredentials = $this->credentials($primary); + $hostCredentials = $this->credentials($host); + $replicationUser = $hostCredentials['replication_username'] + ?: ($primaryCredentials['replication_username'] ?? ''); + $replicationPassword = $hostCredentials['replication_password'] + ?: ($primaryCredentials['replication_password'] ?? ''); + + if ($replicationUser === '' || $replicationPassword === '') { + return [ + 'ok' => false, + 'skipped' => true, + 'denied_hosts' => $deniedHosts, + 'message' => 'Replication credentials are not available for automatic grant repair.', + ]; + } + + try { + $grantHosts = $this->databaseReplicationGrantHosts($host, $status); + $this->ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword, $grantHosts); + + $target = $this->databaseConnection($host, true); + try { + $this->refreshDatabaseReplicationConnection( + $target, + $this->databaseServerStatus($target), + $primary, + $replicationUser, + $replicationPassword + ); + } finally { + $target->close(); + } + + return [ + 'ok' => true, + 'denied_hosts' => $deniedHosts, + 'grant_hosts' => $grantHosts, + ]; + } catch (Throwable $throwable) { + return [ + 'ok' => false, + 'denied_hosts' => $deniedHosts, + 'message' => $throwable->getMessage(), + ]; + } + } + + private function shouldRepairDatabaseReplicationThreads(array $host, array $status): bool + { + if (!empty($host['skip_replication_thread_repair']) + || !empty($host['test_connectivity_only']) + || (string)($host['role'] ?? '') === 'primary') { + return false; + } + + $replicaStatus = $status['raw']['replica_status'] ?? null; + return is_array($replicaStatus) + && $replicaStatus !== [] + && self::databaseOnlyReplicationThreadBlockers($status['blockers'] ?? []); + } + + private static function databaseOnlyReplicationThreadBlockers(array $blockers): bool + { + $blockers = array_values(array_filter(array_map( + static fn(mixed $blocker): string => trim((string)$blocker), + $blockers + ))); + if ($blockers === []) { + return false; + } + + $allowed = [ + 'Database replication IO and SQL threads must both be running.', + 'Database replication IO thread is not running.', + 'Database replication SQL thread is not running.', + ]; + + return array_values(array_diff($blockers, $allowed)) === []; + } + + private function repairDatabaseReplicationThreads(array $host): array + { + try { + $target = $this->databaseConnection($host, true); + try { + $this->restartDatabaseReplicationThreads($target, $this->databaseServerStatus($target)); + } finally { + $target->close(); + } + + return ['ok' => true]; + } catch (Throwable $throwable) { + return [ + 'ok' => false, + 'message' => $throwable->getMessage(), + ]; + } + } + + private function refreshDatabaseReplicationConnection( + mysqli $target, + array $serverStatus, + array $primary, + string $replicationUser, + string $replicationPassword + ): void { + $isMariaDb = self::databaseEngine($serverStatus) === 'mariadb'; + if ($isMariaDb) { + try { + $this->mysqliExec($target, 'STOP SLAVE'); + } catch (Throwable) { + } + $sql = sprintf( + "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, MASTER_USER = '%s', MASTER_PASSWORD = '%s', MASTER_USE_GTID = slave_pos", + $target->real_escape_string((string)$primary['host']), + (int)$primary['port'], + $target->real_escape_string($replicationUser), + $target->real_escape_string($replicationPassword) + ); + $this->mysqliExec($target, $sql); + } else { + try { + $this->mysqliExec($target, 'STOP REPLICA'); + } catch (Throwable) { + } + $sql = sprintf( + "CHANGE REPLICATION SOURCE TO SOURCE_HOST = '%s', SOURCE_PORT = %d, SOURCE_USER = '%s', SOURCE_PASSWORD = '%s', SOURCE_AUTO_POSITION = 1", + $target->real_escape_string((string)$primary['host']), + (int)$primary['port'], + $target->real_escape_string($replicationUser), + $target->real_escape_string($replicationPassword) + ); + $this->mysqliExec($target, $sql); + } + + $this->restartDatabaseReplicationThreads($target, $serverStatus); + } + + private function restartDatabaseReplicationThreads(mysqli $target, array $serverStatus): void + { + $isMariaDb = self::databaseEngine($serverStatus) === 'mariadb'; + $startStatements = $isMariaDb + ? ['START SLAVE', 'START SLAVE IO_THREAD', 'START SLAVE SQL_THREAD'] + : ['START REPLICA', 'START REPLICA IO_THREAD', 'START REPLICA SQL_THREAD']; + + $lastError = null; + $startedAnyThread = false; + foreach ($startStatements as $index => $statement) { + try { + $this->mysqliExec($target, $statement); + if ($index === 0) { + return; + } + $startedAnyThread = true; + } catch (Throwable $throwable) { + $lastError = $throwable; + } + } + + if ($startedAnyThread) { + return; + } + + if ($lastError !== null) { + throw $lastError; + } } private function testRedisHost(array $host): array @@ -1791,9 +3080,7 @@ class replication_manager } else { $primaryClient = $this->redisClient($primary); $primaryInfo = $this->redisInfo($primaryClient); - $primaryOffset = (int)($primaryInfo['master_repl_offset'] ?? 0); - $replicaOffset = (int)($info['slave_repl_offset'] ?? $info['master_repl_offset'] ?? 0); - $percent = self::redisOffsetPercent($primaryOffset, $replicaOffset); + $percent = self::redisReplicationPercentFromInfo($primaryInfo, $info); $raw['primary_replication'] = $primaryInfo; if (!in_array(strtolower((string)($info['role'] ?? '')), ['slave', 'replica'], true)) { @@ -1814,7 +3101,13 @@ class replication_manager $blockers = array_values(array_unique(array_filter($blockers))); return [ - 'status' => !$reachable ? 'down' : ($blockers === [] ? 'ok' : 'degraded'), + 'status' => self::replicationHealthStatus( + $reachable, + (string)($host['role'] ?? ''), + (float)$percent, + $blockers, + empty($host['test_connectivity_only']) + ), 'replication_percent' => round($percent, 2), 'lag_seconds' => null, 'blockers' => $blockers, @@ -1829,6 +3122,12 @@ class replication_manager $raw = []; $percent = (float)(($host['role'] ?? '') === 'primary' ? 100 : 0); $reachable = true; + $connectivityOnly = !empty($host['test_connectivity_only']); + $isPrimary = (string)($host['role'] ?? '') === 'primary'; + $forceStorageScan = !empty($host['force_storage_scan']); + $measureStorage = !$connectivityOnly + && empty($host['skip_storage_scan']) + && $forceStorageScan; $options = $this->decodeOptions($host); $buckets = self::normalizeMinioBuckets($options['buckets'] ?? self::MINIO_DEFAULT_BUCKETS); @@ -1836,52 +3135,92 @@ class replication_manager $client = $this->minioS3Client($host); $client->listBuckets(); - if (($host['role'] ?? '') === 'primary') { - $sourceStats = $this->minioBucketStats($client, $buckets, true, 'MinIO source bucket'); + if ($isPrimary) { + $sourceStats = $this->minioBucketStats($client, $buckets, true, 'MinIO source bucket', $measureStorage); $blockers = array_merge($blockers, $sourceStats['blockers']); $raw['buckets'] = $sourceStats['buckets']; $raw['storage'] = [ 'source_bytes' => $sourceStats['bytes'], 'source_objects' => $sourceStats['objects'], + 'measured' => $measureStorage, ]; } else { - $targetStats = $this->minioBucketStats($client, $buckets, empty($host['test_connectivity_only']), 'MinIO target bucket'); + $retentionDaysByBucket = self::minioReplicaRetentionDaysByBucket($buckets); + $targetStats = $this->minioBucketStats( + $client, + $buckets, + !$connectivityOnly, + 'MinIO target bucket', + $measureStorage, + $retentionDaysByBucket + ); $blockers = array_merge($blockers, $targetStats['blockers']); $raw['target_buckets'] = $targetStats['buckets']; $raw['storage'] = [ 'target_bytes' => $targetStats['bytes'], 'target_objects' => $targetStats['objects'], + 'target_expired_bytes' => $targetStats['expired_bytes'] ?? 0, + 'target_expired_objects' => $targetStats['expired_objects'] ?? 0, + 'measured' => $measureStorage, ]; - $primary = $this->primaryHost(self::KIND_MINIO); - if ($primary === null) { - $blockers[] = 'No MinIO primary is registered.'; - } else { - $sourceStats = $this->minioBucketStats($this->minioS3Client($primary), $buckets, true, 'MinIO source bucket'); - $blockers = array_merge($blockers, $sourceStats['blockers']); - $headroom = (float)($options['space_headroom_percent'] ?? self::MINIO_SPACE_HEADROOM_PERCENT); - $requiredBytes = self::minioRequiredFreeBytes((int)$sourceStats['bytes'], $headroom); - $availableBytes = $this->minioTargetFreeBytes($host); - $spaceBlockers = self::minioSpaceBlockers($availableBytes, $requiredBytes); - $blockers = array_merge($blockers, $spaceBlockers); - $percent = self::minioByteReplicationPercent((int)$sourceStats['bytes'], (int)$targetStats['bytes']); - if (empty($host['test_connectivity_only'])) { + if (!$connectivityOnly) { + $primary = $this->primaryHost(self::KIND_MINIO); + if ($primary === null) { + $blockers[] = 'No MinIO primary is registered.'; + } else { + $sourceStats = $this->minioBucketStats( + $this->minioS3Client($primary), + $buckets, + true, + 'MinIO source bucket', + $measureStorage, + $retentionDaysByBucket + ); + $blockers = array_merge($blockers, $sourceStats['blockers']); + $blockers = array_merge($blockers, self::minioBackupRetentionBlockers($targetStats)); + $headroom = (float)($options['space_headroom_percent'] ?? self::MINIO_SPACE_HEADROOM_PERCENT); + $availableBytes = $measureStorage ? $this->minioTargetFreeBytes($host) : null; + $requiredBytes = $measureStorage ? self::minioRequiredFreeBytes((int)$sourceStats['bytes'], $headroom) : null; + $spaceBlockers = $requiredBytes === null ? [] : self::minioSpaceBlockers($availableBytes, $requiredBytes); + $blockers = array_merge($blockers, $spaceBlockers); + $percent = $measureStorage + ? self::minioByteReplicationPercent((int)$sourceStats['bytes'], (int)$targetStats['bytes']) + : self::minioIncompleteProgress(self::lastStatusReplicationPercent($host, 5.0)); $replicationConfigured = $this->minioReplicationConfigured($primary, $buckets); + $progressStatus = null; + if ($replicationConfigured) { + $progressStatus = $this->minioReplicationProgressStatus($primary, $host, $buckets); + if ($progressStatus !== null) { + $percent = round((float)$progressStatus['replication_percent'], 2); + $raw['progress_source'] = 'minio_replicate_status'; + $raw['replication_status'] = $progressStatus; + } else { + $raw['progress_source'] = 'minio_replicate_status_unavailable'; + } + } if (!$replicationConfigured) { $blockers[] = 'MinIO bucket replication is not configured.'; - } elseif ($percent < 100.0) { - $blockers[] = 'MinIO replica has not caught up.'; + } elseif (in_array(self::MINIO_BACKUP_BUCKET, $buckets, true) + && !$this->minioBackupReplicaRetentionConfigured($host, self::MINIO_BACKUP_BUCKET)) { + $blockers[] = 'MinIO backup replica retention is not configured for the backups bucket.'; + } elseif (($progressStatus !== null || $measureStorage) && $percent < 100.0) { + $blockers[] = self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER; + } elseif ($progressStatus === null && !$measureStorage) { + $blockers[] = self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER; } + $raw['source_buckets'] = $sourceStats['buckets']; + $raw['storage'] = array_merge($raw['storage'], [ + 'source_bytes' => $sourceStats['bytes'], + 'source_objects' => $sourceStats['objects'], + 'source_expired_bytes' => $sourceStats['expired_bytes'] ?? 0, + 'source_expired_objects' => $sourceStats['expired_objects'] ?? 0, + 'required_free_bytes' => $requiredBytes, + 'available_free_bytes' => $availableBytes, + 'space_headroom_percent' => $headroom, + 'space_ok' => $measureStorage ? ($availableBytes === null ? null : $spaceBlockers === []) : null, + ]); } - $raw['source_buckets'] = $sourceStats['buckets']; - $raw['storage'] = array_merge($raw['storage'], [ - 'source_bytes' => $sourceStats['bytes'], - 'source_objects' => $sourceStats['objects'], - 'required_free_bytes' => $requiredBytes, - 'available_free_bytes' => $availableBytes, - 'space_headroom_percent' => $headroom, - 'space_ok' => $spaceBlockers === [], - ]); } } } catch (Throwable $throwable) { @@ -1891,7 +3230,13 @@ class replication_manager $blockers = array_values(array_unique(array_filter($blockers))); return [ - 'status' => !$reachable ? 'down' : ($blockers === [] ? 'ok' : 'degraded'), + 'status' => self::replicationHealthStatus( + $reachable, + (string)($host['role'] ?? ''), + (float)$percent, + $blockers, + !$connectivityOnly + ), 'replication_percent' => round($percent, 2), 'lag_seconds' => null, 'blockers' => $blockers, @@ -1900,6 +3245,150 @@ class replication_manager ]; } + private function minioProgressScanHost(array $host): array + { + if (!$this->minioCanReuseRecentMeasuredStatus($host)) { + return $host; + } + + return array_merge($host, ['skip_storage_scan' => true]); + } + + private function minioCanReuseRecentMeasuredStatus(array $host): bool + { + $lastStatus = self::jsonDecode($host['last_status_json'] ?? null); + if (!is_array($lastStatus)) { + return false; + } + + $storage = $lastStatus['raw']['storage'] ?? null; + if (!is_array($storage) || empty($storage['measured'])) { + return false; + } + + $checkedAt = strtotime((string)($lastStatus['checked_at'] ?? $host['last_checked_at'] ?? '')); + if ($checkedAt === false) { + return false; + } + + return (time() - $checkedAt) < self::MINIO_PROGRESS_SCAN_INTERVAL_SECONDS; + } + + private static function minioOnlyProgressBlockers(array $blockers): bool + { + $blockers = array_values(array_filter(array_map( + static fn(mixed $blocker): string => trim((string)$blocker), + $blockers + ))); + if ($blockers === []) { + return true; + } + + return array_values(array_diff($blockers, [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER])) === []; + } + + private static function minioProvisionProgressMessage(array $status): string + { + $storage = is_array($status['raw']['storage'] ?? null) ? $status['raw']['storage'] : []; + if (!empty($storage['measured'])) { + $targetBytes = (int)($storage['target_bytes'] ?? 0); + $sourceBytes = (int)($storage['source_bytes'] ?? 0); + if ($sourceBytes > 0) { + return 'MinIO replica is syncing. Copied ' . $targetBytes . ' of ' . $sourceBytes . ' bytes.'; + } + } + + if ((string)($status['raw']['progress_source'] ?? '') === 'minio_replicate_status') { + $percent = round((float)($status['replication_percent'] ?? 0), 2); + return 'MinIO replica is syncing. Replication status reports ' . $percent . '% complete.'; + } + + return 'MinIO replica is syncing in the background. Waiting for the next progress sample.'; + } + + private function minioProvisionStatus(array $primary, array $host): array + { + $status = $this->testMinioHost(array_merge($host, [ + 'test_connectivity_only' => true, + 'skip_storage_scan' => true, + ])); + if ($status['blockers'] !== []) { + return $status; + } + + $primaryOptions = $this->decodeOptions($primary); + $targetOptions = $this->decodeOptions($host); + $buckets = self::normalizeMinioBuckets($targetOptions['buckets'] ?? $primaryOptions['buckets'] ?? self::MINIO_DEFAULT_BUCKETS); + $progress = $this->minioReplicationProgressStatus($primary, $host, $buckets); + + if ($progress === null) { + $percent = self::minioIncompleteProgress(self::lastStatusReplicationPercent($host, 5.0)); + $status['replication_percent'] = $percent; + $status['blockers'] = array_values(array_unique(array_merge( + $status['blockers'], + [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER] + ))); + $status['raw']['progress_source'] = 'minio_replicate_status_unavailable'; + $status['raw']['replication_status'] = [ + 'available' => false, + 'message' => 'MinIO replication status did not report progress yet.', + ]; + $status['status'] = self::replicationHealthStatus( + true, + (string)($host['role'] ?? ''), + $percent, + $status['blockers'] + ); + + return $status; + } + + $status['replication_percent'] = round((float)$progress['replication_percent'], 2); + $status['blockers'] = array_values(array_unique(array_merge( + $status['blockers'], + $progress['blockers'] ?? [] + ))); + $status['raw']['progress_source'] = 'minio_replicate_status'; + $status['raw']['replication_status'] = $progress; + $status = self::normalizeMinioCaughtUpStatus($status, (string)($host['role'] ?? '')); + $status['status'] = self::replicationHealthStatus( + true, + (string)($host['role'] ?? ''), + (float)$status['replication_percent'], + $status['blockers'] + ); + + return $status; + } + + private static function lastStatusReplicationPercent(array $host, float $default): float + { + $lastStatus = self::jsonDecode($host['last_status_json'] ?? null); + if (is_array($lastStatus) && is_numeric($lastStatus['replication_percent'] ?? null)) { + return round((float)$lastStatus['replication_percent'], 2); + } + + return $default; + } + + private static function normalizeMinioCaughtUpStatus(array $status, string $role): array + { + if ($role === 'primary' || round((float)($status['replication_percent'] ?? 0), 2) < 100.0) { + return $status; + } + + $blockers = array_values(array_filter(array_map( + static fn(mixed $blocker): string => trim((string)$blocker), + $status['blockers'] ?? [] + ))); + if ($blockers === []) { + return $status; + } + + $status['blockers'] = array_values(array_diff($blockers, [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER])); + return $status; + } + public static function databasePrerequisiteBlockers(array $status): array { if (self::databaseEngine($status) === 'mariadb') { @@ -1980,6 +3469,11 @@ class replication_manager return stripos((string)($status['server_version'] ?? ''), 'mariadb') !== false ? 'mariadb' : 'mysql'; } + private static function databaseEngineKnown(array $status): bool + { + return trim((string)($status['server_version'] ?? '')) !== ''; + } + private static function databaseGtidPosition(array $status): string { if (self::databaseEngine($status) === 'mariadb') { @@ -2752,7 +4246,7 @@ class replication_manager ); $sample = $qualifiedTables !== [] ? ': ' . implode(', ', $qualifiedTables) : ''; - return 'Replica seed includes data for schema-only tables' . $sample . '. Re-run provisioning to rebuild the replica without operational log data.'; + return 'Replica seed includes data for schema-only tables' . $sample . '. Re-run provisioning to rebuild the replica without schema-only table data.'; } private function minioS3Client(array $host): S3Client @@ -2763,6 +4257,11 @@ class replication_manager 'region' => 'us-east-1', 'endpoint' => self::minioEndpoint($host), 'use_path_style_endpoint' => true, + 'retries' => 0, + 'http' => [ + 'connect_timeout' => self::MINIO_S3_CONNECT_TIMEOUT_SECONDS, + 'timeout' => self::MINIO_S3_REQUEST_TIMEOUT_SECONDS, + ], 'credentials' => [ 'key' => $credentials['username'], 'secret' => $credentials['password'], @@ -2770,16 +4269,39 @@ class replication_manager ]); } - private function minioBucketStats(S3Client $client, array $buckets, bool $requireExists, string $missingPrefix): array + private static function minioObjectLastModifiedTimestamp(mixed $value): ?int + { + if ($value instanceof \DateTimeInterface) { + return $value->getTimestamp(); + } + if (is_numeric($value)) { + return (int)$value; + } + $timestamp = strtotime((string)$value); + return $timestamp === false ? null : $timestamp; + } + + private function minioBucketStats( + S3Client $client, + array $buckets, + bool $requireExists, + string $missingPrefix, + bool $measureObjects = true, + array $retentionDaysByBucket = [] + ): array { $stats = [ 'bytes' => 0, 'objects' => 0, + 'expired_bytes' => 0, + 'expired_objects' => 0, 'buckets' => [], 'blockers' => [], ]; foreach ($buckets as $bucket) { + $retentionDays = isset($retentionDaysByBucket[$bucket]) ? (int)$retentionDaysByBucket[$bucket] : null; + $retentionCutoff = $retentionDays !== null ? time() - ($retentionDays * 86400) : null; try { $exists = (bool)$client->doesBucketExist($bucket); if (!$exists) { @@ -2795,8 +4317,24 @@ class replication_manager continue; } + if (!$measureObjects) { + $stats['buckets'][] = [ + 'name' => $bucket, + 'status' => 'ok', + 'bytes' => null, + 'objects' => null, + 'expired_bytes' => null, + 'expired_objects' => null, + 'measured' => false, + 'retention_days' => $retentionDays, + ]; + continue; + } + $bucketBytes = 0; $bucketObjects = 0; + $bucketExpiredBytes = 0; + $bucketExpiredObjects = 0; $token = null; do { $args = ['Bucket' => $bucket]; @@ -2805,7 +4343,15 @@ class replication_manager } $result = $client->listObjectsV2($args); foreach (($result['Contents'] ?? []) as $object) { - $bucketBytes += (int)($object['Size'] ?? 0); + $size = (int)($object['Size'] ?? 0); + $lastModified = self::minioObjectLastModifiedTimestamp($object['LastModified'] ?? null); + if ($retentionCutoff !== null && $lastModified !== null && $lastModified < $retentionCutoff) { + $bucketExpiredBytes += $size; + $bucketExpiredObjects++; + continue; + } + + $bucketBytes += $size; $bucketObjects++; } $token = isset($result['NextContinuationToken']) ? (string)$result['NextContinuationToken'] : null; @@ -2813,11 +4359,17 @@ class replication_manager $stats['bytes'] += $bucketBytes; $stats['objects'] += $bucketObjects; + $stats['expired_bytes'] += $bucketExpiredBytes; + $stats['expired_objects'] += $bucketExpiredObjects; $stats['buckets'][] = [ 'name' => $bucket, 'status' => 'ok', 'bytes' => $bucketBytes, 'objects' => $bucketObjects, + 'expired_bytes' => $bucketExpiredBytes, + 'expired_objects' => $bucketExpiredObjects, + 'retention_days' => $retentionDays, + 'retention_cutoff' => $retentionCutoff !== null ? date('c', $retentionCutoff) : null, ]; } catch (Throwable $throwable) { $stats['buckets'][] = [ @@ -2825,6 +4377,9 @@ class replication_manager 'status' => 'down', 'bytes' => 0, 'objects' => 0, + 'expired_bytes' => 0, + 'expired_objects' => 0, + 'retention_days' => $retentionDays, 'error' => $throwable->getMessage(), ]; $stats['blockers'][] = $missingPrefix . ' ' . $bucket . ' could not be inspected: ' . $throwable->getMessage(); @@ -2840,38 +4395,404 @@ class replication_manager $primaryOptions = $this->decodeOptions($primary); $targetOptions = $this->decodeOptions($target); $buckets = self::normalizeMinioBuckets($targetOptions['buckets'] ?? $primaryOptions['buckets'] ?? self::MINIO_DEFAULT_BUCKETS); + $transferLimit = self::minioReplicationTransferLimitFromOptions($targetOptions); $configDir = $this->createMinioConfigDir(); try { $this->prepareMinioAlias($configDir, 'source', $primary); $this->prepareMinioAlias($configDir, 'target', $target); - foreach ($buckets as $bucket) { - $this->runMinioClient($configDir, ['mb', '--ignore-existing', 'target/' . $bucket]); + foreach ($buckets as $index => $bucket) { + $this->runMinioClient($configDir, ['mb', '--with-lock', '--ignore-existing', 'target/' . $bucket]); $this->runMinioClient($configDir, ['version', 'enable', 'source/' . $bucket]); $this->runMinioClient($configDir, ['version', 'enable', 'target/' . $bucket]); - try { - $this->runMinioClient($configDir, [ - 'replicate', - 'add', - '--remote-bucket', - 'target/' . $bucket, - '--replicate', - 'delete,delete-marker,existing-objects', - 'source/' . $bucket, - ]); - } catch (Throwable $throwable) { - $message = strtolower($throwable->getMessage()); - if (!str_contains($message, 'already') && !str_contains($message, 'exists')) { - throw $throwable; - } + + if (self::minioBucketUsesBoundedReplicaRetention($bucket)) { + $this->configureMinioReplicaBackupRetention($target, $bucket); + // Backups are bounded on replicas; avoid bulk seeding large historical objects. + $this->pruneMinioReplicaBackupRetention($target, $bucket); } + + $this->addMinioReplicationRule($configDir, $target, $bucket, $index + 1, $transferLimit); } } finally { $this->removeDirectory($configDir); } } + private function configureMinioReplicaBackupRetention(array $target, string $bucket): void + { + $client = $this->minioS3Client($target); + $rules = []; + + try { + $current = $client->getBucketLifecycleConfiguration(['Bucket' => $bucket]); + foreach (($current['Rules'] ?? []) as $rule) { + if ((string)($rule['ID'] ?? '') !== self::MINIO_BACKUP_REPLICA_RETENTION_RULE_ID) { + $rules[] = $rule; + } + } + } catch (Throwable $throwable) { + if (!self::minioMissingLifecycleConfiguration($throwable)) { + throw $throwable; + } + } + + $rules[] = self::minioBackupReplicaRetentionLifecycleRule(); + + $client->putBucketLifecycleConfiguration([ + 'Bucket' => $bucket, + 'LifecycleConfiguration' => [ + 'Rules' => $rules, + ], + ]); + } + + private static function minioBackupReplicaRetentionLifecycleRule(): array + { + return [ + 'ID' => self::MINIO_BACKUP_REPLICA_RETENTION_RULE_ID, + 'Status' => 'Enabled', + 'Filter' => ['Prefix' => ''], + 'Expiration' => ['Days' => self::MINIO_BACKUP_REPLICA_RETENTION_DAYS], + 'NoncurrentVersionExpiration' => ['NoncurrentDays' => self::MINIO_BACKUP_REPLICA_RETENTION_DAYS], + 'AbortIncompleteMultipartUpload' => ['DaysAfterInitiation' => 7], + ]; + } + + private static function minioMissingLifecycleConfiguration(Throwable $throwable): bool + { + $message = strtolower($throwable->getMessage()); + return str_contains($message, 'nosuchlifecycleconfiguration') + || str_contains($message, 'lifecycle configuration does not exist') + || str_contains($message, 'the lifecycle configuration does not exist'); + } + + private function minioBackupReplicaRetentionConfigured(array $target, string $bucket): bool + { + try { + $current = $this->minioS3Client($target)->getBucketLifecycleConfiguration(['Bucket' => $bucket]); + } catch (Throwable) { + return false; + } + + foreach (($current['Rules'] ?? []) as $rule) { + if ((string)($rule['ID'] ?? '') !== self::MINIO_BACKUP_REPLICA_RETENTION_RULE_ID) { + continue; + } + if (strtolower((string)($rule['Status'] ?? '')) !== 'enabled') { + return false; + } + + $expirationDays = (int)($rule['Expiration']['Days'] ?? 0); + $noncurrentDays = (int)($rule['NoncurrentVersionExpiration']['NoncurrentDays'] ?? 0); + return $expirationDays > 0 + && $expirationDays <= self::MINIO_BACKUP_REPLICA_RETENTION_DAYS + && $noncurrentDays > 0 + && $noncurrentDays <= self::MINIO_BACKUP_REPLICA_RETENTION_DAYS; + } + + return false; + } + + private function pruneMinioReplicaBackupRetention(array $target, string $bucket): array + { + $client = $this->minioS3Client($target); + $cutoff = time() - (self::MINIO_BACKUP_REPLICA_RETENTION_DAYS * 86400); + $deleted = [ + 'versions' => 0, + 'delete_markers' => 0, + 'cutoff' => date('c', $cutoff), + ]; + + try { + $this->pruneMinioReplicaBackupVersions($client, $bucket, $cutoff, $deleted); + } catch (Throwable $throwable) { + if (!self::minioVersionListingUnsupported($throwable)) { + throw $throwable; + } + $this->pruneMinioReplicaBackupCurrentObjects($client, $bucket, $cutoff, $deleted); + } + + return $deleted; + } + + private static function minioVersionListingUnsupported(Throwable $throwable): bool + { + $message = strtolower($throwable->getMessage()); + return str_contains($message, 'not implemented') + || str_contains($message, 'not supported') + || str_contains($message, 'unsupported') + || str_contains($message, 'listobjectversions'); + } + + private function pruneMinioReplicaBackupVersions(S3Client $client, string $bucket, int $cutoff, array &$deleted): void + { + $keyMarker = null; + $versionIdMarker = null; + do { + $args = ['Bucket' => $bucket]; + if ($keyMarker !== null) { + $args['KeyMarker'] = $keyMarker; + } + if ($versionIdMarker !== null) { + $args['VersionIdMarker'] = $versionIdMarker; + } + + $result = $client->listObjectVersions($args); + $objects = []; + + foreach (($result['Versions'] ?? []) as $version) { + if (self::minioObjectVersionIsOlderThan($version, $cutoff)) { + $objects[] = [ + 'Key' => (string)($version['Key'] ?? ''), + 'VersionId' => (string)($version['VersionId'] ?? ''), + ]; + $deleted['versions']++; + } + } + + foreach (($result['DeleteMarkers'] ?? []) as $marker) { + if (self::minioObjectVersionIsOlderThan($marker, $cutoff)) { + $objects[] = [ + 'Key' => (string)($marker['Key'] ?? ''), + 'VersionId' => (string)($marker['VersionId'] ?? ''), + ]; + $deleted['delete_markers']++; + } + } + + $this->deleteMinioObjectsInBatches($client, $bucket, $objects); + + $keyMarker = isset($result['NextKeyMarker']) ? (string)$result['NextKeyMarker'] : null; + $versionIdMarker = isset($result['NextVersionIdMarker']) ? (string)$result['NextVersionIdMarker'] : null; + } while (!empty($result['IsTruncated'])); + } + + private function pruneMinioReplicaBackupCurrentObjects(S3Client $client, string $bucket, int $cutoff, array &$deleted): void + { + $token = null; + do { + $args = ['Bucket' => $bucket]; + if ($token !== null) { + $args['ContinuationToken'] = $token; + } + + $result = $client->listObjectsV2($args); + $objects = []; + foreach (($result['Contents'] ?? []) as $object) { + if (!self::minioObjectVersionIsOlderThan($object, $cutoff)) { + continue; + } + $objects[] = ['Key' => (string)($object['Key'] ?? '')]; + $deleted['versions']++; + } + + $this->deleteMinioObjectsInBatches($client, $bucket, $objects); + $token = isset($result['NextContinuationToken']) ? (string)$result['NextContinuationToken'] : null; + } while ($token !== null); + } + + private static function minioObjectVersionIsOlderThan(array $object, int $cutoff): bool + { + $key = trim((string)($object['Key'] ?? '')); + if ($key === '') { + return false; + } + + $lastModified = self::minioObjectLastModifiedTimestamp($object['LastModified'] ?? null); + return $lastModified !== null && $lastModified < $cutoff; + } + + private function deleteMinioObjectsInBatches(S3Client $client, string $bucket, array $objects): void + { + foreach (array_chunk($objects, 1000) as $chunk) { + $chunk = array_values(array_filter( + $chunk, + static fn(array $object): bool => trim((string)($object['Key'] ?? '')) !== '' + )); + if ($chunk === []) { + continue; + } + + $client->deleteObjects([ + 'Bucket' => $bucket, + 'Delete' => [ + 'Objects' => $chunk, + 'Quiet' => true, + ], + ]); + } + } + + private function addMinioReplicationRule(string $configDir, array $target, string $bucket, int $priority, string $transferLimit): void + { + try { + $this->runMinioClient($configDir, self::minioReplicationRuleCommand($bucket, $priority, $transferLimit)); + return; + } catch (Throwable $throwable) { + $message = strtolower($throwable->getMessage()); + if (self::minioReplicationRuleAlreadyExists($message)) { + $this->updateMinioReplicationRulesForBucket($configDir, $bucket, $transferLimit); + return; + } + if (!$this->repairMinioTargetBucketObjectLockIfEmpty($configDir, $target, $bucket, $message)) { + throw $throwable; + } + } + + try { + $this->runMinioClient($configDir, self::minioReplicationRuleCommand($bucket, $priority, $transferLimit)); + } catch (Throwable $throwable) { + $message = strtolower($throwable->getMessage()); + if (!self::minioReplicationRuleAlreadyExists($message)) { + throw $throwable; + } + $this->updateMinioReplicationRulesForBucket($configDir, $bucket, $transferLimit); + } + } + + private function updateMinioReplicationRulesForBucket(string $configDir, string $bucket, string $transferLimit): void + { + $result = $this->runMinioClient($configDir, ['replicate', 'ls', '--json', 'source/' . $bucket]); + $ruleIds = self::minioReplicationRuleIdsFromList(self::decodeMinioJsonOutput((string)$result['stdout'])); + foreach ($ruleIds as $ruleId) { + $this->runMinioClient($configDir, array_merge([ + 'replicate', + 'update', + '--id', + $ruleId, + '--replicate', + self::minioReplicationFeatures($bucket), + ], self::minioReplicationTransferLimitArgs($transferLimit), [ + 'source/' . $bucket, + ])); + } + } + + private static function minioReplicationRuleCommand(string $bucket, int $priority, string $transferLimit): array + { + return array_merge([ + 'replicate', + 'add', + '--remote-bucket', + 'target/' . $bucket, + '--replicate', + self::minioReplicationFeatures($bucket), + '--priority', + (string)$priority, + ], self::minioReplicationTransferLimitArgs($transferLimit), [ + 'source/' . $bucket, + ]); + } + + private static function minioReplicationFeatures(string $bucket): string + { + return self::minioBucketUsesBoundedReplicaRetention($bucket) + ? 'delete,delete-marker' + : 'delete,delete-marker,existing-objects'; + } + + private static function minioReplicationRuleIdsFromList(mixed $value): array + { + $ids = []; + self::collectMinioReplicationRuleIds($value, $ids); + return array_values(array_unique(array_filter($ids))); + } + + private static function collectMinioReplicationRuleIds(mixed $value, array &$ids): void + { + if (!is_array($value)) { + return; + } + + foreach ($value as $key => $entry) { + $normalizedKey = strtolower(str_replace(['_', '-'], '', (string)$key)); + if (in_array($normalizedKey, ['id', 'ruleid'], true) && is_scalar($entry)) { + $id = trim((string)$entry); + if ($id !== '') { + $ids[] = $id; + } + continue; + } + + self::collectMinioReplicationRuleIds($entry, $ids); + } + } + + private static function minioReplicationRuleAlreadyExists(string $message): bool + { + return str_contains($message, 'already') + || str_contains($message, 'replication rule exists') + || str_contains($message, 'replication configuration exists'); + } + + private function repairMinioTargetBucketObjectLockIfEmpty(string $configDir, array $target, string $bucket, string $message): bool + { + if (!self::minioObjectLockRequiredError($message)) { + return false; + } + + if ($this->minioBucketHasObjects($target, $bucket)) { + throw new RuntimeException( + 'MinIO target bucket ' . $bucket . ' was created without Object Lock and is not empty. ' + . 'Create a new empty replica bucket with Object Lock enabled, or empty and recreate this bucket before provisioning.' + ); + } + + $this->runMinioClient($configDir, ['rb', 'target/' . $bucket]); + $this->runMinioClient($configDir, ['mb', '--with-lock', 'target/' . $bucket]); + $this->runMinioClient($configDir, ['version', 'enable', 'target/' . $bucket]); + + return true; + } + + private static function minioObjectLockRequiredError(string $message): bool + { + return (str_contains($message, 'object lock') || str_contains($message, 'object locking')) + && str_contains($message, 'destination bucket'); + } + + private function minioBucketHasObjects(array $host, string $bucket): bool + { + $client = $this->minioS3Client($host); + $objects = $client->listObjectsV2([ + 'Bucket' => $bucket, + 'MaxKeys' => 1, + ]); + if (!empty($objects['Contents'])) { + return true; + } + + try { + $versions = $client->listObjectVersions([ + 'Bucket' => $bucket, + 'MaxKeys' => 1, + ]); + return !empty($versions['Versions']) || !empty($versions['DeleteMarkers']); + } catch (Throwable) { + return false; + } + } + + private function minioReplicationConfiguredForHosts(array $primary, array $target): bool + { + $primaryOptions = $this->decodeOptions($primary); + $targetOptions = $this->decodeOptions($target); + $buckets = self::normalizeMinioBuckets($targetOptions['buckets'] ?? $primaryOptions['buckets'] ?? self::MINIO_DEFAULT_BUCKETS); + + if (!$this->minioReplicationConfigured($primary, $buckets)) { + return false; + } + + if (in_array(self::MINIO_BACKUP_BUCKET, $buckets, true) + && !$this->minioBackupReplicaRetentionConfigured($target, self::MINIO_BACKUP_BUCKET)) { + return false; + } + + return true; + } + private function minioReplicationConfigured(array $primary, array $buckets): bool { $configDir = $this->createMinioConfigDir(); @@ -2879,7 +4800,7 @@ class replication_manager $this->prepareMinioAlias($configDir, 'source', $primary); foreach ($buckets as $bucket) { try { - $result = $this->runMinioClient($configDir, ['replicate', 'ls', '--json', 'source/' . $bucket]); + $result = $this->runMinioClient($configDir, ['replicate', 'list', '--json', 'source/' . $bucket]); } catch (Throwable) { return false; } @@ -2894,6 +4815,209 @@ class replication_manager } } + private function minioReplicationProgressStatus(array $primary, array $target, array $buckets): ?array + { + $targetOptions = $this->decodeOptions($target); + $transferLimit = self::minioReplicationTransferLimitFromOptions($targetOptions); + $configDir = $this->createMinioConfigDir(); + $bucketProgress = []; + $bucketOutput = []; + $requiredUnavailableBuckets = []; + + try { + $this->prepareMinioAlias($configDir, 'source', $primary); + foreach ($buckets as $bucket) { + $countsTowardCatchUp = self::minioBucketCountsTowardCatchUp((string)$bucket); + try { + // MinIO keeps removed/re-added ARNs in JSON status output. Prefer standard + // output so stale targets do not keep a healthy current target below 100%. + $result = $this->runMinioClient($configDir, array_merge([ + 'replicate', + 'status', + 'source/' . $bucket, + ], self::minioReplicationTransferLimitArgs($transferLimit))); + $stdout = (string)$result['stdout']; + $raw = $stdout; + $progress = self::minioReplicationProgressFromStatusOutput($stdout); + + if ($progress === null) { + $result = $this->runMinioClient($configDir, array_merge([ + 'replicate', + 'status', + '--json', + 'source/' . $bucket, + ], self::minioReplicationTransferLimitArgs($transferLimit))); + $stdout = (string)$result['stdout']; + $decoded = self::decodeMinioJsonOutput($stdout); + $raw = $decoded ?? $stdout; + $progress = self::minioReplicationProgressFromStatusOutput($raw) + ?? self::minioReplicationProgressFromStatusOutput($stdout); + } + + $bucketOutput[$bucket] = [ + 'ok' => true, + 'counts_toward_catch_up' => $countsTowardCatchUp, + 'raw' => $raw, + 'progress' => $progress, + ]; + if ($progress !== null) { + $bucketProgress[$bucket] = $progress; + } elseif ($countsTowardCatchUp) { + $requiredUnavailableBuckets[] = (string)$bucket; + } + } catch (Throwable $throwable) { + $bucketOutput[$bucket] = [ + 'ok' => false, + 'counts_toward_catch_up' => $countsTowardCatchUp, + 'error' => $throwable->getMessage(), + ]; + if ($countsTowardCatchUp) { + $requiredUnavailableBuckets[] = (string)$bucket; + } + } + } + } finally { + $this->removeDirectory($configDir); + } + + $progress = self::minioCatchUpProgressFromBucketStatuses($bucketProgress); + if ($progress === null) { + return null; + } + + $requiredUnavailableBuckets = array_values(array_unique($requiredUnavailableBuckets)); + if ($requiredUnavailableBuckets !== []) { + $progress['replication_percent'] = self::minioIncompleteProgress((float)$progress['replication_percent']); + $progress['blockers'] = array_values(array_unique(array_merge($progress['blockers'] ?? [], [ + 'MinIO replication status is unavailable for bucket(s): ' . implode(', ', $requiredUnavailableBuckets) . '.', + ]))); + $progress['unavailable_required_buckets'] = $requiredUnavailableBuckets; + } + + $progress['buckets'] = $bucketOutput; + $progress['target_endpoint'] = self::minioEndpoint($target); + + return $progress; + } + + public static function minioCatchUpProgressFromBucketStatuses(array $bucketProgress): ?array + { + $requiredProgress = []; + $ignoredBuckets = []; + + foreach ($bucketProgress as $bucket => $progress) { + $bucket = (string)$bucket; + if (self::minioBucketCountsTowardCatchUp($bucket)) { + $requiredProgress[$bucket] = $progress; + continue; + } + + $ignoredBuckets[] = $bucket; + } + + $progress = self::aggregateMinioReplicationProgress($requiredProgress); + if ($progress === null && $ignoredBuckets !== []) { + $progress = [ + 'replication_percent' => 100.0, + 'blockers' => [], + 'basis' => 'bounded_retention_only', + 'stats' => [ + 'completed_bytes' => 0.0, + 'pending_bytes' => 0.0, + 'failed_bytes' => 0.0, + 'total_bytes' => 0.0, + 'completed_count' => 0.0, + 'pending_count' => 0.0, + 'failed_count' => 0.0, + 'total_count' => 0.0, + ], + 'bucket_count' => 0, + ]; + } + + if ($progress === null) { + return null; + } + + $progress['ignored_buckets'] = $ignoredBuckets; + $progress['catch_up_bucket_count'] = count($requiredProgress); + + return $progress; + } + + private static function aggregateMinioReplicationProgress(array $bucketProgress): ?array + { + if ($bucketProgress === []) { + return null; + } + + $stats = [ + 'completed_bytes' => 0.0, + 'pending_bytes' => 0.0, + 'failed_bytes' => 0.0, + 'total_bytes' => 0.0, + 'completed_count' => 0.0, + 'pending_count' => 0.0, + 'failed_count' => 0.0, + 'total_count' => 0.0, + ]; + $percentages = []; + foreach ($bucketProgress as $progress) { + $percentages[] = (float)($progress['replication_percent'] ?? 0); + $progressStats = is_array($progress['stats'] ?? null) ? $progress['stats'] : []; + foreach (array_keys($stats) as $key) { + $stats[$key] += (float)($progressStats[$key] ?? 0); + } + } + + $completedBytes = $stats['completed_bytes']; + $remainingBytes = $stats['pending_bytes'] + $stats['failed_bytes']; + $totalBytes = $stats['total_bytes']; + $completedCount = $stats['completed_count']; + $remainingCount = $stats['pending_count'] + $stats['failed_count']; + $totalCount = $stats['total_count']; + $basis = 'bucket_average'; + + if ($totalBytes > 0.0) { + $percent = ($completedBytes / $totalBytes) * 100; + $basis = 'total_bytes'; + } elseif (($completedBytes + $remainingBytes) > 0.0) { + $percent = ($completedBytes / ($completedBytes + $remainingBytes)) * 100; + $basis = 'byte_balance'; + } elseif ($totalCount > 0.0) { + $percent = ($completedCount / $totalCount) * 100; + $basis = 'total_count'; + } elseif (($completedCount + $remainingCount) > 0.0) { + $percent = ($completedCount / ($completedCount + $remainingCount)) * 100; + $basis = 'count_balance'; + } else { + $percent = array_sum($percentages) / max(1, count($percentages)); + } + + $percent = round(min(100.0, max(0.0, $percent)), 2); + $withinTolerance = $stats['failed_bytes'] <= 0.0 + && $stats['failed_count'] <= 0.0 + && $stats['pending_bytes'] <= self::MINIO_CATCH_UP_PENDING_BYTES_TOLERANCE + && $stats['pending_count'] <= self::MINIO_CATCH_UP_PENDING_OBJECTS_TOLERANCE; + if ($percent < 100.0 && $withinTolerance) { + $percent = 100.0; + $basis .= '_within_live_tolerance'; + } + + return [ + 'replication_percent' => $percent, + 'blockers' => $percent < 100.0 ? [self::MINIO_REPLICA_NOT_CAUGHT_UP_BLOCKER] : [], + 'basis' => $basis, + 'stats' => $stats, + 'bucket_count' => count($bucketProgress), + 'live_tolerance' => [ + 'pending_bytes' => self::MINIO_CATCH_UP_PENDING_BYTES_TOLERANCE, + 'pending_objects' => self::MINIO_CATCH_UP_PENDING_OBJECTS_TOLERANCE, + 'within_tolerance' => $withinTolerance, + ], + ]; + } + private function minioTargetFreeBytes(array $host): ?int { $configDir = $this->createMinioConfigDir(); @@ -2924,7 +5048,13 @@ class replication_manager private function runMinioClient(string $configDir, array $arguments): array { - $command = array_merge(['mc', '--config-dir', $configDir], array_map('strval', $arguments)); + $binary = self::minioClientBinary(); + if ($binary === null) { + throw new RuntimeException('MinIO Client (mc) is not available in the PHP runtime. Install mc, set MINIO_MC_BINARY, or enable MINIO_MC_AUTO_INSTALL.'); + } + + $command = array_merge([$binary, '--config-dir', $configDir], array_map('strval', $arguments)); + $timeoutSeconds = self::minioClientCommandTimeoutSeconds(); $pipes = []; $process = @proc_open($command, [ 1 => ['pipe', 'w'], @@ -2934,11 +5064,56 @@ class replication_manager throw new RuntimeException('MinIO Client (mc) is not available.'); } - $stdout = stream_get_contents($pipes[1]); + stream_set_blocking($pipes[1], false); + stream_set_blocking($pipes[2], false); + + $stdout = ''; + $stderr = ''; + $exitCode = null; + $timedOut = false; + $deadline = microtime(true) + $timeoutSeconds; + + while (true) { + $stdout .= (string)stream_get_contents($pipes[1]); + $stderr .= (string)stream_get_contents($pipes[2]); + + $status = proc_get_status($process); + if (empty($status['running'])) { + $exitCode = (int)($status['exitcode'] ?? -1); + break; + } + + if (microtime(true) >= $deadline) { + $timedOut = true; + proc_terminate($process); + usleep(100000); + $status = proc_get_status($process); + if (!empty($status['running'])) { + proc_terminate($process, 9); + } + break; + } + + usleep(50000); + } + + $stdout .= (string)stream_get_contents($pipes[1]); fclose($pipes[1]); - $stderr = stream_get_contents($pipes[2]); + $stderr .= (string)stream_get_contents($pipes[2]); fclose($pipes[2]); - $exitCode = proc_close($process); + + if ($timedOut) { + throw new RuntimeException( + 'MinIO Client command timed out after ' . $timeoutSeconds . ' seconds: ' + . self::minioClientCommandLabel($arguments) + ); + } + + $closeCode = proc_close($process); + if ($exitCode === null || $exitCode < 0) { + $exitCode = $closeCode; + } + if ($exitCode !== 0) { $message = trim((string)$stderr) ?: trim((string)$stdout) ?: 'MinIO Client command failed.'; throw new RuntimeException($message); @@ -2951,6 +5126,296 @@ class replication_manager ]; } + private static function minioClientCommandTimeoutSeconds(): int + { + $configured = getenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS'); + if (is_numeric($configured) && (int)$configured > 0) { + return (int)$configured; + } + + return self::MINIO_MC_COMMAND_TIMEOUT_SECONDS; + } + + private static function minioClientCommandLabel(array $arguments): string + { + $parts = array_values(array_map('strval', $arguments)); + if (($parts[0] ?? '') === 'alias' && ($parts[1] ?? '') === 'set') { + if (isset($parts[4])) { + $parts[4] = '[redacted]'; + } + if (isset($parts[5])) { + $parts[5] = '[redacted]'; + } + } + + return 'mc ' . implode(' ', array_slice($parts, 0, 8)); + } + + private static function minioClientBinary(): ?string + { + $configured = trim((string)(getenv('MINIO_MC_BINARY') ?: '')); + if ($configured !== '') { + return $configured; + } + + foreach (['/usr/local/bin/mc', '/usr/bin/mc'] as $candidate) { + if (is_file($candidate) && is_executable($candidate)) { + return $candidate; + } + } + + return self::executableFromPath('mc') ?? self::cachedMinioClientBinary(); + } + + private static function cachedMinioClientBinary(): ?string + { + if (!self::minioClientAutoInstallEnabled()) { + return null; + } + + $cacheDir = trim((string)(getenv('MINIO_MC_CACHE_DIR') ?: '')); + if ($cacheDir === '') { + $cacheDir = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . 'truckwash-minio-client'; + } + $binary = rtrim($cacheDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . 'mc'; + if (is_file($binary) && is_executable($binary)) { + return $binary; + } + + if (!is_dir($cacheDir) && !mkdir($cacheDir, 0700, true) && !is_dir($cacheDir)) { + throw new RuntimeException('Could not create MinIO Client cache directory.'); + } + + $lock = @fopen($cacheDir . DIRECTORY_SEPARATOR . 'mc.lock', 'c'); + if (is_resource($lock)) { + @flock($lock, LOCK_EX); + } + + try { + if (is_file($binary) && is_executable($binary)) { + return $binary; + } + + self::downloadMinioClientBinary($binary); + self::assertMinioClientUsable($binary); + } finally { + if (is_resource($lock)) { + @flock($lock, LOCK_UN); + fclose($lock); + } + } + + return $binary; + } + + private static function minioClientAutoInstallEnabled(): bool + { + $configured = getenv('MINIO_MC_AUTO_INSTALL'); + $value = strtolower(trim((string)($configured === false ? '1' : $configured))); + return !in_array($value, ['0', 'false', 'no', 'off'], true); + } + + private static function downloadMinioClientBinary(string $binary): void + { + $url = self::minioClientDownloadUrl(); + $temp = $binary . '.download-' . getmypid(); + $output = @fopen($temp, 'wb'); + if (!is_resource($output)) { + throw new RuntimeException('Could not write MinIO Client download cache.'); + } + + $ok = false; + $error = ''; + try { + if (function_exists('curl_init')) { + $curl = curl_init($url); + if ($curl === false) { + throw new RuntimeException('Could not initialize MinIO Client download.'); + } + curl_setopt_array($curl, [ + CURLOPT_FILE => $output, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_CONNECTTIMEOUT => min(2, self::MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS), + CURLOPT_TIMEOUT => self::minioClientDownloadTimeoutSeconds(), + CURLOPT_FAILONERROR => true, + CURLOPT_USERAGENT => 'truckwash-replication-manager/1.0', + ]); + $ok = curl_exec($curl) === true; + $error = curl_error($curl); + $status = (int)curl_getinfo($curl, CURLINFO_RESPONSE_CODE); + curl_close($curl); + if (!$ok && $status > 0) { + $error = 'HTTP ' . $status; + } + } else { + $context = stream_context_create([ + 'http' => ['timeout' => self::minioClientDownloadTimeoutSeconds()], + 'https' => ['timeout' => self::minioClientDownloadTimeoutSeconds()], + ]); + $input = @fopen($url, 'rb', false, $context); + if (is_resource($input)) { + $ok = stream_copy_to_stream($input, $output) !== false; + fclose($input); + } else { + $error = 'download stream could not be opened'; + } + } + } finally { + fclose($output); + } + + if (!$ok || !is_file($temp) || (int)filesize($temp) <= 0) { + @unlink($temp); + throw new RuntimeException('Could not download MinIO Client (mc): ' . ($error !== '' ? $error : 'empty response')); + } + + @chmod($temp, 0755); + if (!@rename($temp, $binary)) { + @unlink($temp); + throw new RuntimeException('Could not install downloaded MinIO Client (mc).'); + } + @chmod($binary, 0755); + } + + private static function minioClientDownloadTimeoutSeconds(): int + { + $configured = getenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS'); + if (is_numeric($configured) && (int)$configured > 0) { + return (int)$configured; + } + + return self::MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS; + } + + private static function minioClientDownloadUrl(): string + { + $configured = trim((string)(getenv('MINIO_MC_DOWNLOAD_URL') ?: '')); + if ($configured !== '') { + return $configured; + } + + $platform = self::minioClientDownloadPlatform(); + if ($platform === null) { + throw new RuntimeException('Automatic MinIO Client download is not supported on this PHP runtime platform.'); + } + + return self::MINIO_MC_DOWNLOAD_BASE_URL . '/' . $platform . '/mc'; + } + + private static function minioClientDownloadPlatform(): ?string + { + if (PHP_OS_FAMILY !== 'Linux') { + return null; + } + + $machine = strtolower((string)php_uname('m')); + return match ($machine) { + 'x86_64', 'amd64' => 'linux-amd64', + 'aarch64', 'arm64' => 'linux-arm64', + default => null, + }; + } + + private static function assertMinioClientUsable(string $binary): void + { + $result = self::runProcessWithTimeout([$binary, '--version'], self::MINIO_MC_COMMAND_TIMEOUT_SECONDS); + if (($result['exit_code'] ?? 1) !== 0) { + @unlink($binary); + $message = trim((string)($result['stderr'] ?? '')) ?: trim((string)($result['stdout'] ?? '')) ?: 'mc --version failed'; + throw new RuntimeException('Downloaded MinIO Client (mc) failed verification: ' . $message); + } + } + + private static function runProcessWithTimeout(array $command, int $timeoutSeconds): array + { + $pipes = []; + $process = @proc_open(array_map('strval', $command), [ + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ], $pipes); + if (!is_resource($process)) { + return [ + 'stdout' => '', + 'stderr' => 'Process could not be started.', + 'exit_code' => 127, + 'timed_out' => false, + ]; + } + + stream_set_blocking($pipes[1], false); + stream_set_blocking($pipes[2], false); + + $stdout = ''; + $stderr = ''; + $exitCode = null; + $timedOut = false; + $deadline = microtime(true) + max(1, $timeoutSeconds); + + while (true) { + $stdout .= (string)stream_get_contents($pipes[1]); + $stderr .= (string)stream_get_contents($pipes[2]); + + $status = proc_get_status($process); + if (empty($status['running'])) { + $exitCode = (int)($status['exitcode'] ?? -1); + break; + } + + if (microtime(true) >= $deadline) { + $timedOut = true; + proc_terminate($process); + usleep(100000); + $status = proc_get_status($process); + if (!empty($status['running'])) { + proc_terminate($process, 9); + } + break; + } + + usleep(50000); + } + + $stdout .= (string)stream_get_contents($pipes[1]); + fclose($pipes[1]); + $stderr .= (string)stream_get_contents($pipes[2]); + fclose($pipes[2]); + + if (!$timedOut) { + $closeCode = proc_close($process); + if ($exitCode === null || $exitCode < 0) { + $exitCode = $closeCode; + } + } + + return [ + 'stdout' => $stdout, + 'stderr' => $stderr, + 'exit_code' => $timedOut ? 124 : (int)$exitCode, + 'timed_out' => $timedOut, + ]; + } + + private static function executableFromPath(string $name): ?string + { + $path = (string)(getenv('PATH') ?: ''); + if ($path === '') { + return null; + } + + foreach (explode(PATH_SEPARATOR, $path) as $dir) { + $dir = rtrim((string)$dir, DIRECTORY_SEPARATOR); + if ($dir === '') { + continue; + } + $candidate = $dir . DIRECTORY_SEPARATOR . $name; + if (is_file($candidate) && is_executable($candidate)) { + return $candidate; + } + } + + return null; + } + private function createMinioConfigDir(): string { $dir = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . 'truckwash-mc-' . bin2hex(random_bytes(6)); @@ -3140,7 +5605,22 @@ class replication_manager private function refreshStatuses(): void { foreach ($this->listHosts() as $host) { - if ($this->activeOperation((string)$host['kind'], (int)$host['id']) !== null) { + $activeOperation = $this->activeOperation((string)$host['kind'], (int)$host['id']); + if ($activeOperation !== null) { + if ($this->shouldAdvanceActiveProvisionDuringRefresh($host, $activeOperation)) { + try { + $this->provisionHost((string)$host['kind'], (int)$host['id']); + } catch (Throwable $throwable) { + $this->storeStatus($host, [ + 'status' => 'degraded', + 'replication_percent' => self::lastStatusReplicationPercent($host, 0.0), + 'lag_seconds' => null, + 'blockers' => [$throwable->getMessage()], + 'raw' => ['active_operation_refresh_failed' => true], + 'checked_at' => date('c'), + ]); + } + } continue; } @@ -3163,6 +5643,15 @@ class replication_manager ]); } } + + $this->writeBootstrapSnapshot(); + } + + private function shouldAdvanceActiveProvisionDuringRefresh(array $host, array $activeOperation): bool + { + return (string)($host['kind'] ?? '') === self::KIND_MINIO + && (string)($host['role'] ?? '') !== 'primary' + && (string)($activeOperation['operation'] ?? '') === 'provision'; } private function storeStatus(array $host, array $status): void @@ -3196,6 +5685,31 @@ class replication_manager self::jsonEncode($publicStatus['raw']), ] ); + + $this->completeReadyMinioProvisionOperation($host, $publicStatus); + + if (class_exists(coolify_manager::class)) { + coolify_manager::syncDeploymentStateForReplicationHost((int)$host['id']); + } + } + + private function completeReadyMinioProvisionOperation(array $host, array $status): void + { + if ((string)($host['kind'] ?? '') !== self::KIND_MINIO || (string)($host['role'] ?? '') === 'primary') { + return; + } + + $blockers = array_values(array_filter($status['blockers'] ?? [])); + if ((string)($status['status'] ?? '') !== 'ok' + || round((float)($status['replication_percent'] ?? 0), 2) < 100.0 + || $blockers !== []) { + return; + } + + $operationId = $this->activeOperationId(self::KIND_MINIO, (int)$host['id'], 'provision'); + if ($operationId !== null) { + $this->finishOperation($operationId, 'completed', 100.0, 'MinIO replication target is caught up.', []); + } } private function buildReplicationSummary(string $kind, array $hosts): array @@ -3259,9 +5773,17 @@ class replication_manager $lastStatus = self::jsonDecode($host['last_status_json'] ?? null); $credentials = $this->credentials($host); $options = $this->decodeOptions($host); + $hasCoolifyDeployment = isset($options['coolify_target_id']) + || isset($options['coolify_instance_id']) + || (string)($options['deployment_provider'] ?? '') === 'coolify'; + $coolifyDeployment = $hasCoolifyDeployment && isset($host['id']) + ? coolify_manager::deploymentMetadataForReplicationHost((int)$host['id']) + : null; $activeOperation = isset($host['id']) ? $this->activeOperation((string)$host['kind'], (int)$host['id']) : null; + $replicationPercent = round((float)($lastStatus['replication_percent'] ?? (($host['role'] ?? '') === 'primary' ? 100 : 0)), 2); + $status = self::publicReplicationStatus($host, $lastStatus, $activeOperation, $replicationPercent); $database = match ((string)$host['kind']) { self::KIND_DATABASE => (string)($host['database_name'] ?? ''), self::KIND_REDIS => (int)($host['database_index'] ?? 0), @@ -3280,14 +5802,18 @@ class replication_manager 'scheme' => $options['scheme'] ?? null, 'buckets' => ((string)$host['kind'] === self::KIND_MINIO) ? self::normalizeMinioBuckets($options['buckets'] ?? []) : [], 'console_port' => ((string)$host['kind'] === self::KIND_MINIO) ? (int)($options['console_port'] ?? 9001) : null, + 'replication_transfer_limit' => ((string)$host['kind'] === self::KIND_MINIO) ? self::minioReplicationTransferLimitFromOptions($options) : null, 'space_headroom_percent' => ((string)$host['kind'] === self::KIND_MINIO) ? (float)($options['space_headroom_percent'] ?? self::MINIO_SPACE_HEADROOM_PERCENT) : null, 'role' => (string)$host['role'], - 'status' => (string)$host['status'], + 'status' => $status, 'replication_source_id' => isset($host['replication_source_id']) ? (int)$host['replication_source_id'] : null, 'ssl_mode' => $host['ssl_mode'] ?? null, - 'replication_percent' => round((float)($lastStatus['replication_percent'] ?? (($host['role'] ?? '') === 'primary' ? 100 : 0)), 2), - 'last_status' => $lastStatus, + 'replication_percent' => $replicationPercent, + 'last_status' => array_replace($lastStatus, ['status' => $status]), 'active_operation' => $activeOperation, + 'deployment_provider' => (string)($options['deployment_provider'] ?? ($coolifyDeployment !== null ? 'coolify' : 'manual')), + 'coolify' => $coolifyDeployment, + 'availability_state' => $coolifyDeployment['availability_state'] ?? null, 'last_checked_at' => $host['last_checked_at'] ?? null, 'credential_summary' => [ 'username' => $credentials['username'] !== '' ? replication_secret_box::mask($credentials['username']) : '', @@ -3303,6 +5829,68 @@ class replication_manager ]; } + private static function publicReplicationStatus(array $host, array $lastStatus, ?array $activeOperation, float $replicationPercent): string + { + $hostStatus = (string)($host['status'] ?? 'unknown'); + $status = (string)($lastStatus['status'] ?? $hostStatus); + + if (in_array($hostStatus, ['removed', 'inactive', 'not_configured', 'down'], true)) { + return $hostStatus; + } + if (in_array($status, ['removed', 'inactive', 'not_configured', 'down'], true)) { + return $status; + } + if ($activeOperation !== null || $hostStatus === 'provisioning' || $status === 'provisioning') { + return 'provisioning'; + } + if ($status === 'ok') { + return self::replicationHealthStatus( + true, + (string)($host['role'] ?? ''), + $replicationPercent, + array_values(array_filter($lastStatus['blockers'] ?? [])) + ); + } + + return $status !== '' ? $status : 'unknown'; + } + + private static function sanitizePublicLastStatus(array $host, array $lastStatus): array + { + if ((string)($host['kind'] ?? '') !== self::KIND_MINIO || (string)($host['role'] ?? '') !== 'primary') { + return $lastStatus; + } + + $blockers = array_values(array_filter(array_map( + static fn(mixed $blocker): string => trim((string)$blocker), + $lastStatus['blockers'] ?? [] + ))); + if ($blockers === []) { + return $lastStatus; + } + + $onlyObjectScanTimeouts = true; + foreach ($blockers as $blocker) { + $normalized = strtolower($blocker); + if (!str_contains($normalized, 'could not be inspected') + || !str_contains($normalized, 'listobjectsv2') + || !str_contains($normalized, 'timed out')) { + $onlyObjectScanTimeouts = false; + break; + } + } + + if (!$onlyObjectScanTimeouts || round((float)($lastStatus['replication_percent'] ?? 0), 2) < 100.0) { + return $lastStatus; + } + + $lastStatus['status'] = 'ok'; + $lastStatus['blockers'] = []; + $lastStatus['raw']['suppressed_blockers'] = $blockers; + $lastStatus['raw']['suppressed_reason'] = 'MinIO primary object-scan timeouts do not indicate primary availability failure.'; + return $lastStatus; + } + private static function normalizeMinioAddress(string $host, mixed $port, mixed $scheme): array { $raw = trim($host); @@ -3392,13 +5980,30 @@ class replication_manager } $options = is_array($input['options'] ?? null) ? $input['options'] : []; + if (isset($input['deployment_provider'])) { + $provider = strtolower(trim((string)$input['deployment_provider'])); + if (!in_array($provider, ['manual', 'coolify'], true)) { + throw new RuntimeException('Deployment provider must be manual or coolify.'); + } + $options['deployment_provider'] = $provider; + } + if (isset($input['coolify_target_id'])) { + $options['coolify_target_id'] = (int)$input['coolify_target_id']; + } + if (isset($input['coolify_instance_id'])) { + $options['coolify_instance_id'] = (int)$input['coolify_instance_id']; + } if ($kind === self::KIND_MINIO) { $headroom = (float)($input['space_headroom_percent'] ?? $options['space_headroom_percent'] ?? self::MINIO_SPACE_HEADROOM_PERCENT); + $transferLimit = array_key_exists('replication_transfer_limit', $input) + ? self::normalizeMinioTransferLimit($input['replication_transfer_limit'], false) + : self::minioReplicationTransferLimitFromOptions($options); $options = array_replace($options, [ 'scheme' => $scheme ?: 'http', 'endpoint' => self::minioEndpointFromParts($scheme ?: 'http', $host, $port), 'buckets' => self::normalizeMinioBuckets($input['buckets'] ?? $options['buckets'] ?? self::MINIO_DEFAULT_BUCKETS), 'console_port' => (int)($input['console_port'] ?? $options['console_port'] ?? 9001), + 'replication_transfer_limit' => $transferLimit, 'space_headroom_percent' => max(0.0, $headroom), ]); } @@ -3539,6 +6144,7 @@ class replication_manager if ($databasePrimary !== null) { $credentials = $this->credentials($databasePrimary); $active['database'] = [ + 'id' => (int)$databasePrimary['id'], 'host' => (string)$databasePrimary['host'], 'port' => (int)$databasePrimary['port'], 'database' => (string)$databasePrimary['database_name'], @@ -3551,6 +6157,7 @@ class replication_manager if ($redisPrimary !== null) { $credentials = $this->credentials($redisPrimary); $active['redis'] = [ + 'id' => (int)$redisPrimary['id'], 'host' => (string)$redisPrimary['host'], 'port' => (int)$redisPrimary['port'], 'database' => (int)($redisPrimary['database_index'] ?? 0), @@ -3563,6 +6170,7 @@ class replication_manager $credentials = $this->credentials($minioPrimary); $options = $this->decodeOptions($minioPrimary); $active['minio'] = [ + 'id' => (int)$minioPrimary['id'], 'endpoint' => self::minioEndpoint($minioPrimary), 'access_key' => $credentials['username'], 'secret_key_secret' => $minioPrimary['password_secret'] ?? '', @@ -3574,9 +6182,77 @@ class replication_manager 'version' => 1, 'generated_at' => date('c'), 'active' => $active, + 'failover' => [ + 'config' => $this->failoverConfigForSnapshot(), + 'hosts' => $this->failoverHostsForSnapshot(), + ], ]); } + private function failoverConfigForSnapshot(): array + { + $config = replica_failover_manager::configDefaults(); + + try { + foreach ($this->selectRows("SELECT variable, value FROM module_config WHERE module = 'Failover'") as $row) { + $variable = (string)($row['variable'] ?? ''); + if (!array_key_exists($variable, $config)) { + continue; + } + $config[$variable] = $row['value'] ?? ''; + } + } catch (Throwable) { + } + + return replica_failover_manager::normalizeConfig($config); + } + + private function failoverHostsForSnapshot(): array + { + return [ + self::KIND_DATABASE => array_map( + fn(array $host): array => $this->bootstrapSnapshotHost($host), + $this->listHosts(self::KIND_DATABASE) + ), + self::KIND_REDIS => array_map( + fn(array $host): array => $this->bootstrapSnapshotHost($host), + $this->listHosts(self::KIND_REDIS) + ), + self::KIND_MINIO => array_map( + fn(array $host): array => $this->bootstrapSnapshotHost($host), + $this->listHosts(self::KIND_MINIO) + ), + ]; + } + + private function bootstrapSnapshotHost(array $host): array + { + return [ + 'id' => (int)$host['id'], + 'kind' => (string)$host['kind'], + 'label' => (string)$host['label'], + 'host' => (string)$host['host'], + 'port' => (int)$host['port'], + 'database_name' => $host['database_name'] ?? null, + 'database_index' => isset($host['database_index']) ? (int)$host['database_index'] : null, + 'username' => (string)($host['username'] ?? ''), + 'password_secret' => (string)($host['password_secret'] ?? ''), + 'admin_username' => (string)($host['admin_username'] ?? ''), + 'admin_password_secret' => (string)($host['admin_password_secret'] ?? ''), + 'replication_username' => (string)($host['replication_username'] ?? ''), + 'replication_password_secret' => (string)($host['replication_password_secret'] ?? ''), + 'role' => (string)$host['role'], + 'status' => (string)$host['status'], + 'replication_source_id' => isset($host['replication_source_id']) ? (int)$host['replication_source_id'] : null, + 'ssl_mode' => $host['ssl_mode'] ?? null, + 'options_json' => $host['options_json'] ?? null, + 'last_status_json' => $host['last_status_json'] ?? null, + 'last_checked_at' => $host['last_checked_at'] ?? null, + 'updated_at' => $host['updated_at'] ?? null, + 'deleted_at' => $host['deleted_at'] ?? null, + ]; + } + private function switchPrimary(string $kind, int $newPrimaryId, int $oldPrimaryId): void { $this->execute( diff --git a/services/nginx/app/classes/response.php b/services/nginx/app/classes/response.php index c4ecbb91..7262ebb7 100644 --- a/services/nginx/app/classes/response.php +++ b/services/nginx/app/classes/response.php @@ -14,6 +14,7 @@ class response implements response_i private array $meta = []; private array $includes = []; private users_o $users_o; + private ?array $jsonRequestBody = null; #[NoReturn] public function success(mixed $data, int $status = null): void { @@ -47,6 +48,11 @@ class response implements response_i 'data' => $this->get_data() ]); } + try { + release_manager::recordBackendFailure($success, $data, $status ?? ($success ? 200 : 400)); + } catch (\Throwable) { + // Release failure telemetry is best-effort and must not block responses. + } echo json_encode([ 'success' => $success, 'data' => $data, @@ -175,35 +181,7 @@ class response implements response_i public function getRequestParameter(string $key): mixed { - $data = []; - // Get the request data if the method is POST, PUT or PATCH - if ($_SERVER['REQUEST_METHOD'] === 'POST' || $_SERVER['REQUEST_METHOD'] === 'PUT' || $_SERVER['REQUEST_METHOD'] === 'PATCH') { - $data = json_decode(file_get_contents('php://input'), true); - } - // Get the request data if the method is GET, DELETE or OPTIONS - if ($_SERVER['REQUEST_METHOD'] === 'GET' || $_SERVER['REQUEST_METHOD'] === 'DELETE' || $_SERVER['REQUEST_METHOD'] === 'OPTIONS') { - $data = $_GET; - } - - if (!is_array($data)) { - $data = []; - } - - // If the data key is not set, try to get it from the opposite method - if (!array_key_exists($key, $data)) { - if ($_SERVER['REQUEST_METHOD'] === 'POST' || $_SERVER['REQUEST_METHOD'] === 'PUT' || $_SERVER['REQUEST_METHOD'] === 'PATCH') { - $data = $_GET; - } else { - $data = json_decode(file_get_contents('php://input'), true); - } - } - - if (!is_array($data)) { - $data = []; - } - - // Return the data - return $data[$key] ?? null; + return $this->requestParametersForMethod()[$key] ?? null; } /** @@ -212,21 +190,7 @@ class response implements response_i */ public function getAllRequestParameters(): array { - $data = []; - // Get the request data if the method is POST, PUT or PATCH - if ($_SERVER['REQUEST_METHOD'] === 'POST' || $_SERVER['REQUEST_METHOD'] === 'PUT' || $_SERVER['REQUEST_METHOD'] === 'PATCH') { - $data = json_decode(file_get_contents('php://input'), true); - } - // Get the request data if the method is GET, DELETE or OPTIONS - if ($_SERVER['REQUEST_METHOD'] === 'GET' || $_SERVER['REQUEST_METHOD'] === 'DELETE' || $_SERVER['REQUEST_METHOD'] === 'OPTIONS') { - $data = $_GET; - } - - if (!is_array($data)) { - return []; - } - - return $data; + return $this->requestParametersForMethod(); } /** @@ -237,21 +201,44 @@ class response implements response_i */ public function isRequestParameterSet(string $key): bool { - $data = []; - // Get the request data if the method is POST, PUT or PATCH - if ($_SERVER['REQUEST_METHOD'] === 'POST' || $_SERVER['REQUEST_METHOD'] === 'PUT' || $_SERVER['REQUEST_METHOD'] === 'PATCH') { - $data = json_decode(file_get_contents('php://input'), true); - } - // Get the request data if the method is GET, DELETE or OPTIONS - if ($_SERVER['REQUEST_METHOD'] === 'GET' || $_SERVER['REQUEST_METHOD'] === 'DELETE' || $_SERVER['REQUEST_METHOD'] === 'OPTIONS') { - $data = $_GET; + return array_key_exists($key, $this->requestParametersForMethod()); + } + + private function requestParametersForMethod(): array + { + $method = strtoupper((string)($_SERVER['REQUEST_METHOD'] ?? 'GET')); + + if (in_array($method, ['POST', 'PUT', 'PATCH'], true)) { + return array_replace($_GET, $this->jsonRequestBody()); } - if (!is_array($data)) { - return false; + if ($method === 'DELETE') { + return array_replace($_GET, $this->jsonRequestBody()); } - return array_key_exists($key, $data); + if ($method === 'GET' || $method === 'OPTIONS') { + return $_GET; + } + + return $this->jsonRequestBody(); + } + + private function jsonRequestBody(): array + { + if ($this->jsonRequestBody !== null) { + return $this->jsonRequestBody; + } + + $decoded = json_decode($this->rawRequestBody(), true); + $this->jsonRequestBody = is_array($decoded) ? $decoded : []; + + return $this->jsonRequestBody; + } + + protected function rawRequestBody(): string + { + $body = file_get_contents('php://input'); + return is_string($body) ? $body : ''; } public function parseFilters(?string $filters): array|null diff --git a/services/nginx/app/classes/superuser_system_status_service.php b/services/nginx/app/classes/superuser_system_status_service.php index 4e1680a3..9a20b459 100644 --- a/services/nginx/app/classes/superuser_system_status_service.php +++ b/services/nginx/app/classes/superuser_system_status_service.php @@ -842,11 +842,89 @@ class superuser_system_status_service ['key' => 'licenseplaterecognizer', 'module' => 'licenseplaterecognizer', 'enabled_variable' => 'enabled', 'required' => ['api_key'], 'probe' => fn(array $config): array => $this->probeLicensePlateRecognizerModule($config)], ['key' => 'virkdata', 'module' => 'virkdata', 'enabled_variable' => 'enabled', 'required' => ['secret_key']], ['key' => 'shelly', 'module' => 'shelly', 'enabled_variable' => 'enabled', 'required' => ['server_url', 'secret_key'], 'probe' => fn(array $config): array => $this->probeShellyModule($config)], + ['key' => 'coolify', 'module' => 'Coolify', 'enabled_variable' => 'enabled', 'required' => [], 'probe' => fn(array $config): array => $this->probeCoolifyModule($config)], + ['key' => 'releasemanager', 'module' => 'ReleaseManager', 'enabled_variable' => 'enabled', 'required' => [], 'always_enabled' => true, 'probe' => fn(array $config): array => $this->probeReleaseManagerModule($config)], ['key' => 'selfserve', 'module' => 'selfserve', 'enabled_variable' => 'enabled', 'required' => ['machine_wash_minutes_included', 'minute_product'], 'probe' => fn(array $config): array => $this->probeSelfserveModule($config)], ['key' => 'bird', 'module' => 'bird', 'enabled_variable' => 'enabled', 'required' => ['server_url', 'api_key', 'channelId', 'workplaceId'], 'probe' => fn(array $config): array => $this->probeBirdModule($config)], ]; } + protected function probeReleaseManagerModule(array $config): array + { + return (new release_manager())->healthProbe(); + } + + protected function probeCoolifyModule(array $config): array + { + $startedAt = microtime(true); + + try { + $summary = (new coolify_manager())->summary(); + $instances = is_array($summary['instances'] ?? null) ? $summary['instances'] : []; + $targets = is_array($summary['targets'] ?? null) ? $summary['targets'] : []; + + if ($instances === []) { + return [ + 'status' => 'not_configured', + 'status_reason' => 'Coolify is enabled, but no Coolify API instance is configured.', + 'status_reason_key' => 'coolify_instances_missing', + 'status_reason_params' => [], + 'checked_at' => date('c'), + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + ]; + } + + $downInstances = array_values(array_filter($instances, static fn(array $instance): bool => ($instance['status'] ?? 'unknown') === 'down')); + $blockedTargets = array_values(array_filter($targets, static function (array $target): bool { + $state = (string)($target['availability_state'] ?? 'degraded'); + return $state === 'destructive_action_required' || str_contains($state, 'blocked'); + })); + $failedTargets = array_values(array_filter($targets, static function (array $target): bool { + return in_array((string)($target['deployment_status'] ?? ''), ['reconcile_failed', 'restart_failed', 'provision_blocked'], true); + })); + + $status = 'ok'; + $reason = 'Coolify deployment state is available.'; + $reasonKey = 'coolify_available'; + if ($downInstances !== []) { + $status = 'down'; + $reason = 'One or more Coolify API instances are unreachable.'; + $reasonKey = 'coolify_instances_down'; + } elseif ($blockedTargets !== [] || $failedTargets !== []) { + $status = 'degraded'; + $reason = 'One or more Coolify targets need operator attention before availability can be protected.'; + $reasonKey = 'coolify_targets_need_attention'; + } elseif ($targets === []) { + $status = 'degraded'; + $reason = 'Coolify is connected, but no replicated infrastructure targets are managed yet.'; + $reasonKey = 'coolify_targets_missing'; + } + + return [ + 'status' => $status, + 'status_reason' => $reason, + 'status_reason_key' => $reasonKey, + 'status_reason_params' => [ + 'instances' => count($instances), + 'targets' => count($targets), + 'blocked_targets' => count($blockedTargets), + 'failed_targets' => count($failedTargets), + ], + 'checked_at' => date('c'), + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + ]; + } catch (Throwable $throwable) { + return [ + 'status' => 'down', + 'status_reason' => 'Coolify module probe failed: ' . $throwable->getMessage(), + 'status_reason_key' => 'coolify_probe_failed', + 'status_reason_params' => ['error' => $throwable->getMessage()], + 'checked_at' => date('c'), + 'latency_ms' => round((microtime(true) - $startedAt) * 1000, 2), + ]; + } + } + protected function probeEconomicModule(array $config): array { $appSecretToken = trim((string)($GLOBALS['ECONOMIC_API']['app_secret_token'] ?? '')); diff --git a/services/nginx/app/config.php b/services/nginx/app/config.php index 15cb5b82..d0c623bc 100644 --- a/services/nginx/app/config.php +++ b/services/nginx/app/config.php @@ -166,6 +166,11 @@ if (strtolower(trim((string)($_ENV['USE_ENV'] ?? getenv('USE_ENV') ?? ''))) === require_once __DIR__ . '/classes/replication_secret_box.php'; require_once __DIR__ . '/classes/replication_bootstrap_config.php'; -\classes\replication_bootstrap_config::applyToGlobals( - \classes\replication_bootstrap_config::loadSnapshot() -); +require_once __DIR__ . '/classes/replica_failover_manager.php'; +$replicationBootstrapSnapshot = \classes\replication_bootstrap_config::loadSnapshot(); +\classes\replication_bootstrap_config::applyToGlobals($replicationBootstrapSnapshot); +try { + \classes\replica_failover_manager::applyStartupFailoverFromSnapshot(); +} catch (Throwable $throwable) { + error_log('[replication-bootstrap] Startup failover skipped: ' . $throwable->getMessage()); +} diff --git a/services/nginx/app/cron/Cron.php b/services/nginx/app/cron/Cron.php index ff186dfd..5dd223e5 100644 --- a/services/nginx/app/cron/Cron.php +++ b/services/nginx/app/cron/Cron.php @@ -5,6 +5,8 @@ use classes\backup_store; use classes\economic; use classes\economic_transfer_queue; use classes\invoice_period_flag_service; +use classes\coolify_manager; +use classes\replication_manager; use classes\redis; use classes\system_search_cache; use classes\system_search_document_index; @@ -68,6 +70,24 @@ $cron_tasks = [ 'next_run' => 0, 'function' => 'syncLogsToDatabase', ], + 'ReplicaFailoverMonitorCron' => [ + 'interval' => 60, // 1 minute + 'last_run' => 0, + 'next_run' => 0, + 'function' => 'ReplicaFailoverMonitorCron', + ], + 'CoolifyAvailabilityMonitorCron' => [ + 'interval' => 60, // 1 minute + 'last_run' => 0, + 'next_run' => 0, + 'function' => 'CoolifyAvailabilityMonitorCron', + ], + 'CoolifyLoadBalancerReconcileCron' => [ + 'interval' => 60, // 1 minute + 'last_run' => 0, + 'next_run' => 0, + 'function' => 'CoolifyLoadBalancerReconcileCron', + ], 'SyncUserEconomicCustomerDiscounts' => [ 'interval' => 180, // 3 minutes 'last_run' => 0, @@ -160,6 +180,71 @@ $cron_tasks = [ ], ]; +function ReplicaFailoverMonitorCron(): void +{ + global $db; + + if (!($db instanceof \classes\db)) { + warn('ReplicaFailoverMonitorCron skipped: database connection is unavailable.'); + return; + } + + try { + $result = (new replication_manager())->runAutomaticFailoverMonitor(); + $promoted = array_filter( + $result['results'] ?? [], + static fn(array $entry): bool => ($entry['status'] ?? '') === 'promoted' + ); + echo "[" . date('Y-m-d H:i:s') . "][CRON] ReplicaFailoverMonitorCron: " + . count($promoted) . " promotions.\n"; + } catch (Throwable $throwable) { + warn('ReplicaFailoverMonitorCron failed: ' . $throwable->getMessage()); + } +} + +function CoolifyAvailabilityMonitorCron(): void +{ + global $db; + + if (!($db instanceof \classes\db)) { + warn('CoolifyAvailabilityMonitorCron skipped: database connection is unavailable.'); + return; + } + + try { + $result = (new coolify_manager())->runAvailabilityMaintenance(); + echo "[" . date('Y-m-d H:i:s') . "][CRON] CoolifyAvailabilityMonitorCron: " + . count($result['targets'] ?? []) . " targets checked.\n"; + } catch (Throwable $throwable) { + warn('CoolifyAvailabilityMonitorCron failed: ' . $throwable->getMessage()); + } +} + +function CoolifyLoadBalancerReconcileCron(): void +{ + global $db; + + if (!($db instanceof \classes\db)) { + warn('CoolifyLoadBalancerReconcileCron skipped: database connection is unavailable.'); + return; + } + + try { + $manager = new coolify_manager(); + if (!$manager->loadBalancerAutomationEnabled()) { + echo "[" . date('Y-m-d H:i:s') . "][CRON] CoolifyLoadBalancerReconcileCron: skipped.\n"; + return; + } + + $result = $manager->reconcileLoadBalancer(false); + echo "[" . date('Y-m-d H:i:s') . "][CRON] CoolifyLoadBalancerReconcileCron: " + . count($result['applied'] ?? []) . " applied, " + . count($result['skipped'] ?? []) . " skipped.\n"; + } catch (Throwable $throwable) { + warn('CoolifyLoadBalancerReconcileCron failed: ' . $throwable->getMessage()); + } +} + function WarmInvoicePeriodManualFlagsCron(): void { (new invoice_period_flag_service())->warmManualFlagsCache(); diff --git a/services/nginx/app/index.php b/services/nginx/app/index.php index 270bba2f..013a3059 100644 --- a/services/nginx/app/index.php +++ b/services/nginx/app/index.php @@ -8,6 +8,7 @@ ini_set('zlib.output_compression', false); */ const WD = __DIR__; +require_once __DIR__ . '/vendor/autoload.php'; require_once 'config.php'; /** CORS */ @@ -16,7 +17,7 @@ $allowed_origins = array_map('trim', explode(',', (string)($CORS ?? '*'))); if ($CORS === '*' || ($origin && in_array($origin, $allowed_origins))) { header("Access-Control-Allow-Origin: " . ($origin ?: '*')); header("Access-Control-Allow-Credentials: true"); - header("Access-Control-Allow-Headers: Content-Type, Authorization, X-Customer-Number, *"); + header("Access-Control-Allow-Headers: Content-Type, Authorization, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version, *"); header("Access-Control-Allow-Methods: GET, POST, PUT, PATCH, DELETE, OPTIONS"); } @@ -26,7 +27,7 @@ if ($_SERVER['REQUEST_METHOD'] === 'OPTIONS') { header("Access-Control-Allow-Origin: " . ($origin ?: '*')); header("Access-Control-Allow-Credentials: true"); header('Access-Control-Allow-Methods: GET, POST, PUT, PATCH, DELETE, OPTIONS'); - header('Access-Control-Allow-Headers: *'); + header('Access-Control-Allow-Headers: Content-Type, Authorization, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version, *'); header('Content-Type: application/json'); http_response_code(200); exit; @@ -169,6 +170,8 @@ spl_autoload_register(function (string $class): void { use classes\application_write_freeze; use classes\db; +use classes\replication_manager; +use classes\release_manager; use classes\redis; use classes\request; use classes\response; @@ -196,6 +199,24 @@ try { $response->error($e->getMessage(), 500); } +try { + release_manager::initializeRequestContext(); +} catch (Throwable $e) { + error_log('[release-manager] Could not initialize request context: ' . $e->getMessage()); +} + +try { + $replicationBootstrapSnapshotForRequest = replication_bootstrap_config::loadSnapshot(); + $pendingStartupFailovers = is_array($replicationBootstrapSnapshotForRequest['pending_failovers'] ?? null) + ? $replicationBootstrapSnapshotForRequest['pending_failovers'] + : []; + if ($pendingStartupFailovers !== []) { + (new replication_manager())->syncStartupFailoversFromSnapshot(); + } +} catch (Throwable $e) { + error_log('[replication-bootstrap] Could not sync startup failover metadata: ' . $e->getMessage()); +} + if (application_write_freeze::shouldBlock( $_SERVER['REQUEST_METHOD'] ?? 'GET', $_SERVER['REQUEST_URI'] ?? '/', diff --git a/services/nginx/app/modules/coolify/config/coolify_enabled_c.php b/services/nginx/app/modules/coolify/config/coolify_enabled_c.php new file mode 100644 index 00000000..859c61ee --- /dev/null +++ b/services/nginx/app/modules/coolify/config/coolify_enabled_c.php @@ -0,0 +1,25 @@ +setupConfigVariable( + 'Coolify', + 'enabled', + 'bool', + false, + null, + 'Enable Coolify-managed replicated infrastructure targets.', + 'true', + false, + 'false' + ); + } +} diff --git a/services/nginx/app/modules/coolify/config/coolify_hetzner_cloud_api_token_c.php b/services/nginx/app/modules/coolify/config/coolify_hetzner_cloud_api_token_c.php new file mode 100644 index 00000000..bb502c55 --- /dev/null +++ b/services/nginx/app/modules/coolify/config/coolify_hetzner_cloud_api_token_c.php @@ -0,0 +1,38 @@ +setupConfigVariable( + 'Coolify', + 'hetzner_cloud_api_token', + 'string', + false, + null, + 'Hetzner Cloud API token with Load Balancer read/write permissions.', + 'pat_...', + true, + '' + ); + } + + public function setVariableValue(mixed $value): void + { + $value = trim((string)($value ?? '')); + if ($value !== '' && !str_starts_with($value, 'twsec:v1:')) { + $value = replication_secret_box::encrypt($value); + } + + $this->traitSetVariableValue($value); + } +} diff --git a/services/nginx/app/modules/coolify/config/coolify_hetzner_load_balancer_id_c.php b/services/nginx/app/modules/coolify/config/coolify_hetzner_load_balancer_id_c.php new file mode 100644 index 00000000..7252a38e --- /dev/null +++ b/services/nginx/app/modules/coolify/config/coolify_hetzner_load_balancer_id_c.php @@ -0,0 +1,25 @@ +setupConfigVariable( + 'Coolify', + 'hetzner_load_balancer_id', + 'string', + false, + null, + 'Hetzner Cloud Load Balancer ID used as the public Coolify gateway.', + '1234567', + false, + '' + ); + } +} diff --git a/services/nginx/app/modules/coolify/config/coolify_lb_automation_enabled_c.php b/services/nginx/app/modules/coolify/config/coolify_lb_automation_enabled_c.php new file mode 100644 index 00000000..ae273265 --- /dev/null +++ b/services/nginx/app/modules/coolify/config/coolify_lb_automation_enabled_c.php @@ -0,0 +1,25 @@ +setupConfigVariable( + 'Coolify', + 'lb_automation_enabled', + 'bool', + false, + null, + 'Enable automated Hetzner Load Balancer reconciliation for the Coolify public gateway.', + 'false', + false, + 'false' + ); + } +} diff --git a/services/nginx/app/modules/coolify/config/coolify_lb_automation_mode_c.php b/services/nginx/app/modules/coolify/config/coolify_lb_automation_mode_c.php new file mode 100644 index 00000000..b3183947 --- /dev/null +++ b/services/nginx/app/modules/coolify/config/coolify_lb_automation_mode_c.php @@ -0,0 +1,25 @@ +setupConfigVariable( + 'Coolify', + 'lb_automation_mode', + 'string', + false, + ['report_only', 'enforce'], + 'Controls whether Hetzner Load Balancer reconciliation reports drift only or applies changes.', + 'report_only', + false, + 'report_only' + ); + } +} diff --git a/services/nginx/app/modules/coolify/config/coolify_public_gateway_host_c.php b/services/nginx/app/modules/coolify/config/coolify_public_gateway_host_c.php new file mode 100644 index 00000000..e62cefd7 --- /dev/null +++ b/services/nginx/app/modules/coolify/config/coolify_public_gateway_host_c.php @@ -0,0 +1,25 @@ +setupConfigVariable( + 'Coolify', + 'public_gateway_host', + 'string', + false, + null, + 'Public DNS hostname served by the replicated Coolify gateway Load Balancer.', + 'api-v2.truckwash.io', + false, + 'api-v2.truckwash.io' + ); + } +} diff --git a/services/nginx/app/modules/coolify/coolify_c.php b/services/nginx/app/modules/coolify/coolify_c.php new file mode 100644 index 00000000..68df3b39 --- /dev/null +++ b/services/nginx/app/modules/coolify/coolify_c.php @@ -0,0 +1,63 @@ +setupConfig('Coolify'); + $this->allowUpdate([ + coolify_enabled_c::class, + coolify_lb_automation_enabled_c::class, + coolify_lb_automation_mode_c::class, + coolify_hetzner_load_balancer_id_c::class, + coolify_hetzner_cloud_api_token_c::class, + coolify_public_gateway_host_c::class, + ]); + $this->enabled = new coolify_enabled_c(); + $this->lb_automation_enabled = new coolify_lb_automation_enabled_c(); + $this->lb_automation_mode = new coolify_lb_automation_mode_c(); + $this->hetzner_load_balancer_id = new coolify_hetzner_load_balancer_id_c(); + $this->hetzner_cloud_api_token = new coolify_hetzner_cloud_api_token_c(); + $this->public_gateway_host = new coolify_public_gateway_host_c(); + } + + public function getConfigRequest(): array + { + return array_map(static function (array $row): array { + if (($row['variable'] ?? '') === 'hetzner_cloud_api_token') { + $secretSet = trim((string)($row['value'] ?? '')) !== ''; + $row['value'] = $secretSet ? '[redacted]' : ''; + $row['secret_set'] = $secretSet; + } + return $row; + }, $this->traitGetConfigRequest()); + } +} diff --git a/services/nginx/app/modules/edgegateway/classes/edge_gateway_manager.php b/services/nginx/app/modules/edgegateway/classes/edge_gateway_manager.php index 278670e3..3456fe23 100644 --- a/services/nginx/app/modules/edgegateway/classes/edge_gateway_manager.php +++ b/services/nginx/app/modules/edgegateway/classes/edge_gateway_manager.php @@ -364,6 +364,7 @@ class edge_gateway_manager } $gatewayPayload = $this->getGateway($gatewayId); + $gatewayPayload['broker_url'] = $this->buildBrokerPublicUrl(); edge_gateway_view_cache::syncGateway($gatewayPayload); return $gatewayPayload; diff --git a/services/nginx/app/modules/edgegateway/config/edgegateway_public_broker_url_c.php b/services/nginx/app/modules/edgegateway/config/edgegateway_public_broker_url_c.php index 6e2f93d3..51bc1195 100644 --- a/services/nginx/app/modules/edgegateway/config/edgegateway_public_broker_url_c.php +++ b/services/nginx/app/modules/edgegateway/config/edgegateway_public_broker_url_c.php @@ -21,9 +21,9 @@ class edgegateway_public_broker_url_c false, null, 'The browser-routable edge broker URL, including the proxy path prefix.', - 'https://api.truckwash.io:4433/edge-broker', + 'https://api-v2.truckwash.io/edge-broker', false, - trim((string)(getenv('EDGE_PUBLIC_BROKER_URL') ?: '')) + trim((string)(getenv('EDGE_PUBLIC_BROKER_URL') ?: '')) ?: 'https://api-v2.truckwash.io/edge-broker' ); } } diff --git a/services/nginx/app/modules/failover/config/failover_database_enabled_c.php b/services/nginx/app/modules/failover/config/failover_database_enabled_c.php new file mode 100644 index 00000000..d122d663 --- /dev/null +++ b/services/nginx/app/modules/failover/config/failover_database_enabled_c.php @@ -0,0 +1,29 @@ +setupConfig('Failover'); + $this->allowUpdate([ + failover_enabled_c::class, + failover_database_enabled_c::class, + failover_redis_enabled_c::class, + failover_minio_enabled_c::class, + failover_max_status_age_seconds_c::class, + ]); + + $this->enabled = new failover_enabled_c(); + $this->database_enabled = new failover_database_enabled_c(); + $this->redis_enabled = new failover_redis_enabled_c(); + $this->minio_enabled = new failover_minio_enabled_c(); + $this->max_status_age_seconds = new failover_max_status_age_seconds_c(); + } +} diff --git a/services/nginx/app/modules/washcertificates/index.php b/services/nginx/app/modules/washcertificates/index.php index ea0f02fe..6f58b619 100644 --- a/services/nginx/app/modules/washcertificates/index.php +++ b/services/nginx/app/modules/washcertificates/index.php @@ -27,7 +27,7 @@ require_once 'twc_spreadsheet_class.php'; // Set CORS headers header("Access-Control-Allow-Origin: *"); header("Access-Control-Allow-Methods: GET, POST"); -header("Access-Control-Allow-Headers: Content-Type, X-Customer-Number"); +header("Access-Control-Allow-Headers: Content-Type, Authorization, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"); // Set the timezone date_default_timezone_set('Europe/Copenhagen'); diff --git a/services/nginx/app/objects/order_bookings_o.php b/services/nginx/app/objects/order_bookings_o.php index 729378e2..93b3286a 100644 --- a/services/nginx/app/objects/order_bookings_o.php +++ b/services/nginx/app/objects/order_bookings_o.php @@ -364,11 +364,11 @@ class order_bookings_o extends db $this->createOrderItemsBy($user_id); } - if (!$this->containsWashCertificateItem()) { + $order = $this->getOrder(); + if (!$this->containsWashCertificateItem() && !$order->containsWashCertificateItem()) { return; } - $order = $this->getOrder(); $normalizedSafetySeal = orders_o::normalizeSafetySealValue($safety_seal); if ($normalizedSafetySeal !== null) { $order->setSafetySealValue($normalizedSafetySeal); diff --git a/services/nginx/app/openapi.yaml b/services/nginx/app/openapi.yaml index 9feb084a..350fcb82 100644 --- a/services/nginx/app/openapi.yaml +++ b/services/nginx/app/openapi.yaml @@ -72,6 +72,8 @@ tags: description: Form submissions and management - name: Worker description: System worker status and maintenance + - name: Error Reports + description: Authenticated application error reporting - name: Plate Scans description: License plate scanning operations - name: Config @@ -90,6 +92,160 @@ tags: description: Voice Calls via Bird paths: + /error-reports: + post: + tags: + - Error Reports + summary: Submit an authenticated user error report + operationId: submitErrorReport + security: + - BearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportSubmissionRequest' + responses: + '201': + description: Error report submitted + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportResponse' + '400': + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Authentication required + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /superuser/error-reports: + get: + tags: + - Error Reports + summary: List error reports for superusers + operationId: listSuperuserErrorReports + security: + - BearerAuth: [] + parameters: + - name: status + in: query + required: false + schema: + type: string + enum: [open, resolved, all] + default: open + - name: q + in: query + required: false + schema: + type: string + - name: limit + in: query + required: false + schema: + type: integer + minimum: 1 + maximum: 200 + default: 50 + - name: offset + in: query + required: false + schema: + type: integer + minimum: 0 + default: 0 + responses: + '200': + description: Error reports retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportListResponse' + '403': + description: Missing superuser error report permission + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /superuser/error-reports/{id}: + get: + tags: + - Error Reports + summary: Get an error report detail + operationId: getSuperuserErrorReport + security: + - BearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: integer + minimum: 1 + responses: + '200': + description: Error report retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportResponse' + '404': + description: Error report not found + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /superuser/error-reports/{id}/status: + patch: + tags: + - Error Reports + summary: Mark an error report open or resolved + operationId: updateSuperuserErrorReportStatus + security: + - BearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: integer + minimum: 1 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportStatusUpdateRequest' + responses: + '200': + description: Error report status updated + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorReportResponse' + '400': + description: Validation error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Missing superuser error report resolve permission + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + # Bird Voice Calls /bird/voice/calls: post: @@ -10782,6 +10938,31 @@ paths: '403': { $ref: '#/components/responses/Forbidden' } /superuser/replication/{kind}/{id}: + patch: + tags: + - Superuser + summary: Rename a replication host + operationId: renameSuperuserReplicationHost + parameters: + - $ref: '#/components/parameters/SuperuserReplicationKindParam' + - $ref: '#/components/parameters/SuperuserReplicationHostIdParam' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserReplicationHostRenameRequest' + responses: + '200': + description: Replication host renamed + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserReplicationHostResponse' + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } delete: tags: - Superuser @@ -10801,6 +10982,345 @@ paths: '401': { $ref: '#/components/responses/Unauthorized' } '403': { $ref: '#/components/responses/Forbidden' } + /superuser/coolify: + get: + tags: + - Superuser + summary: Coolify-managed replicated infrastructure state + operationId: getSuperuserCoolify + responses: + '200': + description: Coolify summary returned successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyResponse' + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/load-balancer: + get: + tags: + - Superuser + summary: Coolify public gateway Load Balancer state + operationId: getSuperuserCoolifyLoadBalancer + responses: + '200': + description: Load Balancer summary returned successfully + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerResponse' + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/load-balancer/reconcile: + post: + tags: + - Superuser + summary: Reconcile Coolify public gateway Load Balancer state + operationId: reconcileSuperuserCoolifyLoadBalancer + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + dry_run: + type: boolean + default: true + enforce: + type: boolean + default: false + responses: + '200': + description: Load Balancer reconcile result returned + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerReconcileResponse' + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/gateways: + get: + tags: + - Superuser + summary: List Coolify public gateway Load Balancer targets + operationId: listSuperuserCoolifyGateways + responses: + '200': + description: Gateway targets returned + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyGatewaysResponse' + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + post: + tags: + - Superuser + summary: Create or update a Coolify public gateway Load Balancer target + operationId: saveSuperuserCoolifyGateway + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyGatewaySaveRequest' + responses: + '201': + description: Gateway target saved + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyGatewayResponse' + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/gateways/{id}/test: + post: + tags: + - Superuser + summary: Probe a Coolify public gateway target + operationId: testSuperuserCoolifyGateway + parameters: + - in: path + name: id + required: true + schema: + type: integer + responses: + '200': + description: Gateway target probe result returned + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyOperationResponse' + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/instances: + post: + tags: + - Superuser + summary: Create Coolify API connection + operationId: createSuperuserCoolifyInstance + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyInstanceCreateRequest' + responses: + '201': + description: Coolify instance created + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyInstanceResponse' + '400': { $ref: '#/components/responses/BadRequest' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/instances/{id}/test: + post: + tags: + - Superuser + summary: Test Coolify API connection + operationId: testSuperuserCoolifyInstance + parameters: + - in: path + name: id + required: true + schema: + type: integer + responses: + '200': + description: Coolify connection test returned + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyOperationResponse' + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /superuser/coolify/instances/{id}/placement: + get: + tags: + - Superuser + summary: Discover Coolify placement options + operationId: discoverSuperuserCoolifyInstancePlacement + parameters: + - in: path + name: id + required: true + schema: + type: integer + responses: + '200': + description: Coolify project, environment, and server options returned + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyPlacementResponse' + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /superuser/coolify/targets: + get: + tags: + - Superuser + summary: List Coolify-managed replication targets + operationId: listSuperuserCoolifyTargets + parameters: + - in: query + name: kind + required: false + schema: + type: string + enum: [database, redis, minio] + responses: + '200': + description: Coolify targets returned + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyTargetsResponse' + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + post: + tags: + - Superuser + summary: Create Coolify-managed passive replication target + operationId: createSuperuserCoolifyTarget + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyTargetCreateRequest' + responses: + '201': + description: Coolify target created + content: + application/json: + schema: + $ref: '#/components/schemas/SuperuserCoolifyOperationResponse' + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/targets/{id}/reconcile: + post: + tags: + - Superuser + summary: Reconcile a passive Coolify target + operationId: reconcileSuperuserCoolifyTarget + parameters: + - in: path + name: id + required: true + schema: + type: integer + responses: + '200': { description: Reconcile completed or queued } + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/targets/{id}/deploy: + post: + tags: + - Superuser + summary: Deploy and provision a passive Coolify target + operationId: deploySuperuserCoolifyTarget + parameters: + - in: path + name: id + required: true + schema: + type: integer + responses: + '200': { description: Deploy and provision flow completed or queued } + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/targets/{id}/restart: + post: + tags: + - Superuser + summary: Restart a passive Coolify target + operationId: restartSuperuserCoolifyTarget + parameters: + - in: path + name: id + required: true + schema: + type: integer + responses: + '200': { description: Restart requested } + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/targets/{id}/failover: + post: + tags: + - Superuser + summary: Promote a Coolify-managed replica through replication failover + operationId: failoverSuperuserCoolifyTarget + parameters: + - in: path + name: id + required: true + schema: + type: integer + responses: + '200': { description: Failover action returned } + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + + /superuser/coolify/targets/{id}: + delete: + tags: + - Superuser + summary: Delete a Coolify target mapping with destructive confirmation + operationId: deleteSuperuserCoolifyTarget + parameters: + - in: path + name: id + required: true + schema: + type: integer + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [confirm] + properties: + confirm: + type: string + description: Must equal delete-coolify-target-{id}. + delete_resource: + type: boolean + default: false + responses: + '200': { description: Target deleted } + '409': { $ref: '#/components/responses/Conflict' } + '401': { $ref: '#/components/responses/Unauthorized' } + '403': { $ref: '#/components/responses/Forbidden' } + # Configuration Endpoints /economic/config: get: @@ -12374,6 +12894,182 @@ components: type: integer description: HTTP status code + ErrorReportSubmissionRequest: + type: object + required: + - before_error + - expected + - actual + - data_collection_accepted + - screenshot + properties: + before_error: + type: string + maxLength: 4000 + description: What the user was doing before the error occurred + expected: + type: string + maxLength: 4000 + description: What the user expected would happen + actual: + type: string + maxLength: 4000 + description: What actually happened + data_collection_accepted: + type: boolean + description: Required acceptance of collecting screenshot and diagnostic error data + screenshot: + type: string + description: PNG, JPEG, or WebP data URI of the current app viewport + route_path: + type: string + nullable: true + page_url: + type: string + nullable: true + release_trace_id: + type: string + nullable: true + request_errors: + type: array + items: + type: object + additionalProperties: true + vue_errors: + type: array + items: + type: object + additionalProperties: true + context: + type: object + additionalProperties: true + + ErrorReportStatusUpdateRequest: + type: object + required: + - status + properties: + status: + type: string + enum: [open, resolved] + resolution_note: + type: string + nullable: true + maxLength: 2000 + + ErrorReportResponse: + type: object + properties: + success: + type: boolean + data: + $ref: '#/components/schemas/ErrorReport' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + ErrorReportListResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + items: + type: array + items: + $ref: '#/components/schemas/ErrorReport' + counts: + type: object + properties: + open: + type: integer + resolved: + type: integer + all: + type: integer + limit: + type: integer + offset: + type: integer + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + ErrorReport: + type: object + properties: + id: + type: integer + status: + type: string + enum: [open, resolved] + reporter: + type: object + additionalProperties: true + route_path: + type: string + nullable: true + page_url: + type: string + nullable: true + release_trace_id: + type: string + nullable: true + frontend_version: + type: string + nullable: true + api_version: + type: string + nullable: true + screenshot: + type: object + additionalProperties: true + answers: + type: object + properties: + before_error: + type: string + expected: + type: string + actual: + type: string + request_error_count: + type: integer + vue_error_count: + type: integer + request_errors: + type: array + items: + type: object + additionalProperties: true + vue_errors: + type: array + items: + type: object + additionalProperties: true + runtime_context: + type: object + additionalProperties: true + resolved_at: + type: string + nullable: true + resolved_by_user_id: + type: integer + nullable: true + created_at: + type: string + updated_at: + type: string + nullable: true + SuperuserSystemStatusResponse: type: object properties: @@ -12436,6 +13132,552 @@ components: type: object additionalProperties: true + SuperuserCoolifyResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + generated_at: + type: string + format: date-time + instances: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyInstance' + targets: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyTarget' + availability: + type: object + additionalProperties: true + load_balancer: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancer' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyInstanceResponse: + type: object + properties: + success: + type: boolean + data: + $ref: '#/components/schemas/SuperuserCoolifyInstance' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyTargetsResponse: + type: object + properties: + success: + type: boolean + data: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyTarget' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyOperationResponse: + type: object + properties: + success: + type: boolean + data: + type: object + additionalProperties: true + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyLoadBalancerResponse: + type: object + properties: + success: + type: boolean + data: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancer' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyLoadBalancerReconcileResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + ok: + type: boolean + dry_run: + type: boolean + mutated: + type: boolean + config: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerConfig' + load_balancer: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerPublicState' + drift: + type: object + additionalProperties: true + planned: + type: array + items: + type: object + additionalProperties: true + applied: + type: array + items: + type: object + additionalProperties: true + skipped: + type: array + items: + type: object + additionalProperties: true + errors: + type: array + items: + type: object + additionalProperties: true + gateways: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyGateway' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyGatewaysResponse: + type: object + properties: + success: + type: boolean + data: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyGateway' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyGatewayResponse: + type: object + properties: + success: + type: boolean + data: + $ref: '#/components/schemas/SuperuserCoolifyGateway' + meta: + type: object + additionalProperties: true + includes: + type: object + additionalProperties: true + + SuperuserCoolifyLoadBalancer: + type: object + properties: + configured: + type: boolean + status: + type: string + enum: [not_configured, ok, degraded, down] + config: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerConfig' + gateways: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyGateway' + load_balancer: + nullable: true + allOf: + - $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerPublicState' + drift: + type: object + additionalProperties: true + last_error: + type: string + nullable: true + + SuperuserCoolifyLoadBalancerConfig: + type: object + properties: + automation_enabled: + type: boolean + automation_mode: + type: string + enum: [report_only, enforce] + load_balancer_id: + type: string + public_gateway_host: + type: string + token_set: + type: boolean + token_source: + type: string + nullable: true + required_services: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerService' + + SuperuserCoolifyLoadBalancerPublicState: + type: object + properties: + id: + type: integer + nullable: true + name: + type: string + ipv4: + type: string + nullable: true + ipv6: + type: string + nullable: true + location: + type: string + nullable: true + algorithm: + type: string + nullable: true + targets: + type: array + items: + type: string + services: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyLoadBalancerService' + + SuperuserCoolifyLoadBalancerService: + type: object + properties: + protocol: + type: string + enum: [http, tcp] + listen_port: + type: integer + destination_port: + type: integer + proxyprotocol: + type: boolean + + SuperuserCoolifyGateway: + type: object + properties: + id: + type: integer + instance_id: + type: integer + nullable: true + hostname: + type: string + target_ip: + type: string + enabled: + type: boolean + priority: + type: integer + health_state: + type: string + lb_state: + type: string + last_probe: + type: object + nullable: true + additionalProperties: true + last_probed_at: + type: string + nullable: true + last_reconciled_at: + type: string + nullable: true + created_at: + type: string + nullable: true + updated_at: + type: string + nullable: true + + SuperuserCoolifyGatewaySaveRequest: + type: object + required: + - hostname + - target_ip + properties: + id: + type: integer + instance_id: + type: integer + nullable: true + hostname: + type: string + target_ip: + type: string + enabled: + type: boolean + default: true + priority: + type: integer + default: 100 + + SuperuserCoolifyInstance: + type: object + properties: + id: + type: integer + label: + type: string + base_url: + type: string + api_token_set: + type: boolean + default_project_uuid: + type: string + nullable: true + default_environment_uuid: + type: string + nullable: true + default_environment_name: + type: string + nullable: true + default_server_uuid: + type: string + nullable: true + default_destination_uuid: + type: string + nullable: true + status: + type: string + last_checked_at: + type: string + nullable: true + last_error: + type: string + nullable: true + + SuperuserCoolifyPlacementResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + generated_at: + type: string + instance: + $ref: '#/components/schemas/SuperuserCoolifyInstance' + servers: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyPlacementServer' + projects: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyPlacementProject' + environments: + type: array + items: + $ref: '#/components/schemas/SuperuserCoolifyPlacementEnvironment' + destination_discovery_supported: + type: boolean + errors: + type: object + additionalProperties: true + + SuperuserCoolifyPlacementServer: + type: object + properties: + id: + type: integer + nullable: true + uuid: + type: string + name: + type: string + description: + type: string + ip: + type: string + user: + type: string + port: + type: integer + nullable: true + proxy_type: + type: string + swarm_cluster: + type: string + is_reachable: + type: boolean + nullable: true + is_usable: + type: boolean + nullable: true + + SuperuserCoolifyPlacementProject: + type: object + properties: + id: + type: integer + nullable: true + uuid: + type: string + name: + type: string + description: + type: string + + SuperuserCoolifyPlacementEnvironment: + type: object + properties: + id: + type: integer + nullable: true + uuid: + type: string + name: + type: string + description: + type: string + project_id: + type: integer + nullable: true + project_uuid: + type: string + project_name: + type: string + + SuperuserCoolifyTarget: + type: object + properties: + id: + type: integer + instance_id: + type: integer + instance_label: + type: string + kind: + type: string + enum: [database, redis, minio] + label: + type: string + role: + type: string + enum: [replica] + server_uuid: + type: string + nullable: true + project_uuid: + type: string + nullable: true + environment_uuid: + type: string + nullable: true + environment_name: + type: string + nullable: true + destination_uuid: + type: string + nullable: true + resource_uuid: + type: string + nullable: true + resource_type: + type: string + resource_name: + type: string + nullable: true + deployment_status: + type: string + availability_state: + type: string + enum: [protected, degraded, failover_ready, failover_blocked, failing_over, destructive_action_required] + last_reconcile_status: + type: string + nullable: true + replication: + type: object + additionalProperties: true + + SuperuserCoolifyInstanceCreateRequest: + type: object + required: + - label + - base_url + - api_token + properties: + label: + type: string + base_url: + type: string + api_token: + type: string + format: password + + SuperuserCoolifyTargetCreateRequest: + allOf: + - $ref: '#/components/schemas/SuperuserReplicationHostCreateRequest' + - type: object + required: + - kind + - host + properties: + instance_id: + type: integer + kind: + type: string + enum: [database, redis, minio] + role: + type: string + enum: [replica] + default: replica + server_uuid: + type: string + project_uuid: + type: string + environment_uuid: + type: string + environment_name: + type: string + destination_uuid: + type: string + deploy: + type: boolean + default: false + SuperuserReplicationComposeTemplateResponse: type: object properties: @@ -12508,6 +13750,9 @@ components: type: string console_port: type: integer + replication_transfer_limit: + type: string + nullable: true space_headroom_percent: type: number format: float @@ -12618,6 +13863,10 @@ components: console_port: type: integer nullable: true + replication_transfer_limit: + type: string + nullable: true + description: MinIO replication and seed bandwidth cap passed to mc --limit-upload/--limit-download, for example 25Mi. Use 0 to disable. space_headroom_percent: type: number format: float @@ -12639,6 +13888,17 @@ components: credential_summary: type: object additionalProperties: true + deployment_provider: + type: string + enum: [manual, coolify] + coolify: + type: object + nullable: true + additionalProperties: true + availability_state: + type: string + nullable: true + enum: [protected, degraded, failover_ready, failover_blocked, failing_over, destructive_action_required] SuperuserReplicationHostCreateRequest: type: object @@ -12679,6 +13939,9 @@ components: console_port: type: integer description: Optional MinIO console port for UI display. + replication_transfer_limit: + type: string + description: Optional MinIO replication and seed bandwidth cap. Defaults to 25Mi. Use 0 to disable. space_headroom_percent: type: number format: float @@ -12695,6 +13958,9 @@ components: format: password ssl_mode: type: string + deployment_provider: + type: string + enum: [manual, coolify] options: type: object properties: @@ -12710,11 +13976,23 @@ components: type: string console_port: type: integer + replication_transfer_limit: + type: string space_headroom_percent: type: number format: float additionalProperties: true + SuperuserReplicationHostRenameRequest: + type: object + required: + - label + properties: + label: + type: string + minLength: 1 + maxLength: 128 + SuperuserReplicationUnsavedCredentialTestRequest: allOf: - $ref: '#/components/schemas/SuperuserReplicationHostCreateRequest' @@ -12800,6 +14078,9 @@ components: minimum: 1 maximum: 65535 description: MinIO console port exposed by the generated compose service. + replication_transfer_limit: + type: string + description: MinIO replication and seed bandwidth cap included in generated credentials. Defaults to 25Mi. Use 0 to disable. SuperuserSystemStatusPayload: type: object diff --git a/services/nginx/app/resources/edge-gateway-agent/agent.php b/services/nginx/app/resources/edge-gateway-agent/agent.php index 149798c9..136c613a 100644 --- a/services/nginx/app/resources/edge-gateway-agent/agent.php +++ b/services/nginx/app/resources/edge-gateway-agent/agent.php @@ -1018,6 +1018,7 @@ final class TruckwashEdgeAgent private int $lastHeartbeatAt = 0; private int $lastMachineSignalPollAt = 0; private int $lastMachineSignalMonitorRefreshAt = 0; + private ?array $lastControlPlaneResponse = null; private string $agentInstanceId; public function __construct(string $configPath) @@ -1283,6 +1284,7 @@ final class TruckwashEdgeAgent } $heartbeatSucceededAt = date('c'); + $this->applyBrokerUrlFromControlPlaneResponse($this->lastControlPlaneResponse); $this->lastHeartbeatAt = time(); $this->recordSuccessfulSync($heartbeatSucceededAt, [ 'last_heartbeat_success_at' => $heartbeatSucceededAt, @@ -2430,13 +2432,15 @@ final class TruckwashEdgeAgent private function sendControlPlaneEvent(string $endpoint, array $payload, string $type): bool { + $this->lastControlPlaneResponse = null; $brokerDispatch = $this->dispatchBrokerControlPlaneEvent($endpoint, $payload); if ($brokerDispatch === true && !$this->shouldPersistControlPlaneEventOverHttp($endpoint)) { return true; } try { - $this->http->post($endpoint, $payload, 20); + $response = $this->http->post($endpoint, $payload, 20); + $this->lastControlPlaneResponse = is_array($response) ? $response : null; $this->recordSuccessfulSync(); return true; } catch (Throwable $throwable) { @@ -2457,6 +2461,29 @@ final class TruckwashEdgeAgent } } + private function applyBrokerUrlFromControlPlaneResponse(?array $response): void + { + if (!is_array($response)) { + return; + } + + $payload = is_array($response['data'] ?? null) ? (array)$response['data'] : $response; + $brokerUrl = trim((string)($payload['broker_url'] ?? $payload['gateway']['broker_url'] ?? '')); + if ($brokerUrl === '') { + return; + } + + $current = trim((string)$this->config->get('brokerUrl')); + if (rtrim($current, '/') === rtrim($brokerUrl, '/')) { + return; + } + + $this->config->set('brokerUrl', rtrim($brokerUrl, '/')); + $this->config->save(); + $this->configureBrokerClient(); + $this->logger->info('Updated broker URL from control plane heartbeat response.'); + } + private function shouldPersistControlPlaneEventOverHttp(string $endpoint): bool { return preg_match('#/edge-agent/gateways/\d+/heartbeat$#', $endpoint) === 1; diff --git a/services/nginx/app/routes/authRoute.php b/services/nginx/app/routes/authRoute.php index f8d0ff6c..482a2d79 100644 --- a/services/nginx/app/routes/authRoute.php +++ b/services/nginx/app/routes/authRoute.php @@ -5,6 +5,7 @@ namespace routes; use classes\authentication; use classes\economic; use classes\email; +use classes\release_manager; use classes\recaptcha; use classes\totp; use classes\virkdata; @@ -827,6 +828,7 @@ class authRoute 'transaction_draft_customer_number' => (new economic())->getTransactionDraftCustomerNumber(), 'default_distribution_department_id' => (new economic())->getDefaultDistributionDepartmentId(), ], + 'release' => (new release_manager())->runtimeForPayload($payload), ] ); diff --git a/services/nginx/app/routes/errorReportRoute.php b/services/nginx/app/routes/errorReportRoute.php new file mode 100644 index 00000000..7d691f61 --- /dev/null +++ b/services/nginx/app/routes/errorReportRoute.php @@ -0,0 +1,95 @@ +post('/error-reports', function () { + global $response; + try { + $response->success((new error_report_service())->createFromCurrentPrincipal($this->requestPayload()), 201); + } catch (Throwable $throwable) { + $status = str_contains(strtolower($throwable->getMessage()), 'authentication failed') ? 401 : 400; + $response->error(['message' => $throwable->getMessage()], $status); + } + }); + + $this->get('/superuser/error-reports', function () { + global $response; + $this->requirePermission('superuser_error_reports_view'); + $response->success((new error_report_service())->list($this->getParametersAsArray())); + }, [ + 'superuser_error_reports_view' => 'View authenticated user error reports', + ]); + + $this->get('/superuser/error-reports/{id}', function () { + global $response; + $this->requirePermission('superuser_error_reports_view'); + try { + $response->success((new error_report_service())->get($this->routeId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 404); + } + }, [ + 'superuser_error_reports_view' => 'View authenticated user error report details', + ]); + + $this->patch('/superuser/error-reports/{id}/status', function () { + global $response; + $this->requirePermission('superuser_error_reports_resolve'); + try { + $payload = $this->requestPayload(); + $response->success((new error_report_service())->updateStatus( + $this->routeId(), + (string)($payload['status'] ?? ''), + isset($payload['resolution_note']) ? (string)$payload['resolution_note'] : null, + $this->actorUserId() + )); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_error_reports_resolve' => 'Resolve and reopen authenticated user error reports', + ]); + } + + private function routeId(): int + { + $id = (int)$this->fromRoute('id'); + $this->requireParameterIntPositive($id, 'id'); + return $id; + } + + private function actorUserId(): ?int + { + try { + $user = (new authentication())->get_user(); + return $user !== false && isset($user->id) ? (int)$user->id : null; + } catch (Throwable) { + return null; + } + } + + private function requestPayload(): array + { + $payload = json_decode(file_get_contents('php://input'), true); + if (!is_array($payload)) { + $payload = []; + } + + if ($_GET !== []) { + $payload = array_replace($payload, $_GET); + } + + return $payload; + } +} diff --git a/services/nginx/app/routes/moduleConfigRoute.php b/services/nginx/app/routes/moduleConfigRoute.php index beb67981..040b5e49 100644 --- a/services/nginx/app/routes/moduleConfigRoute.php +++ b/services/nginx/app/routes/moduleConfigRoute.php @@ -218,6 +218,82 @@ class moduleConfigRoute ] ); + $this->get('/failover/config', function () { + global $response; + $this->requirePermission('modules_failover_config'); + $user = (new authentication())->get_user(); + if ($user) { + (new logs_o())->add('failover_config', 'global', 1, $user->id, 'FAILOVER_CONFIG', 'Successfully fetched failover config'); + $response->success( + (new \classes\failover())->config->getConfigRequest() + ); + } else { + (new logs_o())->add('failover_config', 'global', 1, 0, 'FAILOVER_CONFIG', 'No user found, or invalid session'); + $response->error('Invalid session', 400); + } + }, + [ + 'modules_failover_config' => 'Get failover config' + ] + ); + + $this->post('/failover/config', function () { + global $response; + $this->requirePermission('modules_failover_config'); + $user = (new authentication())->get_user(); + if ($user) { + (new logs_o())->add('failover_config', 'global', 1, $user->id, 'FAILOVER_CONFIG', 'Successfully updated failover config'); + $response->success( + (new \classes\failover())->config->postConfigRequest() + ); + } else { + (new logs_o())->add('failover_config', 'global', 1, 0, 'FAILOVER_CONFIG', 'No user found, or invalid session'); + $response->error('Invalid session', 400); + } + }, + [ + 'modules_failover_config' => 'Update failover config' + ] + ); + + $this->get('/coolify/config', function () { + global $response; + $this->requirePermission('superuser_coolify_manage'); + $user = (new authentication())->get_user(); + if ($user) { + (new logs_o())->add('coolify_config', 'global', 1, $user->id, 'COOLIFY_CONFIG', 'Successfully fetched Coolify config'); + $response->success( + (new \classes\coolify())->config->getConfigRequest() + ); + } else { + (new logs_o())->add('coolify_config', 'global', 1, 0, 'COOLIFY_CONFIG', 'No user found, or invalid session'); + $response->error('Invalid session', 400); + } + }, + [ + 'superuser_coolify_manage' => 'Get Coolify config' + ] + ); + + $this->post('/coolify/config', function () { + global $response; + $this->requirePermission('superuser_coolify_manage'); + $user = (new authentication())->get_user(); + if ($user) { + (new logs_o())->add('coolify_config', 'global', 1, $user->id, 'COOLIFY_CONFIG', 'Successfully updated Coolify config'); + $response->success( + (new \classes\coolify())->config->postConfigRequest() + ); + } else { + (new logs_o())->add('coolify_config', 'global', 1, 0, 'COOLIFY_CONFIG', 'No user found, or invalid session'); + $response->error('Invalid session', 400); + } + }, + [ + 'superuser_coolify_manage' => 'Update Coolify config' + ] + ); + /** Bird config > GET */ $this->get('/bird/config', function () { global $response; diff --git a/services/nginx/app/routes/moduleScannerRoute.php b/services/nginx/app/routes/moduleScannerRoute.php index cc53a85f..1cb3049e 100644 --- a/services/nginx/app/routes/moduleScannerRoute.php +++ b/services/nginx/app/routes/moduleScannerRoute.php @@ -49,8 +49,10 @@ class moduleScannerRoute // Success $response->success(['success' => true, 'license_plate_number' => $lpr_result['license_plate_number']]); } else { - throw new Exception('License plate extraction failed.'); - //$response->error($lpr_result['message'] ?? 'License plate recognition failed.', $lpr_result); + $response->response(false, [ + 'message' => $lpr_result['message'] ?? 'No license plate detected.', + 'reason' => 'no_license_plate_detected', + ], 200); } exit; // For future use with OpenAI. @@ -73,4 +75,4 @@ class moduleScannerRoute ] ); } -} \ No newline at end of file +} diff --git a/services/nginx/app/routes/releaseManagerRoute.php b/services/nginx/app/routes/releaseManagerRoute.php new file mode 100644 index 00000000..55f36589 --- /dev/null +++ b/services/nginx/app/routes/releaseManagerRoute.php @@ -0,0 +1,377 @@ +get('/release/bootstrap', function () { + global $response; + $response->success((new release_manager())->bootstrap()); + }); + + $this->get('/release/runtime', function () { + global $response; + $response->success((new release_manager())->runtimeForCurrentPrincipal()); + }); + + $this->post('/release/timeline/events', function () { + global $response; + $payload = $this->requestPayload(); + $events = is_array($payload['events'] ?? null) ? $payload['events'] : ($payload['event'] ?? $payload); + $context = is_array($payload['context'] ?? null) ? $payload['context'] : []; + $response->success((new release_manager())->ingestTimelineEvents( + is_array($events) ? $events : [], + $context + ), 202); + }); + + $this->post('/release/github/webhook', function () { + global $response; + try { + $headers = function_exists('getallheaders') ? getallheaders() : []; + $rawBody = file_get_contents('php://input') ?: ''; + $response->success((new release_manager())->handleGithubWebhook($headers, $rawBody), 202); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 401); + } + }); + + $this->get('/superuser/releases', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + $response->success((new release_manager())->summary()); + }, [ + 'superuser_release_manager_view' => 'View release manager channels, deployments, and health', + ]); + + $this->get('/superuser/releases/config', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + $response->success((new release_manager())->releaseConfig()); + }, [ + 'superuser_release_manager_view' => 'View Release Manager source configuration', + ]); + + $this->post('/superuser/releases/config', function () { + global $response; + $this->requirePermission('superuser_release_manager_manage'); + try { + $response->success((new release_manager())->updateReleaseConfig($this->requestPayload(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_manage' => 'Update Release Manager source configuration', + ]); + + $this->get('/superuser/releases/github/repositories', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + try { + $response->success((new release_manager())->listGithubRepositories($this->getParametersAsArray())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_view' => 'List private GitHub repositories available to Release Manager', + ]); + + $this->get('/superuser/releases/github/branches', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->listGithubBranches($this->getParametersAsArray())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_deploy' => 'List GitHub branches for a Release Manager repository', + ]); + + $this->get('/superuser/releases/github/commits', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->listGithubCommits($this->getParametersAsArray())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_deploy' => 'List or resolve GitHub commits for a Release Manager repository', + ]); + + $this->post('/superuser/releases/github/test', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + $response->success((new release_manager())->testGithubRepositoryAccess($this->requestPayload())); + }, [ + 'superuser_release_manager_deploy' => 'Test Release Manager access to a GitHub repository, branch, and commit', + ]); + + $this->get('/superuser/releases/channels', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + $response->success((new release_manager())->listChannels()); + }, [ + 'superuser_release_manager_view' => 'View release channels', + ]); + + $this->post('/superuser/releases/channels', function () { + global $response; + $this->requirePermission('superuser_release_manager_manage'); + try { + $response->success((new release_manager())->createChannel($this->requestPayload(), $this->actorUserId()), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_manage' => 'Create release channels', + ]); + + $this->patch('/superuser/releases/channels/{id}', function () { + global $response; + $this->requirePermission('superuser_release_manager_manage'); + try { + $response->success((new release_manager())->updateChannel($this->routeId(), $this->requestPayload(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_manage' => 'Update release channels', + ]); + + $this->post('/superuser/releases/channels/{id}/rollback', function () { + global $response; + $this->requirePermission('superuser_release_manager_rollback'); + try { + $response->success((new release_manager())->rollbackChannel($this->routeId(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_release_manager_rollback' => 'Rollback an active release channel', + ]); + + $this->get('/superuser/releases/assignments', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + $response->success((new release_manager())->listAssignments()); + }, [ + 'superuser_release_manager_view' => 'View release channel assignments', + ]); + + $this->post('/superuser/releases/assignments', function () { + global $response; + $this->requirePermission('superuser_release_manager_manage'); + try { + $response->success((new release_manager())->createAssignment($this->requestPayload(), $this->actorUserId()), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_manage' => 'Assign users, subusers, or customers to release channels', + ]); + + $this->delete('/superuser/releases/assignments/{id}', function () { + global $response; + $this->requirePermission('superuser_release_manager_manage'); + try { + $response->success((new release_manager())->deleteAssignment($this->routeId(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 404); + } + }, [ + 'superuser_release_manager_manage' => 'Remove release channel assignments', + ]); + + $this->get('/superuser/releases/targets', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + $response->success((new release_manager())->listDeploymentTargets()); + }, [ + 'superuser_release_manager_deploy' => 'View release deployment targets', + ]); + + $this->post('/superuser/releases/targets', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->upsertDeploymentTarget($this->requestPayload(), $this->actorUserId()), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_deploy' => 'Create or update GitHub to Coolify release deployment targets', + ]); + + $this->delete('/superuser/releases/targets/{id}', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->deleteDeploymentTarget($this->routeId(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 404); + } + }, [ + 'superuser_release_manager_deploy' => 'Delete release deployment targets', + ]); + + $this->get('/superuser/releases/service-sets', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + $response->success((new release_manager())->listServiceSets()); + }, [ + 'superuser_release_manager_view' => 'View reusable Release Manager service sets', + ]); + + $this->post('/superuser/releases/service-sets', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->createServiceSet($this->requestPayload(), $this->actorUserId()), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_deploy' => 'Create reusable Release Manager service sets', + ]); + + $this->get('/superuser/releases/bundles', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + $limit = (int)($this->getParameter('limit') ?? 50); + $response->success((new release_manager())->listBundles($limit)); + }, [ + 'superuser_release_manager_view' => 'View Release Manager bundles', + ]); + + $this->post('/superuser/releases/bundles', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->createBundle($this->requestPayload(), $this->actorUserId()), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_deploy' => 'Create Release Manager full-stack bundles', + ]); + + $this->post('/superuser/releases/bundles/{id}/deploy', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->deployBundle($this->routeId(), $this->actorUserId()), 202); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_release_manager_deploy' => 'Deploy Release Manager full-stack bundles', + ]); + + $this->post('/superuser/releases/bundles/{id}/promote', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->promoteBundle($this->routeId(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_release_manager_deploy' => 'Promote Release Manager bundles without data failover', + ]); + + $this->get('/superuser/releases/deployments', function () { + global $response; + $this->requirePermission('superuser_release_manager_view'); + $limit = (int)($this->getParameter('limit') ?? 50); + $response->success((new release_manager())->listDeployments($limit)); + }, [ + 'superuser_release_manager_view' => 'View release deployments', + ]); + + $this->post('/superuser/releases/deployments', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->startDeployment($this->requestPayload(), $this->actorUserId()), 202); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_release_manager_deploy' => 'Trigger release deployments from GitHub/Coolify targets', + ]); + + $this->post('/superuser/releases/deployments/{id}/promote', function () { + global $response; + $this->requirePermission('superuser_release_manager_deploy'); + try { + $response->success((new release_manager())->promoteDeployment($this->routeId(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_release_manager_deploy' => 'Promote a deployment to its release channel', + ]); + + $this->post('/superuser/releases/replay-targets', function () { + global $response; + $this->requirePermission('superuser_release_manager_replay'); + try { + $response->success((new release_manager())->setReplayTarget($this->requestPayload(), $this->actorUserId()), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_release_manager_replay' => 'Enable release timeline replay capture for a user, customer, subuser, or channel', + ]); + + $this->get('/superuser/releases/timeline', function () { + global $response; + $this->requirePermission('superuser_release_manager_replay'); + $response->success((new release_manager())->searchTimeline($this->getParametersAsArray())); + }, [ + 'superuser_release_manager_replay' => 'Replay release failure timelines', + ]); + } + + private function routeId(): int + { + $id = (int)$this->fromRoute('id'); + $this->requireParameterIntPositive($id, 'id'); + return $id; + } + + private function actorUserId(): ?int + { + try { + $user = (new authentication())->get_user(); + return $user !== false && isset($user->id) ? (int)$user->id : null; + } catch (Throwable) { + return null; + } + } + + private function requestPayload(): array + { + $payload = json_decode(file_get_contents('php://input'), true); + if (!is_array($payload)) { + $payload = []; + } + + if ($_GET !== []) { + $payload = array_replace($payload, $_GET); + } + + return $payload; + } +} diff --git a/services/nginx/app/routes/superuserCoolifyRoute.php b/services/nginx/app/routes/superuserCoolifyRoute.php new file mode 100644 index 00000000..4fee2f92 --- /dev/null +++ b/services/nginx/app/routes/superuserCoolifyRoute.php @@ -0,0 +1,258 @@ +get('/superuser/coolify', function () { + global $response; + + $this->requirePermission('superuser_coolify_view'); + $response->success((new coolify_manager())->summary()); + }, [ + 'superuser_coolify_view' => 'View Coolify-managed replicated infrastructure targets', + ]); + + $this->get('/superuser/coolify/load-balancer', function () { + global $response; + + $this->requirePermission('superuser_coolify_view'); + $response->success((new coolify_manager())->loadBalancerSummary()); + }, [ + 'superuser_coolify_view' => 'View the Coolify public gateway Load Balancer state', + ]); + + $this->post('/superuser/coolify/load-balancer/reconcile', function () { + global $response; + + $this->requirePermission('superuser_coolify_manage'); + try { + $parameters = $this->getParametersAsArray(); + $dryRun = array_key_exists('dry_run', $parameters) + ? filter_var($parameters['dry_run'], FILTER_VALIDATE_BOOLEAN) + : !filter_var($parameters['enforce'] ?? false, FILTER_VALIDATE_BOOLEAN); + $result = (new coolify_manager())->reconcileLoadBalancer($dryRun, $this->actorUserId()); + if (($result['ok'] ?? false) !== true) { + $response->error($result, 409); + } + $response->success($result); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_manage' => 'Reconcile Hetzner Load Balancer targets and services for the Coolify gateway', + ]); + + $this->get('/superuser/coolify/gateways', function () { + global $response; + + $this->requirePermission('superuser_coolify_view'); + $response->success((new coolify_manager())->listLoadBalancerGateways()); + }, [ + 'superuser_coolify_view' => 'List Coolify public gateway Load Balancer targets', + ]); + + $this->post('/superuser/coolify/gateways', function () { + global $response; + + $this->requirePermission('superuser_coolify_manage'); + try { + $response->success((new coolify_manager())->saveLoadBalancerGateway( + $this->getParametersAsArray(), + $this->actorUserId() + ), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_coolify_manage' => 'Create or update Coolify public gateway Load Balancer targets', + ]); + + $this->post('/superuser/coolify/gateways/{id}/test', function () { + global $response; + + $this->requirePermission('superuser_coolify_manage'); + try { + $response->success((new coolify_manager())->testLoadBalancerGateway( + $this->routeId(), + $this->actorUserId() + )); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_manage' => 'Probe an individual Coolify public gateway target', + ]); + + $this->post('/superuser/coolify/instances', function () { + global $response; + + $this->requirePermission('superuser_coolify_manage'); + try { + $response->success((new coolify_manager())->createInstance( + $this->getParametersAsArray(), + $this->actorUserId() + ), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_coolify_manage' => 'Create and update Coolify API connections', + ]); + + $this->post('/superuser/coolify/instances/{id}/test', function () { + global $response; + + $this->requirePermission('superuser_coolify_manage'); + $response->success((new coolify_manager())->testInstance($this->routeId(), $this->actorUserId())); + }, [ + 'superuser_coolify_manage' => 'Test Coolify API connectivity', + ]); + + $this->get('/superuser/coolify/instances/{id}/placement', function () { + global $response; + + $this->requirePermission('superuser_coolify_view'); + try { + $response->success((new coolify_manager())->discoverInstancePlacement($this->routeId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 502); + } + }, [ + 'superuser_coolify_view' => 'Discover Coolify projects, environments, and servers for target placement', + ]); + + $this->get('/superuser/coolify/targets', function () { + global $response; + + $this->requirePermission('superuser_coolify_view'); + $kind = (string)($this->getParameter('kind') ?? ''); + $response->success((new coolify_manager())->listTargets($kind !== '' ? $kind : null)); + }, [ + 'superuser_coolify_view' => 'List Coolify-managed replication targets', + ]); + + $this->post('/superuser/coolify/targets', function () { + global $response; + + $this->requirePermission('superuser_coolify_manage'); + try { + $response->success((new coolify_manager())->createTarget( + $this->getParametersAsArray(), + $this->actorUserId() + ), 201); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_manage' => 'Create Coolify-managed MariaDB, Redis, and MinIO replication targets', + ]); + + $this->post('/superuser/coolify/targets/{id}/reconcile', function () { + global $response; + + $this->requirePermission('superuser_coolify_reconcile'); + try { + $result = (new coolify_manager())->reconcileTarget($this->routeId(), $this->actorUserId()); + if (($result['ok'] ?? false) !== true) { + $response->error($result, 409); + } + $response->success($result); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_reconcile' => 'Reconcile expected Coolify deployment state without primary downtime', + ]); + + $this->post('/superuser/coolify/targets/{id}/deploy', function () { + global $response; + + $this->requirePermission('superuser_coolify_reconcile'); + try { + $result = (new coolify_manager())->deployTarget($this->routeId(), $this->actorUserId()); + if (($result['ok'] ?? false) !== true) { + $response->error($result, 409); + } + $response->success($result); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_reconcile' => 'Deploy and provision a passive Coolify-managed replication target', + ]); + + $this->post('/superuser/coolify/targets/{id}/restart', function () { + global $response; + + $this->requirePermission('superuser_coolify_reconcile'); + try { + $result = (new coolify_manager())->restartTarget($this->routeId(), $this->actorUserId()); + if (($result['ok'] ?? false) !== true) { + $response->error($result, 409); + } + $response->success($result); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_reconcile' => 'Restart a passive Coolify target without restarting the active primary', + ]); + + $this->post('/superuser/coolify/targets/{id}/failover', function () { + global $response; + + $this->requirePermission('superuser_coolify_failover'); + try { + $response->success((new coolify_manager())->failoverTarget($this->routeId(), $this->actorUserId())); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_failover' => 'Promote a healthy Coolify-managed replica through the replication module', + ]); + + $this->delete('/superuser/coolify/targets/{id}', function () { + global $response; + + $this->requirePermission('superuser_coolify_manage'); + try { + $response->success((new coolify_manager())->deleteTarget( + $this->routeId(), + $this->getParametersAsArray(), + $this->actorUserId() + )); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 409); + } + }, [ + 'superuser_coolify_manage' => 'Delete Coolify target mappings with explicit destructive confirmation', + ]); + } + + private function routeId(): int + { + $id = (int)$this->fromRoute('id'); + $this->requireParameterIntPositive($id, 'id'); + return $id; + } + + private function actorUserId(): ?int + { + try { + $user = (new authentication())->get_user(); + return $user !== false && isset($user->id) ? (int)$user->id : null; + } catch (Throwable) { + return null; + } + } +} diff --git a/services/nginx/app/routes/superuserReplicationRoute.php b/services/nginx/app/routes/superuserReplicationRoute.php index 5c98e821..b9dddddc 100644 --- a/services/nginx/app/routes/superuserReplicationRoute.php +++ b/services/nginx/app/routes/superuserReplicationRoute.php @@ -96,7 +96,8 @@ class superuserReplicationRoute $result = (new replication_manager())->provisionHost( (string)$this->fromRoute('kind'), $this->routeId(), - $this->actorUserId() + $this->actorUserId(), + true ); if (($result['ok'] ?? false) !== true) { $response->error($result, 409); @@ -126,6 +127,24 @@ class superuserReplicationRoute 'superuser_replication_promote' => 'Promote a healthy caught-up database, Redis, or MinIO replica to primary', ]); + $this->patch('/superuser/replication/{kind}/{id}', function () { + global $response; + + $this->requirePermission('superuser_replication_manage'); + try { + $response->success((new replication_manager())->renameHost( + (string)$this->fromRoute('kind'), + $this->routeId(), + $this->getParametersAsArray(), + $this->actorUserId() + )); + } catch (Throwable $throwable) { + $response->error(['message' => $throwable->getMessage()], 400); + } + }, [ + 'superuser_replication_manage' => 'Rename database, Redis, and MinIO replication hosts', + ]); + $this->delete('/superuser/replication/{kind}/{id}', function () { global $response; diff --git a/services/nginx/app/storage/replication-bootstrap.json b/services/nginx/app/storage/replication-bootstrap.json new file mode 100644 index 00000000..3be0506e --- /dev/null +++ b/services/nginx/app/storage/replication-bootstrap.json @@ -0,0 +1,222 @@ +{ + "version": 1, + "generated_at": "2026-05-19T00:57:20+02:00", + "active": { + "database": { + "id": 1, + "host": "23.88.23.183", + "port": 5432, + "database": "nnks_db", + "user": "root", + "password_secret": "twsec:v1:eyJub25jZSI6ImRBSDRkejROdGNKTm5NNHQiLCJ0YWciOiJFRFNCMmlNRWI3eXZXUDdvT1FIZnB3PT0iLCJjaXBoZXJ0ZXh0IjoiNEpSUGJnSjExVmV1cWIrV3ByZDV0b0l0NlE9PSJ9", + "ssl_mode": "DISABLED" + }, + "redis": { + "id": 2, + "host": "23.88.23.183", + "port": 5433, + "database": 0, + "user": "default", + "password_secret": "twsec:v1:eyJub25jZSI6Ii9PVTB0bExoend6RWlPRGoiLCJ0YWciOiIxVGRyWEFlVkV4NnpieHIxQVBrcGR3PT0iLCJjaXBoZXJ0ZXh0IjoiZG02ekVoRjI4blZjUWFuN0R5UWRxYWExY0E9PSJ9" + }, + "minio": { + "id": 4, + "endpoint": "http://162.55.225.220:9000", + "access_key": "d7u6RaFyYmckAIWYGUYr", + "secret_key_secret": "twsec:v1:eyJub25jZSI6Ikx5SjdUWTZqVTIrV0lWK1UiLCJ0YWciOiI5dkF5MVB1MWRwcEZNU3NzVVNETUZRPT0iLCJjaXBoZXJ0ZXh0IjoiUkRCQldZdkE3SksxWG1CMWN2TnZpUnFiSXY5ckNuL3Raanc5K3lIbGwzQkJlbTBLOUV1c0ZRPT0ifQ==", + "buckets": [ + "attachments", + "backups", + "invoices", + "pdfs", + "uploads", + "truckwashdev" + ] + } + }, + "failover": { + "config": { + "enabled": true, + "database_enabled": true, + "redis_enabled": false, + "minio_enabled": true, + "max_status_age_seconds": 90 + }, + "hosts": { + "database": [ + { + "id": 1, + "kind": "database", + "label": "Current database primary", + "host": "23.88.23.183", + "port": 5432, + "database_name": "nnks_db", + "database_index": null, + "username": "root", + "password_secret": "twsec:v1:eyJub25jZSI6ImRBSDRkejROdGNKTm5NNHQiLCJ0YWciOiJFRFNCMmlNRWI3eXZXUDdvT1FIZnB3PT0iLCJjaXBoZXJ0ZXh0IjoiNEpSUGJnSjExVmV1cWIrV3ByZDV0b0l0NlE9PSJ9", + "admin_username": "", + "admin_password_secret": "", + "replication_username": "", + "replication_password_secret": "", + "role": "primary", + "status": "ok", + "replication_source_id": null, + "ssl_mode": "DISABLED", + "options_json": "{\"source\":\"environment\"}", + "last_status_json": "{\"status\":\"ok\",\"replication_percent\":100,\"lag_seconds\":null,\"blockers\":[],\"raw\":{\"server_version\":\"11.8.6-MariaDB-ubu2404-log\",\"gtid_binlog_pos\":\"0-1-1114923\",\"gtid_current_pos\":\"0-1-1114923\",\"gtid_slave_pos\":\"\",\"gtid_strict_mode\":\"ON\",\"log_bin\":\"ON\",\"read_only\":\"OFF\",\"server_id\":\"1\",\"clone_plugin_active\":false},\"checked_at\":\"2026-05-19T00:57:17+02:00\"}", + "last_checked_at": "2026-05-18 22:57:17", + "updated_at": "2026-05-18 22:57:17", + "deleted_at": null + }, + { + "id": 3, + "kind": "database", + "label": "mariadb-replica-1", + "host": "65.21.214.30", + "port": 5441, + "database_name": "nnks_db", + "database_index": null, + "username": "nnks_db_user", + "password_secret": "twsec:v1:eyJub25jZSI6IkFzazZBNCsxWFh0L3dUWDEiLCJ0YWciOiJ2WVhNNXhteGpZZ0NHeXEzb0hILzBRPT0iLCJjaXBoZXJ0ZXh0IjoiNjVNaUVwRFVNeDMrdmtUZ2lqelhtbkJnK05HV0p3dlpyUFFkWWhKazdBMD0ifQ==", + "admin_username": "root", + "admin_password_secret": "twsec:v1:eyJub25jZSI6IjVRNkVIdGNEV2JhZXVEc04iLCJ0YWciOiJRZitRUHhQYzM1NE1jVVpYbGFmbzBBPT0iLCJjaXBoZXJ0ZXh0IjoiZXAzbzJuQW0vMldHSkFHSXhGWmM4SkFyeFVnNWt1K1JrcmpxN3gxZUtnYz0ifQ==", + "replication_username": "replication", + "replication_password_secret": "twsec:v1:eyJub25jZSI6IlJUYW1pUHQ2UjhZTFlIUloiLCJ0YWciOiJDaVI5Y251eUVkRHo2RmorYkp3N2NRPT0iLCJjaXBoZXJ0ZXh0IjoiREp5WTRTSS9qVXcyOEFGdVJBTE5BdzVnY2ROVFNiNmR2NkQ5Y1M3RkJSZz0ifQ==", + "role": "replica", + "status": "ok", + "replication_source_id": 1, + "ssl_mode": "DISABLED", + "options_json": "{\"allow_preseeded_replica\":true}", + "last_status_json": "{\"status\":\"ok\",\"replication_percent\":100,\"lag_seconds\":0,\"blockers\":[],\"raw\":{\"server_version\":\"11.8.6-MariaDB-ubu2404-log\",\"gtid_binlog_pos\":\"0-2-1098\",\"gtid_current_pos\":\"0-1-1114860\",\"gtid_slave_pos\":\"0-1-1114860\",\"gtid_strict_mode\":\"ON\",\"log_bin\":\"ON\",\"read_only\":\"ON\",\"server_id\":\"2\",\"clone_plugin_active\":false,\"source_gtid_executed\":\"0-1-1114955\",\"replica_status\":{\"Slave_IO_State\":\"Waiting for master to send event\",\"Master_Host\":\"23.88.23.183\",\"Master_User\":\"replication\",\"Master_Port\":\"5432\",\"Connect_Retry\":\"60\",\"Master_Log_File\":\"mariadb-bin.000017\",\"Read_Master_Log_Pos\":\"237616337\",\"Relay_Log_File\":\"mysqld-relay-bin.000002\",\"Relay_Log_Pos\":\"97422\",\"Relay_Master_Log_File\":\"mariadb-bin.000017\",\"Slave_IO_Running\":\"Yes\",\"Slave_SQL_Running\":\"Yes\",\"Replicate_Do_DB\":\"\",\"Replicate_Ignore_DB\":\"\",\"Replicate_Do_Table\":\"\",\"Replicate_Ignore_Table\":\"nnks_db.edge_gateway_shell_sessions,nnks_db.logs,nnks_db.edge_gateway_log_entries,nnks_db.replication_status_snapshots,nnks_db.system_search_documents,nnks_db.replication_operations,nnks_db.replication_audit_logs,nnks_db.edge_gateway_audit_logs\",\"Replicate_Wild_Do_Table\":\"\",\"Replicate_Wild_Ignore_Table\":\"\",\"Last_Errno\":\"0\",\"Last_Error\":\"\",\"Skip_Counter\":\"0\",\"Exec_Master_Log_Pos\":\"237616337\",\"Relay_Log_Space\":\"97732\",\"Until_Condition\":\"None\",\"Until_Log_File\":\"\",\"Until_Log_Pos\":\"0\",\"Master_SSL_Allowed\":\"Yes\",\"Master_SSL_CA_File\":\"\",\"Master_SSL_CA_Path\":\"\",\"Master_SSL_Cert\":\"\",\"Master_SSL_Cipher\":\"\",\"Master_SSL_Key\":\"\",\"Seconds_Behind_Master\":\"0\",\"Master_SSL_Verify_Server_Cert\":\"Yes\",\"Last_IO_Errno\":\"0\",\"Last_IO_Error\":\"\",\"Last_SQL_Errno\":\"0\",\"Last_SQL_Error\":\"\",\"Replicate_Ignore_Server_Ids\":\"\",\"Master_Server_Id\":\"1\",\"Master_SSL_Crl\":\"\",\"Master_SSL_Crlpath\":\"\",\"Using_Gtid\":\"Slave_Pos\",\"Gtid_IO_Pos\":\"0-1-1114955\",\"Replicate_Do_Domain_Ids\":\"\",\"Replicate_Ignore_Domain_Ids\":\"\",\"Parallel_Mode\":\"optimistic\",\"SQL_Delay\":\"0\",\"SQL_Remaining_Delay\":null,\"Slave_SQL_Running_State\":\"Slave has read all relay log; waiting for more updates\",\"Slave_DDL_Groups\":\"180\",\"Slave_Non_Transactional_Groups\":\"15\",\"Slave_Transactional_Groups\":\"900658\",\"Replicate_Rewrite_DB\":\"\"}},\"checked_at\":\"2026-05-19T00:57:17+02:00\"}", + "last_checked_at": "2026-05-18 22:57:18", + "updated_at": "2026-05-18 22:57:18", + "deleted_at": null + }, + { + "id": 8, + "kind": "database", + "label": "mariadb-replica-2", + "host": "65.21.214.30", + "port": 3307, + "database_name": "nnks_db", + "database_index": null, + "username": "nnks_db_user", + "password_secret": "twsec:v1:eyJub25jZSI6IitoVThzV0UxVEEycDhZYkYiLCJ0YWciOiJwZSs4Z2g1QWxKNzdwb3F4c1pQelV3PT0iLCJjaXBoZXJ0ZXh0IjoiUXdFWnhMUGx1ejczbHc0aUpxc2xaeUExOFlRcWwralhGWkdtSWE2TzVHMD0ifQ==", + "admin_username": "root", + "admin_password_secret": "twsec:v1:eyJub25jZSI6Ik9BbElsV0RZSUZpUHRsZmQiLCJ0YWciOiJNczZzWFBueUVneUVNRUh1RGlSQjZnPT0iLCJjaXBoZXJ0ZXh0IjoiUW52Mmh2MUJOQXdIbm12eDg1MnUyeFRsOW9xTUxpOXI4Y1pldXV1bEdEaz0ifQ==", + "replication_username": "replication", + "replication_password_secret": "twsec:v1:eyJub25jZSI6IlJ5aTBVbCtUODFDRVArTWsiLCJ0YWciOiJKMVZ6MVZpQ2lRZU04SkpLYmlKa3hRPT0iLCJjaXBoZXJ0ZXh0IjoiajJTQUl6OW40T0xYTFZXdjhSeHlVWGJSZ2M0ZzY1R0U4ZWdIRHN4UHo5dz0ifQ==", + "role": "replica", + "status": "ok", + "replication_source_id": 1, + "ssl_mode": "DISABLED", + "options_json": "{\"allow_preseeded_replica\":true,\"deployment_provider\":\"coolify\",\"coolify_instance_id\":1,\"coolify_target_id\":1}", + "last_status_json": "{\"status\":\"ok\",\"replication_percent\":100,\"lag_seconds\":0,\"blockers\":[],\"raw\":{\"server_version\":\"11.8.6-MariaDB-ubu2404-log\",\"gtid_binlog_pos\":\"0-2-1098\",\"gtid_current_pos\":\"0-1-1114955\",\"gtid_slave_pos\":\"0-1-1114955\",\"gtid_strict_mode\":\"ON\",\"log_bin\":\"ON\",\"read_only\":\"ON\",\"server_id\":\"2\",\"clone_plugin_active\":false,\"source_gtid_executed\":\"0-1-1114989\",\"replica_status\":{\"Slave_IO_State\":\"Waiting for master to send event\",\"Master_Host\":\"23.88.23.183\",\"Master_User\":\"replication\",\"Master_Port\":\"5432\",\"Connect_Retry\":\"60\",\"Master_Log_File\":\"\",\"Read_Master_Log_Pos\":\"4\",\"Relay_Log_File\":\"mysqld-relay-bin.000001\",\"Relay_Log_Pos\":\"4\",\"Relay_Master_Log_File\":\"\",\"Slave_IO_Running\":\"Yes\",\"Slave_SQL_Running\":\"Yes\",\"Replicate_Do_DB\":\"\",\"Replicate_Ignore_DB\":\"\",\"Replicate_Do_Table\":\"\",\"Replicate_Ignore_Table\":\"nnks_db.edge_gateway_shell_sessions,nnks_db.logs,nnks_db.replication_status_snapshots,nnks_db.edge_gateway_log_entries,nnks_db.edge_gateway_audit_logs,nnks_db.replication_operations,nnks_db.replication_audit_logs,nnks_db.system_search_documents\",\"Replicate_Wild_Do_Table\":\"\",\"Replicate_Wild_Ignore_Table\":\"\",\"Last_Errno\":\"0\",\"Last_Error\":\"\",\"Skip_Counter\":\"0\",\"Exec_Master_Log_Pos\":\"4\",\"Relay_Log_Space\":\"256\",\"Until_Condition\":\"None\",\"Until_Log_File\":\"\",\"Until_Log_Pos\":\"0\",\"Master_SSL_Allowed\":\"Yes\",\"Master_SSL_CA_File\":\"\",\"Master_SSL_CA_Path\":\"\",\"Master_SSL_Cert\":\"\",\"Master_SSL_Cipher\":\"\",\"Master_SSL_Key\":\"\",\"Seconds_Behind_Master\":\"0\",\"Master_SSL_Verify_Server_Cert\":\"Yes\",\"Last_IO_Errno\":\"0\",\"Last_IO_Error\":\"\",\"Last_SQL_Errno\":\"0\",\"Last_SQL_Error\":\"\",\"Replicate_Ignore_Server_Ids\":\"\",\"Master_Server_Id\":\"1\",\"Master_SSL_Crl\":\"\",\"Master_SSL_Crlpath\":\"\",\"Using_Gtid\":\"Slave_Pos\",\"Gtid_IO_Pos\":\"0-1-1114955\",\"Replicate_Do_Domain_Ids\":\"\",\"Replicate_Ignore_Domain_Ids\":\"\",\"Parallel_Mode\":\"optimistic\",\"SQL_Delay\":\"0\",\"SQL_Remaining_Delay\":null,\"Slave_SQL_Running_State\":\"Slave has read all relay log; waiting for more updates\",\"Slave_DDL_Groups\":\"171\",\"Slave_Non_Transactional_Groups\":\"14\",\"Slave_Transactional_Groups\":\"114109\",\"Replicate_Rewrite_DB\":\"\"},\"replication_access_repair\":{\"ok\":true,\"denied_hosts\":[\"10.0.1.13\"],\"grant_hosts\":[\"%\",\"65.21.214.30\",\"65.21.214.%\",\"10.0.1.13\",\"10.0.1.%\"]}},\"checked_at\":\"2026-05-19T00:57:20+02:00\"}", + "last_checked_at": "2026-05-18 22:57:20", + "updated_at": "2026-05-18 22:57:20", + "deleted_at": null + } + ], + "redis": [ + { + "id": 2, + "kind": "redis", + "label": "Current Redis primary", + "host": "23.88.23.183", + "port": 5433, + "database_name": null, + "database_index": 0, + "username": "default", + "password_secret": "twsec:v1:eyJub25jZSI6Ii9PVTB0bExoend6RWlPRGoiLCJ0YWciOiIxVGRyWEFlVkV4NnpieHIxQVBrcGR3PT0iLCJjaXBoZXJ0ZXh0IjoiZG02ekVoRjI4blZjUWFuN0R5UWRxYWExY0E9PSJ9", + "admin_username": "", + "admin_password_secret": "", + "replication_username": "", + "replication_password_secret": "", + "role": "primary", + "status": "ok", + "replication_source_id": null, + "ssl_mode": null, + "options_json": "{\"source\":\"environment\"}", + "last_status_json": "{\"status\":\"ok\",\"replication_percent\":100,\"lag_seconds\":null,\"blockers\":[],\"raw\":{\"role\":[\"master\",158560415,[]],\"replication\":{\"role\":\"master\",\"connected_slaves\":\"0\",\"master_failover_state\":\"no-failover\",\"master_replid\":\"d66ae5e486a819a693d4dc9cd12883ed8f987c74\",\"master_replid2\":\"0000000000000000000000000000000000000000\",\"master_repl_offset\":\"158560415\",\"second_repl_offset\":\"-1\",\"repl_backlog_active\":\"0\",\"repl_backlog_size\":\"1048576\",\"repl_backlog_first_byte_offset\":\"0\",\"repl_backlog_histlen\":\"0\"}},\"checked_at\":\"2026-05-19T00:57:17+02:00\"}", + "last_checked_at": "2026-05-18 22:57:17", + "updated_at": "2026-05-18 22:57:17", + "deleted_at": null + } + ], + "minio": [ + { + "id": 4, + "kind": "minio", + "label": "Current MinIO primary", + "host": "162.55.225.220", + "port": 9000, + "database_name": null, + "database_index": null, + "username": "d7u6RaFyYmckAIWYGUYr", + "password_secret": "twsec:v1:eyJub25jZSI6Ikx5SjdUWTZqVTIrV0lWK1UiLCJ0YWciOiI5dkF5MVB1MWRwcEZNU3NzVVNETUZRPT0iLCJjaXBoZXJ0ZXh0IjoiUkRCQldZdkE3SksxWG1CMWN2TnZpUnFiSXY5ckNuL3Raanc5K3lIbGwzQkJlbTBLOUV1c0ZRPT0ifQ==", + "admin_username": "", + "admin_password_secret": "", + "replication_username": "", + "replication_password_secret": "", + "role": "primary", + "status": "ok", + "replication_source_id": null, + "ssl_mode": null, + "options_json": "{\"source\":\"environment\",\"scheme\":\"http\",\"endpoint\":\"http://162.55.225.220:9000\",\"buckets\":[\"attachments\",\"backups\",\"invoices\",\"pdfs\",\"uploads\",\"truckwashdev\"],\"console_port\":9001,\"space_headroom_percent\":20}", + "last_status_json": "{\"status\":\"ok\",\"replication_percent\":100,\"lag_seconds\":null,\"blockers\":[],\"raw\":{\"buckets\":[{\"name\":\"attachments\",\"status\":\"ok\",\"bytes\":null,\"objects\":null,\"expired_bytes\":null,\"expired_objects\":null,\"measured\":false,\"retention_days\":null},{\"name\":\"backups\",\"status\":\"ok\",\"bytes\":null,\"objects\":null,\"expired_bytes\":null,\"expired_objects\":null,\"measured\":false,\"retention_days\":null},{\"name\":\"invoices\",\"status\":\"ok\",\"bytes\":null,\"objects\":null,\"expired_bytes\":null,\"expired_objects\":null,\"measured\":false,\"retention_days\":null},{\"name\":\"pdfs\",\"status\":\"ok\",\"bytes\":null,\"objects\":null,\"expired_bytes\":null,\"expired_objects\":null,\"measured\":false,\"retention_days\":null},{\"name\":\"uploads\",\"status\":\"ok\",\"bytes\":null,\"objects\":null,\"expired_bytes\":null,\"expired_objects\":null,\"measured\":false,\"retention_days\":null},{\"name\":\"truckwashdev\",\"status\":\"ok\",\"bytes\":null,\"objects\":null,\"expired_bytes\":null,\"expired_objects\":null,\"measured\":false,\"retention_days\":null}],\"storage\":{\"source_bytes\":0,\"source_objects\":0,\"measured\":false}},\"checked_at\":\"2026-05-19T00:57:17+02:00\"}", + "last_checked_at": "2026-05-18 22:57:17", + "updated_at": "2026-05-18 22:57:17", + "deleted_at": null + }, + { + "id": 9, + "kind": "minio", + "label": "minio-replica-2", + "host": "94.130.142.41", + "port": 9010, + "database_name": null, + "database_index": null, + "username": "twminio38e30e538cf07101a9db00ae", + "password_secret": "twsec:v1:eyJub25jZSI6Ild2eUxvK2ZtdjFNeTQ4bm4iLCJ0YWciOiI5M24rRDM2UnBRaUQvQUkxaDhkSTBBPT0iLCJjaXBoZXJ0ZXh0IjoiM0l1a2dvaEQvS0MyQmZLY0JHVG5wWDlkWWNIUklRK0svUjBodTJ5NEtyTT0ifQ==", + "admin_username": "", + "admin_password_secret": "twsec:v1:eyJub25jZSI6ImRiblNwSEJJWHNXeWtWMXciLCJ0YWciOiI3dTcxZ0FYN1lMclNkWFgwUzNMRHV3PT0iLCJjaXBoZXJ0ZXh0IjoiIn0=", + "replication_username": "", + "replication_password_secret": "twsec:v1:eyJub25jZSI6IkZlQ29vMDY3WVFIb1ZjWDAiLCJ0YWciOiJqN3J5RytlSGs0T2s1Skhyc2gxSW53PT0iLCJjaXBoZXJ0ZXh0IjoiIn0=", + "role": "replica", + "status": "down", + "replication_source_id": 4, + "ssl_mode": "DISABLED", + "options_json": "{\"scheme\":\"http\",\"buckets\":[\"attachments\",\"backups\",\"invoices\",\"pdfs\",\"uploads\",\"truckwashdev\"],\"console_port\":9011,\"space_headroom_percent\":20,\"deployment_provider\":\"coolify\",\"coolify_instance_id\":1,\"endpoint\":\"http://94.130.142.41:9010\",\"coolify_target_id\":2}", + "last_status_json": "{\"status\":\"down\",\"replication_percent\":0,\"lag_seconds\":null,\"blockers\":[\"Class \\\"Aws\\\\S3\\\\S3Client\\\" not found\"],\"raw\":[],\"checked_at\":\"2026-05-19T00:57:20+02:00\"}", + "last_checked_at": "2026-05-18 22:57:21", + "updated_at": "2026-05-18 22:57:21", + "deleted_at": null + }, + { + "id": 13, + "kind": "minio", + "label": "minio-replica-1", + "host": "23.88.23.183", + "port": 9010, + "database_name": null, + "database_index": null, + "username": "twminio1a6370c14ab3a949a07a9e16", + "password_secret": "twsec:v1:eyJub25jZSI6ImFzK0tLZlBRYWlERzU5OWQiLCJ0YWciOiJISmtJZFpwaG5JbktnZkxHZ09DcEVBPT0iLCJjaXBoZXJ0ZXh0IjoiWTkxYm1SZ2JXWEJpdGJpeWtUUTAvMHZjSU14Rzg3a2VrcHpNSDk5UzYxRT0ifQ==", + "admin_username": "", + "admin_password_secret": "twsec:v1:eyJub25jZSI6IjZLRm5ETlZUaDNwUU0xZHQiLCJ0YWciOiJnNnB1QmNHbWliMEpuUVA2VFpGcE9nPT0iLCJjaXBoZXJ0ZXh0IjoiIn0=", + "replication_username": "", + "replication_password_secret": "twsec:v1:eyJub25jZSI6Ik5yOU1SN1gybC9pdGVheXYiLCJ0YWciOiJMbk5vWjZmNjRLMVpRemlleHNKVFZRPT0iLCJjaXBoZXJ0ZXh0IjoiIn0=", + "role": "replica", + "status": "down", + "replication_source_id": 4, + "ssl_mode": "DISABLED", + "options_json": "{\"scheme\":\"http\",\"buckets\":[\"attachments\",\"backups\",\"invoices\",\"pdfs\",\"uploads\",\"truckwashdev\"],\"console_port\":9011,\"replication_transfer_limit\":\"25Mi\",\"space_headroom_percent\":20,\"deployment_provider\":\"coolify\",\"coolify_instance_id\":1,\"endpoint\":\"http://23.88.23.183:9010\",\"coolify_target_id\":6}", + "last_status_json": "{\"status\":\"down\",\"replication_percent\":0,\"lag_seconds\":null,\"blockers\":[\"Class \\\"Aws\\\\S3\\\\S3Client\\\" not found\"],\"raw\":[],\"checked_at\":\"2026-05-19T00:57:20+02:00\"}", + "last_checked_at": "2026-05-18 22:57:21", + "updated_at": "2026-05-18 22:57:21", + "deleted_at": null + } + ] + } + } +} diff --git a/services/nginx/app/tests/Api/EdgeGatewayAgentApiTest.php b/services/nginx/app/tests/Api/EdgeGatewayAgentApiTest.php index 2b11f63a..f56d354a 100644 --- a/services/nginx/app/tests/Api/EdgeGatewayAgentApiTest.php +++ b/services/nginx/app/tests/Api/EdgeGatewayAgentApiTest.php @@ -100,7 +100,10 @@ it('serves installer artifacts and recovers gateway runtime status after fresh h ->assertSuccess(); expect($heartbeatResponse->data()) - ->toHaveKey('status', 'ONLINE'); + ->toHaveKey('status', 'ONLINE') + ->and($heartbeatResponse->data()['broker_url'] ?? null) + ->toBeString() + ->toContain('/edge-broker'); $recoveredDetail = api_client()->get('/edge-gateways/' . $gatewayId, $session['headers']); diff --git a/services/nginx/app/tests/Unit/Coolify/CoolifyManagerTest.php b/services/nginx/app/tests/Unit/Coolify/CoolifyManagerTest.php new file mode 100644 index 00000000..07a17e87 --- /dev/null +++ b/services/nginx/app/tests/Unit/Coolify/CoolifyManagerTest.php @@ -0,0 +1,323 @@ +toBe('https://coolify.example.com/api/v1'); + expect(coolify_api_client::normalizeBaseUrl('https://coolify.example.com/api/v1'))->toBe('https://coolify.example.com/api/v1'); + expect(coolify_api_client::normalizeBaseUrl(' https://coolify.example.com/ '))->toBe('https://coolify.example.com/api/v1'); +}); + +it('parses generated env files for Coolify service env bulk updates', function (): void { + $env = implode("\n", [ + '# generated', + 'MARIADB_ROOT_PASSWORD=root-secret', + 'MARIADB_PASSWORD=app-secret', + '', + 'REDIS_PRIMARY_USERNAME=', + ]); + + expect(coolify_manager::parseEnvFile($env))->toBe([ + 'MARIADB_ROOT_PASSWORD' => 'root-secret', + 'MARIADB_PASSWORD' => 'app-secret', + 'REDIS_PRIMARY_USERNAME' => '', + ]); +}); + +it('prefers public Coolify server hosts over Docker-local addresses', function (): void { + expect(coolify_manager::publicServerHostFromCoolifyServer([ + 'ip' => 'host.docker.internal', + 'public_ip' => '94.130.142.41', + 'name' => 'node3.truckwash.io', + ]))->toBe('94.130.142.41'); + + expect(coolify_manager::publicServerHostFromCoolifyServer([ + 'ip' => 'host.docker.internal', + 'name' => 'node3.truckwash.io', + ]))->toBe('node3.truckwash.io'); + + expect(coolify_manager::publicServerHostFromCoolifyServer([ + 'ip' => 'host.docker.internal', + 'name' => 'node3.truckwash.io', + ], null, false))->toBeNull(); + + expect(coolify_manager::publicServerHostFromCoolifyServer([ + 'ip' => '10.0.0.10', + 'name' => 'Production Server', + ]))->toBe('10.0.0.10'); + + expect(coolify_manager::publicServerHostFromCoolifyServer([ + 'ip' => 'host.docker.internal', + 'name' => 'Production Server', + ]))->toBeNull(); + + expect(coolify_manager::publicDnsServerNameFromCoolifyServer([ + 'ip' => 'host.docker.internal', + 'name' => 'node3.truckwash.io', + ]))->toBe('node3.truckwash.io'); + + expect(coolify_manager::publicDnsServerNameFromCoolifyServer([ + 'ip' => 'host.docker.internal', + 'name' => 'Production Server', + ]))->toBeNull(); +}); + +it('blocks planned downtime operations against active replication primaries', function (): void { + expect(coolify_manager::blocksPrimaryMutation(['role' => 'primary'], 'deploy'))->toBeTrue(); + expect(coolify_manager::blocksPrimaryMutation(['role' => 'primary'], 'restart'))->toBeTrue(); + expect(coolify_manager::blocksPrimaryMutation(['role' => 'replica'], 'restart'))->toBeFalse(); + expect(coolify_manager::blocksPrimaryMutation(['role' => 'primary'], 'failover'))->toBeFalse(); +}); + +it('allows failed Coolify replica targets to be removed after the service disappears', function (): void { + expect(coolify_manager::targetAllowsReplicaRemoval([ + 'role' => 'replica', + 'deployment_status' => 'reconcile_failed', + 'last_reconcile_status' => 'reconcile_failed', + ]))->toBeTrue(); + + expect(coolify_manager::targetAllowsReplicaRemoval([ + 'role' => 'replica', + 'deployment_status' => 'deploying', + 'last_reconcile_json' => json_encode(['message' => 'Coolify API request failed: HTTP 404']), + ]))->toBeTrue(); + + expect(coolify_manager::targetAllowsReplicaRemoval([ + 'role' => 'replica', + 'deployment_status' => 'provisioned', + 'last_reconcile_status' => 'ok', + ]))->toBeFalse(); + + expect(coolify_manager::targetAllowsReplicaRemoval([ + 'role' => 'primary', + 'deployment_status' => 'reconcile_failed', + ]))->toBeFalse(); +}); + +it('retries Coolify maintenance while linked replication provisioning is still incomplete', function (): void { + $method = new ReflectionMethod(coolify_manager::class, 'replicationHostStillNeedsProvisioning'); + $method->setAccessible(true); + + expect($method->invoke(null, [ + 'role' => 'replica', + 'status' => 'provisioning', + 'last_status_json' => json_encode([ + 'status' => 'provisioning', + 'replication_percent' => 99.9, + 'blockers' => ['MinIO replica has not caught up.'], + ]), + ]))->toBeTrue(); + + expect($method->invoke(null, [ + 'role' => 'replica', + 'status' => 'ok', + 'last_status_json' => json_encode([ + 'status' => 'ok', + 'replication_percent' => 100, + 'blockers' => [], + ]), + ]))->toBeFalse(); +}); + +it('plans Hetzner load balancer target and service drift without mutating state', function (): void { + $manager = new coolify_manager(); + $method = new ReflectionMethod(coolify_manager::class, 'planLoadBalancerReconcile'); + $method->setAccessible(true); + + $plan = $method->invoke($manager, [ + 'targets' => [ + ['type' => 'ip', 'ip' => ['ip' => '94.130.142.41']], + ], + 'services' => [ + ['protocol' => 'http', 'listen_port' => 80, 'destination_port' => 80, 'proxyprotocol' => false], + ], + ], [ + ['hostname' => 'node1.truckwash.io', 'target_ip' => '94.130.142.41', 'enabled' => true], + ['hostname' => 'node2.truckwash.io', 'target_ip' => '65.21.214.30', 'enabled' => true], + ]); + + $actionTypes = array_map(static fn(array $action): string => (string)$action['type'], $plan['actions']); + + expect($plan['has_drift'])->toBeTrue() + ->and($plan['missing_targets'])->toContain('65.21.214.30') + ->and($actionTypes)->toContain('add_target') + ->and($actionTypes)->toContain('add_service') + ->and($plan['missing_services'])->toContain([ + 'protocol' => 'tcp', + 'listen_port' => 443, + 'destination_port' => 443, + ]); +}); + +it('does not plan removal of the last Hetzner load balancer target', function (): void { + $manager = new coolify_manager(); + $method = new ReflectionMethod(coolify_manager::class, 'planLoadBalancerReconcile'); + $method->setAccessible(true); + + $plan = $method->invoke($manager, [ + 'targets' => [ + ['type' => 'ip', 'ip' => ['ip' => '94.130.142.41']], + ], + 'services' => [ + ['protocol' => 'http', 'listen_port' => 80, 'destination_port' => 80, 'proxyprotocol' => false], + ['protocol' => 'tcp', 'listen_port' => 443, 'destination_port' => 443, 'proxyprotocol' => false], + ], + ], [ + ['hostname' => 'node1.truckwash.io', 'target_ip' => '94.130.142.41', 'enabled' => false], + ]); + + expect($plan['actions'][0]) + ->toHaveKey('type', 'skip_remove_target') + ->toHaveKey('reason', 'last_reachable_target_guard'); +}); + +it('plans removal only for disabled or deleted Hetzner load balancer targets', function (): void { + $manager = new coolify_manager(); + $method = new ReflectionMethod(coolify_manager::class, 'planLoadBalancerReconcile'); + $method->setAccessible(true); + + $plan = $method->invoke($manager, [ + 'targets' => [ + ['type' => 'ip', 'ip' => ['ip' => '94.130.142.41']], + ['type' => 'ip', 'ip' => ['ip' => '65.21.214.30']], + ], + 'services' => [ + ['protocol' => 'http', 'listen_port' => 80, 'destination_port' => 80, 'proxyprotocol' => false], + ['protocol' => 'tcp', 'listen_port' => 443, 'destination_port' => 443, 'proxyprotocol' => false], + ], + ], [ + ['hostname' => 'node1.truckwash.io', 'target_ip' => '94.130.142.41', 'enabled' => true], + ['hostname' => 'node2.truckwash.io', 'target_ip' => '65.21.214.30', 'enabled' => true, 'deleted_at' => '2026-05-19 10:00:00'], + ]); + + expect($plan['actions']) + ->toHaveCount(1) + ->and($plan['actions'][0]) + ->toHaveKey('type', 'remove_target') + ->toHaveKey('target_ip', '65.21.214.30'); +}); + +it('defines Coolify schema, route permissions, and replication integration hooks', function (): void { + $schema = file_get_contents(app_path('classes/coolify_schema_bootstrap.php')); + $manager = file_get_contents(app_path('classes/coolify_manager.php')); + $route = file_get_contents(app_path('routes/superuserCoolifyRoute.php')); + $replication = file_get_contents(app_path('classes/replication_manager.php')); + $status = file_get_contents(app_path('classes/superuser_system_status_service.php')); + $cron = file_get_contents(app_path('cron/Cron.php')); + $openapi = file_get_contents(app_path('openapi.yaml')); + $coolifyConfig = file_get_contents(app_path('modules/coolify/coolify_c.php')); + $tokenConfig = file_get_contents(app_path('modules/coolify/config/coolify_hetzner_cloud_api_token_c.php')); + + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS coolify_instances'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS coolify_targets'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS coolify_operations'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS coolify_audit_logs'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS coolify_instance_gateways'); + expect($schema)->toContain("ensureModuleConfigDefault('Coolify', 'lb_automation_enabled'"); + expect($schema)->toContain("ensureModuleConfigDefault('Coolify', 'lb_automation_mode'"); + expect($schema)->toContain("ensureModuleConfigDefault('Coolify', 'hetzner_load_balancer_id'"); + expect($schema)->toContain("ensureModuleConfigDefault('Coolify', 'hetzner_cloud_api_token'"); + expect($schema)->toContain("ensureModuleConfigDefault('Coolify', 'public_gateway_host', 'api-v2.truckwash.io'"); + expect($schema)->toContain('94.130.142.41'); + expect($schema)->toContain('65.21.214.30'); + expect($schema)->toContain('23.88.23.183'); + expect($schema)->toContain("ensureModuleConfigDefault('Coolify', 'enabled'"); + + expect($route)->toContain('/superuser/coolify'); + expect($route)->toContain('/superuser/coolify/load-balancer'); + expect($route)->toContain('/superuser/coolify/load-balancer/reconcile'); + expect($route)->toContain('/superuser/coolify/gateways'); + expect($route)->toContain('/superuser/coolify/gateways/{id}/test'); + expect($route)->toContain('/superuser/coolify/instances/{id}/test'); + expect($route)->toContain('/superuser/coolify/instances/{id}/placement'); + expect($route)->toContain('/superuser/coolify/targets/{id}/reconcile'); + expect($route)->toContain('/superuser/coolify/targets/{id}/deploy'); + expect($route)->toContain('/superuser/coolify/targets/{id}/restart'); + expect($route)->toContain('/superuser/coolify/targets/{id}/failover'); + expect($route)->toContain("requirePermission('superuser_coolify_view')"); + expect($route)->toContain("requirePermission('superuser_coolify_manage')"); + expect($route)->toContain("requirePermission('superuser_coolify_reconcile')"); + expect($route)->toContain("requirePermission('superuser_coolify_failover')"); + + expect($manager)->toContain("Coolify-managed targets must be deployed as replicas first"); + expect($manager)->toContain('ensureFailoverEnabled($kind)'); + expect($manager)->toContain("deployment_provider' => 'coolify'"); + expect($manager)->toContain('discoverInstancePlacement'); + expect($manager)->toContain('applyCoolifyDeploymentDefaults($input, $instance)'); + expect($manager)->toContain('resolveCoolifyServerHost'); + expect($manager)->toContain('publicServerHostFromCoolifyServer'); + expect($manager)->toContain('applyCoolifyPortDefaults'); + expect($manager)->toContain('syncReplicationHostPortsForTarget'); + expect($manager)->toContain('usedPublicPortsForCoolifyServer'); + expect($manager)->toContain('nextAvailablePublicPorts'); + expect($manager)->toContain('syncReplicationHostEndpointForTarget'); + expect($manager)->toContain('knownPublicHostForCoolifyServer'); + expect($manager)->toContain('publicDnsServerNameFromCoolifyServer'); + expect($manager)->toContain('resolvedPublicDnsServerHostFromCoolifyServer'); + expect($manager)->toContain('recordCreatedResource'); + expect($manager)->toContain("'start_requested'"); + expect($manager)->toContain('primaryCredentials'); + expect($manager)->toContain('primary_admin_password'); + expect($manager)->toContain('replication_transfer_limit'); + expect($manager)->toContain('startOrRestartService'); + expect($manager)->toContain('already running'); + expect($manager)->toContain('restart_requested'); + expect($manager)->toContain('deferredProvisionResult'); + expect($manager)->toContain('isTransientProvisionBlock'); + expect($manager)->toContain('provision_deferred'); + expect($manager)->toContain('shouldRetryProvisioning'); + expect($manager)->toContain('shouldRetryProvisioning($target, $host)'); + expect($manager)->toContain('hasRunningReplicationProvisionOperation'); + expect($manager)->toContain('replicationHostStillNeedsProvisioning'); + expect($manager)->toContain('syncDeploymentStateForReplicationHost'); + expect($manager)->toContain('syncLabelForReplicationHost'); + expect($manager)->toContain('syncTargetsForReplicationHost'); + expect($manager)->toContain('targetAllowsReplicaRemoval'); + expect($manager)->toContain('markTargetsRemovedForReplicationHost'); + expect($manager)->toContain('replicationHostIsReady'); + expect($manager)->toContain("'provisioned'"); + expect($manager)->not->toContain('is_container_label_escape_enabled'); + expect($manager)->not->toContain("\$payload['type'] = 'docker-compose';"); + expect($manager)->toContain('encodedDockerCompose'); + expect($manager)->toContain('base64_encode'); + expect($manager)->toContain('if (!$update)'); + expect($manager)->toContain("'project_uuid' => \$target['project_uuid']"); + expect($manager)->toContain('/api/v1/services'); + expect($manager)->toContain('/envs/bulk'); + expect($manager)->toContain('loadBalancerSummary'); + expect($manager)->toContain('reconcileLoadBalancer'); + expect($manager)->toContain('target_already_defined'); + expect($manager)->toContain('REQUIRED_LOAD_BALANCER_SERVICES'); + expect($manager)->toContain('skip_remove_target'); + + $client = file_get_contents(app_path('classes/coolify_api_client.php')); + expect($client)->toContain("request('GET', '/health', null, false)"); + expect($client)->toContain('CURL_HTTP_VERSION_1_1'); + expect($client)->toContain('validationErrorSummary'); + + expect($replication)->toContain('deployment_provider'); + expect($replication)->toContain('coolify_manager::deploymentMetadataForReplicationHost'); + expect($replication)->toContain('coolify_manager::syncDeploymentStateForReplicationHost'); + expect($replication)->toContain('databaseEngineKnown'); + expect($status)->toContain("'key' => 'coolify'"); + expect($status)->toContain('probeCoolifyModule'); + expect($cron)->toContain('CoolifyAvailabilityMonitorCron'); + expect($cron)->toContain('CoolifyLoadBalancerReconcileCron'); + expect($coolifyConfig)->toContain('[redacted]'); + expect($coolifyConfig)->toContain('secret_set'); + expect($tokenConfig)->toContain('replication_secret_box::encrypt'); + expect($openapi)->toContain('/superuser/coolify:'); + expect($openapi)->toContain('operationId: getSuperuserCoolifyLoadBalancer'); + expect($openapi)->toContain('operationId: reconcileSuperuserCoolifyLoadBalancer'); + expect($openapi)->toContain('operationId: listSuperuserCoolifyGateways'); + expect($openapi)->toContain('operationId: testSuperuserCoolifyGateway'); + expect($openapi)->toContain('operationId: discoverSuperuserCoolifyInstancePlacement'); + expect($openapi)->toContain('operationId: createSuperuserCoolifyTarget'); + expect($openapi)->toContain('SuperuserCoolifyTarget'); + expect($openapi)->toContain('SuperuserCoolifyLoadBalancer'); + expect($openapi)->toContain('SuperuserCoolifyGateway'); +}); diff --git a/services/nginx/app/tests/Unit/ErrorReports/ErrorReportTest.php b/services/nginx/app/tests/Unit/ErrorReports/ErrorReportTest.php new file mode 100644 index 00000000..b4fa2684 --- /dev/null +++ b/services/nginx/app/tests/Unit/ErrorReports/ErrorReportTest.php @@ -0,0 +1,79 @@ + 'Bearer secret-token', + 'request' => [ + 'headers' => [ + 'X-Api-Key' => 'hidden', + 'Accept' => 'application/json', + ], + 'body' => [ + 'password' => 'not stored', + 'safe' => 'visible', + ], + ], + ]; + + expect(error_report_service::redactPayload($payload))->toBe([ + 'Authorization' => '[redacted]', + 'request' => [ + 'headers' => [ + 'X-Api-Key' => '[redacted]', + 'Accept' => 'application/json', + ], + 'body' => [ + 'password' => '[redacted]', + 'safe' => 'visible', + ], + ], + ]); +}); + +it('validates supported screenshot data uris', function (): void { + $decoded = error_report_service::decodeScreenshotDataUri('data:image/png;base64,' . base64_encode('png-bytes')); + + expect($decoded['mime_type'])->toBe('image/png'); + expect($decoded['contents'])->toBe('png-bytes'); + expect($decoded['size_bytes'])->toBe(strlen('png-bytes')); + + expect(fn() => error_report_service::decodeScreenshotDataUri('data:text/plain;base64,' . base64_encode('nope'))) + ->toThrow(RuntimeException::class, 'Screenshot must be a PNG, JPEG, or WebP data URI.'); +}); + +it('defines error report schema, routes, permissions, storage, and OpenAPI docs', function (): void { + $schema = file_get_contents(app_path('classes/error_report_schema_bootstrap.php')); + $service = file_get_contents(app_path('classes/error_report_service.php')); + $store = file_get_contents(app_path('classes/error_report_store.php')); + $route = file_get_contents(app_path('routes/errorReportRoute.php')); + $openapi = file_get_contents(app_path('openapi.yaml')); + + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS error_reports'); + expect($schema)->toContain('request_errors_json'); + expect($schema)->toContain('vue_errors_json'); + expect($schema)->toContain('data_collection_accepted_at'); + expect($schema)->toContain('resolved_by_user_id'); + + expect($service)->toContain('createFromCurrentPrincipal'); + expect($service)->toContain('decodeScreenshotDataUri'); + expect($service)->toContain('data_collection_accepted'); + expect($service)->toContain('request_error_count'); + expect($service)->toContain('vue_error_count'); + expect($store)->toContain("error-reports/%s/%s.%s"); + + expect($route)->toContain('/error-reports'); + expect($route)->toContain('/superuser/error-reports'); + expect($route)->toContain('/superuser/error-reports/{id}/status'); + expect($route)->toContain("requirePermission('superuser_error_reports_view')"); + expect($route)->toContain("requirePermission('superuser_error_reports_resolve')"); + + expect($openapi)->toContain('/error-reports:'); + expect($openapi)->toContain('ErrorReportSubmissionRequest'); + expect($openapi)->toContain('ErrorReportStatusUpdateRequest'); +}); diff --git a/services/nginx/app/tests/Unit/Http/ResponseRequestParametersTest.php b/services/nginx/app/tests/Unit/Http/ResponseRequestParametersTest.php new file mode 100644 index 00000000..969a6d3d --- /dev/null +++ b/services/nginx/app/tests/Unit/Http/ResponseRequestParametersTest.php @@ -0,0 +1,71 @@ +body; + } + }; +} + +it('reads JSON payloads for DELETE request parameter arrays', function (): void { + $previousMethod = $_SERVER['REQUEST_METHOD'] ?? null; + $previousGet = $_GET; + + try { + $_SERVER['REQUEST_METHOD'] = 'DELETE'; + $_GET = []; + + $response = response_request_parameters_response_with_body(json_encode([ + 'confirm' => 'delete-coolify-target-3', + 'delete_resource' => false, + ])); + + expect($response->getAllRequestParameters())->toBe([ + 'confirm' => 'delete-coolify-target-3', + 'delete_resource' => false, + ]); + expect($response->getRequestParameter('confirm'))->toBe('delete-coolify-target-3'); + expect($response->isRequestParameterSet('delete_resource'))->toBeTrue(); + } finally { + $_GET = $previousGet; + if ($previousMethod === null) { + unset($_SERVER['REQUEST_METHOD']); + } else { + $_SERVER['REQUEST_METHOD'] = $previousMethod; + } + } +}); + +it('keeps DELETE query parameters when no JSON body is present', function (): void { + $previousMethod = $_SERVER['REQUEST_METHOD'] ?? null; + $previousGet = $_GET; + + try { + $_SERVER['REQUEST_METHOD'] = 'DELETE'; + $_GET = ['confirm' => 'delete-coolify-target-5']; + + $response = response_request_parameters_response_with_body(''); + + expect($response->getAllRequestParameters())->toBe([ + 'confirm' => 'delete-coolify-target-5', + ]); + } finally { + $_GET = $previousGet; + if ($previousMethod === null) { + unset($_SERVER['REQUEST_METHOD']); + } else { + $_SERVER['REQUEST_METHOD'] = $previousMethod; + } + } +}); diff --git a/services/nginx/app/tests/Unit/Infrastructure/CorsReleaseHeadersTest.php b/services/nginx/app/tests/Unit/Infrastructure/CorsReleaseHeadersTest.php new file mode 100644 index 00000000..b3acb342 --- /dev/null +++ b/services/nginx/app/tests/Unit/Infrastructure/CorsReleaseHeadersTest.php @@ -0,0 +1,23 @@ +toBeTrue(); + $content = (string)file_get_contents($file); + + foreach ($requiredHeaders as $header) { + expect($content)->toContain($header); + } + } +}); diff --git a/services/nginx/app/tests/Unit/Orders/OrderBookingsCompletionDedupTest.php b/services/nginx/app/tests/Unit/Orders/OrderBookingsCompletionDedupTest.php index 810f6172..737dbca4 100644 --- a/services/nginx/app/tests/Unit/Orders/OrderBookingsCompletionDedupTest.php +++ b/services/nginx/app/tests/Unit/Orders/OrderBookingsCompletionDedupTest.php @@ -20,11 +20,17 @@ if (!class_exists('OrderBookingsCompletionOrderDouble')) { } public bool $washCertificateAttached = false; + public bool $containsWashCertificate = false; public function objectChanged(): void { } + public function containsWashCertificateItem(): bool + { + return $this->containsWashCertificate; + } + public function hasWashCertificateAttached(): bool { return $this->washCertificateAttached; @@ -98,6 +104,20 @@ it('attaches and emails a wash certificate when a booking is already linked to a expect($booking->sendCalls)->toBe(1); }); +it('uses the linked pos order wash certificate item added during mobile completion', function (): void { + $order = new OrderBookingsCompletionOrderDouble(); + $order->containsWashCertificate = true; + $booking = new OrderBookingsCompletionDouble($order); + $booking->order_id->set(321); + $booking->containsWashCertificate = false; + + $booking->completeBooking(77, 'MOBILE-SEAL'); + + expect($order->getSafetySealValue())->toBe('MOBILE-SEAL'); + expect($booking->attachCalls)->toBe(1); + expect($booking->sendCalls)->toBe(1); +}); + it('does not create or email a duplicate wash certificate when a linked pos order already has one', function (): void { $order = new OrderBookingsCompletionOrderDouble(); $order->washCertificateAttached = true; diff --git a/services/nginx/app/tests/Unit/ReleaseManager/ReleaseManagerTest.php b/services/nginx/app/tests/Unit/ReleaseManager/ReleaseManagerTest.php new file mode 100644 index 00000000..803ccbd4 --- /dev/null +++ b/services/nginx/app/tests/Unit/ReleaseManager/ReleaseManagerTest.php @@ -0,0 +1,243 @@ + 'Bearer secret-token', + 'nested' => [ + 'api_key' => 'key-value', + 'safe' => 'visible', + 'items' => [ + ['password' => 'hidden', 'status' => 500], + ], + ], + ]; + + expect(release_manager::redactPayload($payload))->toBe([ + 'Authorization' => '[redacted]', + 'nested' => [ + 'api_key' => '[redacted]', + 'safe' => 'visible', + 'items' => [ + ['password' => '[redacted]', 'status' => 500], + ], + ], + ]); +}); + +it('verifies GitHub sha256 webhook signatures', function (): void { + $secret = 'release-webhook-secret'; + $payload = '{"ref":"refs/heads/main"}'; + $signature = 'sha256=' . hash_hmac('sha256', $payload, $secret); + + expect(release_manager::verifyGithubSignature($secret, $payload, $signature))->toBeTrue(); + expect(release_manager::verifyGithubSignature($secret, $payload, 'sha256=bad'))->toBeFalse(); + expect(release_manager::verifyGithubSignature('', $payload, $signature))->toBeFalse(); +}); + +it('normalizes GitHub repository identifiers for private repository access checks', function (): void { + expect(release_manager::normalizeGithubRepositoryName('truckwash/backend-php'))->toBe('truckwash/backend-php'); + expect(release_manager::normalizeGithubRepositoryName('https://github.com/truckwash/front-end-vue.git'))->toBe('truckwash/front-end-vue'); + expect(release_manager::normalizeGithubRepositoryName('git@github.com:truckwash/backend-php.git'))->toBe('truckwash/backend-php'); + expect(release_manager::normalizeGithubRepositoryName('not a repository'))->toBe(''); +}); + +it('summarizes failed deployments and blocks promotion until a deployment succeeds', function (): void { + $summary = release_manager::deploymentFailureSummary( + new RuntimeException('Coolify API request failed: HTTP 404'), + [ + 'app' => 'api', + 'repository' => 'copenhagentruckwash/api', + 'branch' => 'master', + 'commit_sha' => 'ab31cd6dbb288606a24fcdbcb7bb7d58d843e4f3', + 'coolify_service_uuid' => 'api-service', + ] + ); + + expect($summary['category'])->toBe('coolify_target'); + expect($summary['stage'])->toBe('provider_target'); + expect($summary['promotion_blocked'])->toBeTrue(); + expect($summary['evidence']['commit_sha'])->toBe('ab31cd6dbb288606a24fcdbcb7bb7d58d843e4f3'); + expect(release_manager::deploymentCanBePromoted('deployed'))->toBeTrue(); + expect(release_manager::deploymentCanBePromoted('failed'))->toBeFalse(); + + $reason = release_manager::deploymentPromotionBlockedReason([ + 'status' => 'failed', + 'result_json' => json_encode(['failure_summary' => $summary]), + ]); + + expect($reason)->toContain('Deployment failed'); + expect($reason)->toContain('HTTP 404'); +}); + +it('uses the selected Coolify project and resolves server UUID from the instance default', function (): void { + $manager = new release_manager(); + $method = new ReflectionMethod(release_manager::class, 'releaseCoolifyServicePayload'); + $method->setAccessible(true); + + $payload = $method->invoke($manager, [ + 'channel_slug' => 'internal', + 'app' => 'api', + 'repository' => 'copenhagentruckwash/api', + 'branch' => 'master', + ], [ + 'coolify_project_uuid' => 'project-selected', + 'coolify_deploy_now' => true, + ], [ + 'default_project_uuid' => 'project-default', + 'default_environment_name' => 'production', + 'default_server_uuid' => 'server-default', + ]); + + expect($payload['project_uuid'])->toBe('project-selected'); + expect($payload['server_uuid'])->toBe('server-default'); + expect($payload)->not->toHaveKey('coolify_server_uuid'); +}); + +it('supports isolated stack mode and names new Coolify services explicitly', function (): void { + $manager = new release_manager(); + + $normalizeMode = new ReflectionMethod(release_manager::class, 'normalizeServiceSetMode'); + $normalizeMode->setAccessible(true); + expect($normalizeMode->invoke($manager, ' isolated_stack '))->toBe('isolated_stack'); + + $payloadMethod = new ReflectionMethod(release_manager::class, 'releaseCoolifyServicePayload'); + $payloadMethod->setAccessible(true); + $payload = $payloadMethod->invoke($manager, [ + 'channel_slug' => 'internal', + 'app' => 'frontend', + 'repository' => 'copenhagentruckwash/front-end-vue', + 'branch' => 'main', + ], [ + 'coolify_service_name' => 'release-internal-safe-stack-frontend', + 'coolify_project_uuid' => 'project-internal', + 'coolify_deploy_now' => true, + ], [ + 'default_environment_name' => 'production', + 'default_server_uuid' => 'server-default', + ]); + + expect($payload['name'])->toBe('release-internal-safe-stack-frontend'); + expect($payload['project_uuid'])->toBe('project-internal'); +}); + +it('defines release manager schema, routes, permissions, and system-status integration hooks', function (): void { + $schema = file_get_contents(app_path('classes/release_manager_schema_bootstrap.php')); + $manager = file_get_contents(app_path('classes/release_manager.php')); + $route = file_get_contents(app_path('routes/releaseManagerRoute.php')); + $auth = file_get_contents(app_path('routes/authRoute.php')); + $response = file_get_contents(app_path('classes/response.php')); + $status = file_get_contents(app_path('classes/superuser_system_status_service.php')); + $index = file_get_contents(app_path('index.php')); + + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_channels'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_versions'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_channel_versions'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_assignments'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_deployment_targets'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_service_sets'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_deployments'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_bundles'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_replay_targets'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_timeline_sessions'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_timeline_events'); + expect($schema)->toContain('CREATE TABLE IF NOT EXISTS release_module_health_snapshots'); + expect($schema)->toContain("ensureColumn('release_channel_versions', 'service_set_id'"); + expect($schema)->toContain("ensureColumn('release_channel_versions', 'bundle_id'"); + expect($schema)->toContain("ensureColumn('release_deployments', 'deployment_kind'"); + expect($schema)->toContain("ensureModuleConfigDefault('ReleaseManager', 'enabled'"); + expect($schema)->toContain("ensureModuleConfigDefault('ReleaseManager', 'github_token'"); + expect($schema)->toContain("ensureModuleConfigDefault('ReleaseManager', 'github_api_url'"); + expect($schema)->toContain("['stable', 'Stable'"); + expect($schema)->toContain("['canary', 'Canary'"); + expect($schema)->toContain("['beta', 'Beta'"); + expect($schema)->toContain("['internal', 'Internal'"); + + expect($route)->toContain('/release/bootstrap'); + expect($route)->toContain('/release/runtime'); + expect($route)->toContain('/release/timeline/events'); + expect($route)->toContain('/release/github/webhook'); + expect($route)->toContain('/superuser/releases/config'); + expect($route)->toContain('/superuser/releases/github/repositories'); + expect($route)->toContain('/superuser/releases/github/branches'); + expect($route)->toContain('/superuser/releases/github/commits'); + expect($route)->toContain('/superuser/releases/github/test'); + expect($route)->toContain('/superuser/releases/channels'); + expect($route)->toContain('/superuser/releases/assignments'); + expect($route)->toContain('/superuser/releases/service-sets'); + expect($route)->toContain('/superuser/releases/bundles'); + expect($route)->toContain('/superuser/releases/bundles/{id}/deploy'); + expect($route)->toContain('/superuser/releases/bundles/{id}/promote'); + expect($route)->toContain('/superuser/releases/deployments'); + expect($route)->toContain('/superuser/releases/replay-targets'); + expect($route)->toContain('/superuser/releases/timeline'); + expect($route)->toContain("requirePermission('superuser_release_manager_view')"); + expect($route)->toContain("requirePermission('superuser_release_manager_manage')"); + expect($route)->toContain("requirePermission('superuser_release_manager_deploy')"); + expect($route)->toContain("requirePermission('superuser_release_manager_rollback')"); + expect($route)->toContain("requirePermission('superuser_release_manager_replay')"); + + expect($manager)->toContain('verifyGithubSignature'); + expect($manager)->toContain('normalizeGithubRepositoryName'); + expect($manager)->toContain('releaseConfig'); + expect($manager)->toContain('updateReleaseConfig'); + expect($manager)->toContain('github_token_variable'); + expect($manager)->toContain('github_token_env_variable'); + expect($manager)->toContain('listGithubRepositories'); + expect($manager)->toContain('listGithubBranches'); + expect($manager)->toContain('listGithubCommits'); + expect($manager)->toContain('testGithubRepositoryAccess'); + expect($manager)->toContain('githubRepositoryAccess'); + expect($manager)->toContain('github_token'); + expect($manager)->toContain('commit_mode'); + expect($manager)->toContain('restartCoolifyService'); + expect($manager)->toContain('deployCoolifyReleaseTarget'); + expect($manager)->toContain('listServiceSets'); + expect($manager)->toContain('createServiceSet'); + expect($manager)->toContain('createBundle'); + expect($manager)->toContain('deployBundle'); + expect($manager)->toContain('promoteBundle'); + expect($manager)->toContain('clone_replica_from_source'); + expect($manager)->toContain('register_isolated_empty_service'); + expect($manager)->toContain('isolated_stack'); + expect($manager)->toContain('create_isolated_empty_stack_service'); + expect($manager)->toContain('assertIsolatedStackTarget'); + expect($manager)->toContain('must not point at an existing Coolify service'); + expect($manager)->toContain("'data_promotion' => false"); + expect($manager)->toContain("'replica_failover' => false"); + expect($manager)->toContain('deploymentCanBePromoted'); + expect($manager)->toContain('deploymentFailureSummary'); + expect($manager)->toContain('coolifyProjectSuggestions'); + expect($manager)->toContain('releaseCoolifyServerUuid'); + expect($manager)->toContain('coolify_project_uuid'); + expect($manager)->toContain('releaseCoolifyServicePayload'); + expect($manager)->toContain('release_deployment_targets'); + expect($manager)->toContain('resolveChannel'); + expect($manager)->toContain('capturePolicyFor'); + expect($manager)->toContain('channelAvailability'); + expect($manager)->toContain("'availability' =>"); + expect($manager)->toContain('redactPayload'); + expect($manager)->toContain('moduleKeys'); + expect($manager)->toContain('releaseSuggestions'); + expect($manager)->toContain('load_balancer_domains'); + expect($manager)->toContain('appendDomainSuggestion'); + expect($manager)->toContain('Coolify SSL requires a DNS domain routed to the load balancer.'); + expect($manager)->toContain('coolify_services'); + expect($manager)->toContain('coolify_enable_ssl'); + expect($manager)->toContain('createService'); + expect($manager)->toContain('updateService'); + expect($manager)->toContain('channel_presets'); + expect($manager)->toContain('target_presets'); + + expect($auth)->toContain("'release' => (new release_manager())->runtimeForPayload"); + expect($response)->toContain('recordBackendFailure'); + expect($status)->toContain("'key' => 'releasemanager'"); + expect($status)->toContain('probeReleaseManagerModule'); + expect($index)->toContain('release_manager::initializeRequestContext'); + expect($index)->toContain('X-Release-Trace'); + expect($manager)->not->toContain('X-Release-Channel'); +}); diff --git a/services/nginx/app/tests/Unit/Replication/ReplicaFailoverManagerTest.php b/services/nginx/app/tests/Unit/Replication/ReplicaFailoverManagerTest.php new file mode 100644 index 00000000..8f07203f --- /dev/null +++ b/services/nginx/app/tests/Unit/Replication/ReplicaFailoverManagerTest.php @@ -0,0 +1,236 @@ + $id, + 'kind' => $kind, + 'label' => $kind . ' replica ' . $id, + 'host' => $kind . '-replica-' . $id, + 'port' => match ($kind) { + 'database' => 3306, + 'redis' => 6379, + 'minio' => 9000, + default => 1, + }, + 'database_name' => $kind === 'database' ? 'truckwash' : null, + 'database_index' => $kind === 'redis' ? 0 : null, + 'username' => $kind === 'minio' ? 'access-key' : 'app', + 'password_secret' => 'secret', + 'admin_username' => '', + 'admin_password_secret' => '', + 'role' => 'replica', + 'status' => 'ok', + 'options_json' => $kind === 'minio' + ? json_encode(['endpoint' => 'http://minio-replica-' . $id . ':9000', 'buckets' => ['attachments']]) + : null, + 'last_status_json' => json_encode([ + 'status' => 'ok', + 'replication_percent' => 100, + 'blockers' => [], + 'checked_at' => $checkedAt, + ]), + 'last_checked_at' => $checkedAt, + 'deleted_at' => null, + ]; + + return array_replace($base, $overrides); +} + +it('normalizes failover config defaults and per-kind enablement', function (): void { + $defaults = replica_failover_manager::normalizeConfig([]); + + expect($defaults)->toMatchArray([ + 'enabled' => false, + 'database_enabled' => false, + 'redis_enabled' => false, + 'minio_enabled' => false, + 'max_status_age_seconds' => 90, + ]); + + $config = replica_failover_manager::normalizeConfig([ + 'enabled' => 'true', + 'database_enabled' => '1', + 'redis_enabled' => false, + 'minio_enabled' => 'yes', + 'max_status_age_seconds' => '120', + ]); + + expect(replica_failover_manager::kindEnabled($config, 'database'))->toBeTrue(); + expect(replica_failover_manager::kindEnabled($config, 'redis'))->toBeFalse(); + expect(replica_failover_manager::kindEnabled($config, 'minio'))->toBeTrue(); + expect($config['max_status_age_seconds'])->toBe(120); +}); + +it('requires strict fresh 100 percent replica status for candidates', function (): void { + $now = time(); + $fresh = failover_test_host('database', 2, date('c', $now - 30)); + $stale = failover_test_host('database', 3, date('c', $now - 120)); + $notCaughtUp = failover_test_host('database', 4, date('c', $now - 10), [ + 'last_status_json' => json_encode([ + 'status' => 'ok', + 'replication_percent' => 99.99, + 'blockers' => [], + 'checked_at' => date('c', $now - 10), + ]), + ]); + $blocked = failover_test_host('database', 5, date('c', $now - 10), [ + 'last_status_json' => json_encode([ + 'status' => 'degraded', + 'replication_percent' => 100, + 'blockers' => ['lagging'], + 'checked_at' => date('c', $now - 10), + ]), + ]); + + expect(replica_failover_manager::snapshotHostIsStrictlyFresh($fresh, 90, $now))->toBeTrue(); + expect(replica_failover_manager::snapshotHostIsStrictlyFresh($stale, 90, $now))->toBeFalse(); + expect(replica_failover_manager::snapshotHostIsStrictlyFresh($notCaughtUp, 90, $now))->toBeFalse(); + expect(replica_failover_manager::snapshotHostIsStrictlyFresh($blocked, 90, $now))->toBeFalse(); +}); + +it('selects the freshest eligible replica for failover', function (): void { + $now = time(); + $older = failover_test_host('redis', 2, date('c', $now - 40)); + $newer = failover_test_host('redis', 3, date('c', $now - 10)); + $wrongKind = failover_test_host('minio', 4, date('c', $now - 5)); + + $candidate = replica_failover_manager::snapshotFailoverCandidate([$older, $newer, $wrongKind], 'redis', 90, $now); + + expect($candidate['id'])->toBe(3); +}); + +it('promotes enabled startup dependencies from snapshot in dependency order', function (): void { + $path = tempnam(sys_get_temp_dir(), 'failover-snapshot-'); + $events = []; + $now = time(); + + replication_bootstrap_config::writeSnapshot([ + 'active' => [ + 'database' => ['host' => 'db-primary', 'database' => 'truckwash', 'user' => 'app', 'password_secret' => 'secret'], + 'redis' => ['host' => 'redis-primary', 'database' => 0, 'user' => '', 'password_secret' => 'secret'], + 'minio' => ['endpoint' => 'http://minio-primary:9000', 'access_key' => 'access-key', 'secret_key_secret' => 'secret'], + ], + 'failover' => [ + 'config' => [ + 'enabled' => true, + 'database_enabled' => true, + 'redis_enabled' => true, + 'minio_enabled' => true, + 'max_status_age_seconds' => 90, + ], + 'hosts' => [ + 'database' => [failover_test_host('database', 11, date('c', $now - 10))], + 'redis' => [failover_test_host('redis', 12, date('c', $now - 10))], + 'minio' => [failover_test_host('minio', 13, date('c', $now - 10))], + ], + ], + ], $path); + + try { + $result = replica_failover_manager::applyStartupFailoverFromSnapshot($path, [ + 'primary_down' => function (string $kind) use (&$events): bool { + $events[] = 'primary:' . $kind; + return true; + }, + 'candidate_reachable' => function (string $kind) use (&$events): bool { + $events[] = 'reachable:' . $kind; + return true; + }, + 'promote_candidate' => function (string $kind) use (&$events): void { + $events[] = 'promote:' . $kind; + }, + ]); + + $snapshot = replication_bootstrap_config::loadSnapshot($path); + + expect($result['changed'])->toBeTrue(); + expect($events)->toBe([ + 'primary:database', + 'reachable:database', + 'promote:database', + 'primary:redis', + 'reachable:redis', + 'promote:redis', + 'primary:minio', + 'reachable:minio', + 'promote:minio', + ]); + expect($snapshot['active']['database']['id'])->toBe(11); + expect($snapshot['active']['redis']['id'])->toBe(12); + expect($snapshot['active']['minio']['id'])->toBe(13); + expect($snapshot['pending_failovers'])->toHaveCount(3); + } finally { + @unlink($path); + } +}); + +it('does not promote a disabled dependency during startup failover', function (): void { + $path = tempnam(sys_get_temp_dir(), 'failover-snapshot-'); + $events = []; + + replication_bootstrap_config::writeSnapshot([ + 'active' => [ + 'redis' => ['host' => 'redis-primary', 'database' => 0, 'user' => '', 'password_secret' => 'secret'], + ], + 'failover' => [ + 'config' => [ + 'enabled' => true, + 'redis_enabled' => false, + 'max_status_age_seconds' => 90, + ], + 'hosts' => [ + 'redis' => [failover_test_host('redis', 12, date('c'))], + ], + ], + ], $path); + + try { + $result = replica_failover_manager::applyStartupFailoverFromSnapshot($path, [ + 'primary_down' => function (string $kind) use (&$events): bool { + $events[] = $kind; + return true; + }, + ]); + + expect($result['changed'])->toBeFalse(); + expect($result['results']['redis']['reason'])->toBe('disabled'); + expect($events)->toBe([]); + } finally { + @unlink($path); + } +}); + +it('wires the failover module config endpoint and promotion paths', function (): void { + $route = file_get_contents(app_path('routes/moduleConfigRoute.php')); + $module = file_get_contents(app_path('modules/failover/failover_c.php')); + $enabledConfig = file_get_contents(app_path('modules/failover/config/failover_enabled_c.php')); + $manager = file_get_contents(app_path('classes/replication_manager.php')); + $startup = file_get_contents(app_path('classes/replica_failover_manager.php')); + + expect($route)->toContain("'/failover/config'"); + expect($route)->toContain('modules_failover_config'); + expect($enabledConfig)->toContain("'enabled'"); + expect($module)->toContain('failover_database_enabled_c::class'); + expect($module)->toContain('failover_redis_enabled_c::class'); + expect($module)->toContain('failover_minio_enabled_c::class'); + expect($module)->toContain('failover_max_status_age_seconds_c::class'); + expect($manager)->toContain('promoteDatabaseHostForFailover'); + expect($manager)->toContain('promoteRedisHostForFailover'); + expect($manager)->toContain('promoteMinioHostForFailover'); + expect($manager)->toContain("['REPLICAOF', 'NO', 'ONE']"); + expect($manager)->toContain('runAutomaticFailoverMonitor'); + expect($startup)->toContain('replication_bootstrap_config::loadSnapshot($path)'); + expect($startup)->not->toContain('module_config'); +}); diff --git a/services/nginx/app/tests/Unit/Replication/ReplicationManagerStatusTest.php b/services/nginx/app/tests/Unit/Replication/ReplicationManagerStatusTest.php index 04e05eac..14f986c4 100644 --- a/services/nginx/app/tests/Unit/Replication/ReplicationManagerStatusTest.php +++ b/services/nginx/app/tests/Unit/Replication/ReplicationManagerStatusTest.php @@ -20,6 +20,23 @@ it('computes Redis offset percentages safely', function (): void { expect(replication_manager::redisOffsetPercent(1000, 750))->toBe(75.0); expect(replication_manager::redisOffsetPercent(0, 0))->toBe(100.0); expect(replication_manager::redisOffsetPercent(1000, 1250))->toBe(100.0); + expect(replication_manager::redisReplicationPercentFromInfo( + ['master_repl_offset' => 1000], + ['slave_repl_offset' => 750] + ))->toBe(75.0); + expect(replication_manager::redisReplicationPercentFromInfo( + ['master_repl_offset' => 1000], + ['master_sync_in_progress' => '1', 'master_sync_total_bytes' => '1000', 'master_sync_left_bytes' => '250'] + ))->toBe(75.0); + expect(replication_manager::redisReplicationPercentFromInfo( + ['master_repl_offset' => 1000], + ['master_sync_in_progress' => '1'] + ))->toBe(5.0); + expect(replication_manager::redisProvisionProgress(0.0))->toBe(5.0); + expect(replication_manager::redisProvisionProgress(100.0, ['Redis replica link to primary is not up.']))->toBe(99.99); + expect(replication_manager::replicationHealthStatus(true, 'replica', 5.0, []))->toBe('degraded'); + expect(replication_manager::replicationHealthStatus(true, 'replica', 5.0, [], false))->toBe('ok'); + expect(replication_manager::replicationHealthStatus(true, 'replica', 100.0, []))->toBe('ok'); }); it('computes MariaDB GTID coverage by domain sequence', function (): void { @@ -62,14 +79,17 @@ it('generates replication-ready MariaDB compose templates without embedding secr expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.logs"'); expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.edge_gateway_log_entries"'); expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.replication_status_snapshots"'); + expect($template['compose'])->toContain('"--replicate-ignore-table=nnks_db.system_search_documents"'); expect($template['compose'])->toContain('"5433:3306"'); expect($template['compose'])->toContain('mariadb-replica-2-seed'); expect($template['compose'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD'); - expect($template['compose'])->toContain('mariadb-dump --host="$MARIADB_PRIMARY_HOST"'); - expect($template['compose'])->toContain('--ignore-table="$MARIADB_SEED_DATABASE.logs"'); - expect($template['compose'])->toContain('--ignore-table="$MARIADB_SEED_DATABASE.edge_gateway_log_entries"'); - expect($template['compose'])->toContain('--ignore-table="$MARIADB_SEED_DATABASE.replication_status_snapshots"'); - expect($template['compose'])->toContain('--no-data "$MARIADB_SEED_DATABASE" "$table"'); + expect($template['compose'])->toContain('mariadb-dump --host="$${MARIADB_PRIMARY_HOST}"'); + expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.logs"'); + expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.edge_gateway_log_entries"'); + expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.replication_status_snapshots"'); + expect($template['compose'])->toContain('--ignore-table="$${MARIADB_SEED_DATABASE}.system_search_documents"'); + expect($template['compose'])->toContain('--no-data "$${MARIADB_SEED_DATABASE}" "$${table}"'); + expect($template['compose'])->toContain('touch "$${marker}"'); expect($template['compose'])->toContain('${MARIADB_ROOT_PASSWORD:?set MARIADB_ROOT_PASSWORD}'); expect($template['compose'])->not->toContain(''); expect($template['env'])->toMatch('/MARIADB_ROOT_PASSWORD=[A-Za-z0-9_-]{32}/'); @@ -85,6 +105,19 @@ it('generates replication-ready MariaDB compose templates without embedding secr expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.logs\''); expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.edge_gateway_log_entries\''); expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.replication_status_snapshots\''); + expect($template['seed_command'])->toContain('--ignore-table=\'nnks_db.system_search_documents\''); +}); + +it('can embed primary admin credentials in generated MariaDB replica env files', function (): void { + $template = replication_manager::composeTemplate([ + 'kind' => 'database', + 'role' => 'replica', + 'primary_admin_username' => 'primary-root', + 'primary_admin_password' => 'primary-secret', + ]); + + expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_USER=primary-root'); + expect($template['env'])->toContain('MARIADB_PRIMARY_ADMIN_PASSWORD=primary-secret'); }); it('detects missing database tables before provisioning a preseeded replica', function (): void { @@ -112,7 +145,7 @@ it('seeds MariaDB replicas in place instead of requiring container recreation', expect($content)->toContain("'connect_without_database' => \$usePreseededReplica"); }); -it('keeps operational log tables schema-only during MariaDB seeding and replication', function (): void { +it('keeps operational and derived tables schema-only during MariaDB seeding and replication', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); expect($content)->toContain('MARIADB_SCHEMA_ONLY_TABLES'); @@ -121,6 +154,7 @@ it('keeps operational log tables schema-only during MariaDB seeding and replicat expect($content)->toContain("'replication_status_snapshots'"); expect($content)->toContain("'replication_operations'"); expect($content)->toContain("'replication_audit_logs'"); + expect($content)->toContain("'system_search_documents'"); expect($content)->toContain("'skip_data' => \$skipData"); expect($content)->toContain('createMariaDbReplicaTable('); expect($content)->toContain('SET GLOBAL replicate_ignore_table'); @@ -138,6 +172,20 @@ it('allows failed replicas to be removed without allowing primary or healthy rep expect(replication_manager::replicationHostCanBeRemoved(['role' => 'inactive', 'status' => 'inactive']))->toBeTrue(); expect(replication_manager::replicationHostCanBeRemoved(['role' => 'replica', 'status' => 'degraded']))->toBeTrue(); expect(replication_manager::replicationHostCanBeRemoved(['role' => 'replica', 'status' => 'ok']))->toBeFalse(); + + $content = file_get_contents(app_path('classes/replication_manager.php')); + expect($content)->toContain('coolify_manager::replicationHostCanBeRemoved'); + expect($content)->toContain('coolify_manager::markTargetsRemovedForReplicationHost'); +}); + +it('supports metadata-only replication host renames', function (): void { + $content = file_get_contents(app_path('classes/replication_manager.php')); + + expect($content)->toContain('function renameHost('); + expect($content)->toContain('host_renamed'); + expect($content)->toContain('coolify_manager::syncLabelForReplicationHost'); + expect($content)->toContain('writeBootstrapSnapshot()'); + expect($content)->toContain('Replication host label must be 128 characters or fewer.'); }); it('generates Redis replica compose templates with primary connection placeholders', function (): void { @@ -178,6 +226,8 @@ it('generates MinIO replica compose templates without embedding secrets', functi 'kind' => 'minio', 'role' => 'replica', 'service_name' => 'MinIO Replica 1', + 'host' => 'node2.truckwash.dk', + 'scheme' => 'http', 'host_port' => 9010, 'console_port' => 9011, 'buckets' => ['attachments', 'uploads'], @@ -191,32 +241,277 @@ it('generates MinIO replica compose templates without embedding secrets', functi expect($template['compose'])->toContain('image: "minio/minio:latest"'); expect($template['compose'])->toContain('MINIO_ROOT_USER: "${MINIO_ROOT_USER:?set MINIO_ROOT_USER}"'); expect($template['compose'])->toContain('MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD}"'); + expect($template['compose'])->toContain('MINIO_SERVER_URL: "${MINIO_SERVER_URL:-}"'); + expect($template['compose'])->toContain('MINIO_BROWSER_REDIRECT_URL: "${MINIO_BROWSER_REDIRECT_URL:-}"'); expect($template['compose'])->toContain('"9010:9000"'); expect($template['compose'])->toContain('"9011:9001"'); - expect($template['compose'])->toContain('mc mb --ignore-existing'); + expect($template['compose'])->toContain('mc mb --with-lock --ignore-existing'); expect($template['compose'])->toContain('mc version enable'); expect($template['compose'])->toContain('MINIO_PRIMARY_ENDPOINT'); expect($template['compose'])->not->toContain($template['credentials']['password']); + expect($template['env'])->toMatch('/MINIO_ROOT_USER=twminio[a-f0-9]{24}/'); expect($template['env'])->toMatch('/MINIO_ROOT_PASSWORD=[A-Za-z0-9_-]{32}/'); + expect($template['env'])->toContain('MINIO_SERVER_URL=http://node2.truckwash.dk:9010'); + expect($template['env'])->toContain('MINIO_BROWSER_REDIRECT_URL=http://node2.truckwash.dk:9011'); expect($template['env'])->toContain('MINIO_BUCKETS=attachments,uploads'); + expect($template['env'])->toContain('MINIO_REPLICATION_TRANSFER_LIMIT=25Mi'); expect($template['env'])->toContain('MINIO_PRIMARY_ENDPOINT='); + expect($template['credentials']['username'])->toMatch('/^twminio[a-f0-9]{24}$/'); expect($template['credentials']['scheme'])->toBe('http'); expect($template['credentials']['buckets'])->toBe(['attachments', 'uploads']); + expect($template['credentials']['replication_transfer_limit'])->toBe('25Mi'); expect($template['credentials']['space_headroom_percent'])->toBe(20.0); }); +it('keeps MinIO backup replicas bounded to the recent backup window', function (): void { + $template = replication_manager::composeTemplate([ + 'kind' => 'minio', + 'role' => 'replica', + 'service_name' => 'minio-replica-1', + 'buckets' => ['backups', 'uploads'], + ]); + $content = file_get_contents(app_path('classes/replication_manager.php')); + + expect(replication_manager::minioBackupReplicaRetentionDays())->toBe(30); + expect($template['compose'])->toContain('mc ilm rule add --expire-days "30" --noncurrent-expire-days "30"'); + expect($template['env'])->toContain('MINIO_BACKUP_REPLICA_RETENTION_DAYS=30'); + expect($template['steps'])->toContain('The backups bucket is retained on replicas for 30 days; other buckets are fully replicated.'); + expect($content)->not->toContain('seedMinioReplicaBackupWindow'); + expect($content)->not->toContain("'--newer-than'"); + expect($content)->toContain("'--limit-upload'"); + expect($content)->toContain("'--limit-download'"); + expect($content)->toContain("? 'delete,delete-marker'"); + expect($content)->toContain('putBucketLifecycleConfiguration'); + expect($content)->toContain('listObjectVersions'); + expect($content)->toContain('minioBackupReplicaRetentionConfigured'); + expect(replication_manager::minioBackupRetentionBlockers([ + 'buckets' => [ + ['name' => 'backups', 'expired_objects' => 2], + ], + ]))->toBe([ + 'MinIO backup replica contains 2 backup objects older than 30 days. Run provisioning to prune retained backups.', + ]); +}); + +it('prefills MinIO replica compose primary values from current config when available', function (): void { + $previousMinio = $GLOBALS['MINIO'] ?? null; + + $GLOBALS['MINIO'] = [ + 'endpoint' => 'https://minio-primary.internal:9000', + 'access_key' => 'primary-access', + 'secret_key' => 'primary-secret', + ]; + + try { + $template = replication_manager::composeTemplate([ + 'kind' => 'minio', + 'role' => 'replica', + 'service_name' => 'minio-replica-1', + ]); + + expect($template['env'])->toContain('MINIO_PRIMARY_ENDPOINT=https://minio-primary.internal:9000'); + expect($template['env'])->toContain('MINIO_PRIMARY_ACCESS_KEY=primary-access'); + expect($template['env'])->toContain('MINIO_PRIMARY_SECRET_KEY=primary-secret'); + expect($template['compose'])->not->toContain('primary-secret'); + } finally { + if ($previousMinio === null) { + unset($GLOBALS['MINIO']); + } else { + $GLOBALS['MINIO'] = $previousMinio; + } + } +}); + it('computes MinIO free-space and catch-up math safely', function (): void { expect(replication_manager::minioRequiredFreeBytes(1000))->toBe(1200); expect(replication_manager::minioByteReplicationPercent(1000, 750))->toBe(75.0); expect(replication_manager::minioByteReplicationPercent(0, 0))->toBe(100.0); + expect(replication_manager::minioProvisionProgress([ + 'replication_percent' => 3.2, + 'raw' => ['storage' => ['measured' => true]], + ]))->toBe(3.2); + expect(replication_manager::minioProvisionProgress([ + 'replication_percent' => 0, + 'raw' => ['storage' => ['measured' => false]], + ]))->toBe(5.0); + expect(replication_manager::minioProvisionProgress([ + 'replication_percent' => 2.5, + 'raw' => ['progress_source' => 'minio_replicate_status'], + ]))->toBe(2.5); + expect(replication_manager::minioProvisionProgress([ + 'replication_percent' => 100, + 'raw' => ['storage' => ['measured' => true]], + ]))->toBe(100.0); + expect(replication_manager::minioProvisionProgress([ + 'replication_percent' => 100, + 'blockers' => ['MinIO replica has not caught up.'], + 'raw' => ['storage' => ['measured' => true]], + ]))->toBe(99.9); + expect(replication_manager::minioReplicationProgressFromStatusOutput([ + 'target' => [ + 'replicated' => ['size' => 750], + 'pending' => ['size' => 250], + ], + ])['replication_percent'])->toBe(75.0); + expect(replication_manager::minioReplicationProgressFromStatusOutput([ + 'objects' => [ + 'completed' => 9, + 'pending' => 1, + ], + ])['replication_percent'])->toBe(90.0); + expect(replication_manager::minioReplicationProgressFromStatusOutput([ + 'status' => 'complete', + ])['replication_percent'])->toBe(100.0); + expect(replication_manager::minioReplicationProgressFromStatusOutput( + '{"replicatedSize": 750, "pendingSize": 250}' + )['replication_percent'])->toBe(75.0); + expect(replication_manager::minioReplicationProgressFromStatusOutput( + '{"replicaSize": 750, "pendingSize": 250}' + )['replication_percent'])->toBe(75.0); + expect(replication_manager::minioReplicationProgressFromStatusOutput( + 'target-a: 100%, target-b: 5%' + )['replication_percent'])->toBe(5.0); + expect(replication_manager::minioReplicationProgressFromStatusOutput( + 'target-a: 100%, target-b: 100%' + )['replication_percent'])->toBe(100.0); + expect(replication_manager::minioReplicationProgressFromStatusOutput([ + 'completedReplicationSize' => 1000, + 'queued' => [ + 'curr' => ['count' => 0, 'bytes' => 0], + 'avg' => ['count' => 42, 'bytes' => 25000000], + 'peak' => ['count' => 100, 'bytes' => 50000000], + ], + ])['replication_percent'])->toBe(100.0); + expect(replication_manager::minioBucketCountsTowardCatchUp('uploads'))->toBeTrue(); + expect(replication_manager::minioBucketCountsTowardCatchUp('backups'))->toBeFalse(); + $boundedBackupProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([ + 'uploads' => ['replication_percent' => 100.0, 'blockers' => [], 'stats' => []], + 'backups' => ['replication_percent' => 99.74, 'blockers' => ['MinIO replica has not caught up.'], 'stats' => []], + ]); + expect($boundedBackupProgress['replication_percent'])->toBe(100.0); + expect($boundedBackupProgress['blockers'])->toBe([]); + expect($boundedBackupProgress['ignored_buckets'])->toBe(['backups']); + $onlyBoundedBackupProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([ + 'backups' => ['replication_percent' => 5.0, 'blockers' => ['MinIO replica has not caught up.'], 'stats' => []], + ]); + expect($onlyBoundedBackupProgress['replication_percent'])->toBe(100.0); + expect($onlyBoundedBackupProgress['basis'])->toBe('bounded_retention_only'); + $liveQueueProgress = replication_manager::minioCatchUpProgressFromBucketStatuses([ + 'uploads' => [ + 'replication_percent' => 99.98, + 'blockers' => ['MinIO replica has not caught up.'], + 'stats' => [ + 'completed_bytes' => 53149249190, + 'pending_bytes' => 3439936, + 'failed_bytes' => 0, + 'total_bytes' => 0, + 'completed_count' => 199368, + 'pending_count' => 7, + 'failed_count' => 0, + 'total_count' => 0, + ], + ], + ]); + expect($liveQueueProgress['replication_percent'])->toBe(100.0); + expect($liveQueueProgress['blockers'])->toBe([]); + expect($liveQueueProgress['live_tolerance']['within_tolerance'])->toBeTrue(); expect(replication_manager::minioSpaceBlockers(1199, 1200))->toContain('MinIO target does not have enough free space. Required 1200 bytes, available 1199 bytes.'); - expect(replication_manager::minioSpaceBlockers(null, 1200))->toContain('MinIO target free space could not be determined.'); + expect(replication_manager::minioSpaceBlockers(null, 1200))->toBe([]); + expect(replication_manager::minioSpaceBlockers(1200, 1200))->toBe([]); expect(replication_manager::minioAvailableBytesFromAdminInfo([ 'servers' => [ ['drives' => [['availableSpace' => 4096]]], ], ]))->toBe(4096); expect(replication_manager::normalizeMinioBuckets('Attachments, uploads backups'))->toBe(['attachments', 'uploads', 'backups']); + expect(replication_manager::minioDefaultReplicationTransferLimit())->toBe('25Mi'); + expect(replication_manager::normalizeMinioTransferLimit('25MiB/s'))->toBe('25Mi'); + expect(replication_manager::normalizeMinioTransferLimit('100 MB'))->toBe('100M'); + expect(replication_manager::normalizeMinioTransferLimit('0'))->toBe(''); +}); + +it('allows the MinIO client binary to be configured explicitly', function (): void { + $previous = getenv('MINIO_MC_BINARY'); + putenv('MINIO_MC_BINARY=/opt/minio/mc'); + + try { + $method = new ReflectionMethod(replication_manager::class, 'minioClientBinary'); + $method->setAccessible(true); + + expect($method->invoke(null))->toBe('/opt/minio/mc'); + } finally { + if ($previous === false) { + putenv('MINIO_MC_BINARY'); + } else { + putenv('MINIO_MC_BINARY=' . $previous); + } + } +}); + +it('supports MinIO client runtime fallback configuration', function (): void { + $previousDownloadUrl = getenv('MINIO_MC_DOWNLOAD_URL'); + $previousAutoInstall = getenv('MINIO_MC_AUTO_INSTALL'); + $previousCommandTimeout = getenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS'); + $previousDownloadTimeout = getenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS'); + putenv('MINIO_MC_DOWNLOAD_URL=https://example.test/mc'); + putenv('MINIO_MC_AUTO_INSTALL=0'); + putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS=3'); + putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS=4'); + + try { + $downloadUrl = new ReflectionMethod(replication_manager::class, 'minioClientDownloadUrl'); + $downloadUrl->setAccessible(true); + $autoInstall = new ReflectionMethod(replication_manager::class, 'minioClientAutoInstallEnabled'); + $autoInstall->setAccessible(true); + $commandTimeout = new ReflectionMethod(replication_manager::class, 'minioClientCommandTimeoutSeconds'); + $commandTimeout->setAccessible(true); + $downloadTimeout = new ReflectionMethod(replication_manager::class, 'minioClientDownloadTimeoutSeconds'); + $downloadTimeout->setAccessible(true); + $commandLabel = new ReflectionMethod(replication_manager::class, 'minioClientCommandLabel'); + $commandLabel->setAccessible(true); + + expect($downloadUrl->invoke(null))->toBe('https://example.test/mc'); + expect($autoInstall->invoke(null))->toBeFalse(); + expect($commandTimeout->invoke(null))->toBe(3); + expect($downloadTimeout->invoke(null))->toBe(4); + expect($commandLabel->invoke(null, [ + 'alias', + 'set', + 'target', + 'http://minio.example.test:9010', + 'access-key', + 'secret-key', + ]))->toContain('[redacted]'); + expect($commandLabel->invoke(null, [ + 'alias', + 'set', + 'target', + 'http://minio.example.test:9010', + 'access-key', + 'secret-key', + ]))->not->toContain('secret-key'); + } finally { + if ($previousDownloadUrl === false) { + putenv('MINIO_MC_DOWNLOAD_URL'); + } else { + putenv('MINIO_MC_DOWNLOAD_URL=' . $previousDownloadUrl); + } + if ($previousAutoInstall === false) { + putenv('MINIO_MC_AUTO_INSTALL'); + } else { + putenv('MINIO_MC_AUTO_INSTALL=' . $previousAutoInstall); + } + if ($previousCommandTimeout === false) { + putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS'); + } else { + putenv('MINIO_MC_COMMAND_TIMEOUT_SECONDS=' . $previousCommandTimeout); + } + if ($previousDownloadTimeout === false) { + putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS'); + } else { + putenv('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS=' . $previousDownloadTimeout); + } + } }); it('wires MinIO replication through routes and bootstrap snapshots', function (): void { @@ -230,6 +525,35 @@ it('wires MinIO replication through routes and bootstrap snapshots', function () expect($manager)->toContain('testMinioHost($host)'); expect($manager)->toContain('minioTargetFreeBytes($host)'); expect($manager)->toContain('minioRequiredFreeBytes'); + expect($manager)->toContain('minioReplicationConfiguredForHosts($primary, $host)'); + expect($manager)->toContain("'--priority',"); + expect($manager)->toContain('minioReplicationTransferLimitArgs'); + expect($manager)->toContain('MINIO_PROGRESS_SCAN_INTERVAL_SECONDS'); + expect($manager)->toContain('MINIO_INCOMPLETE_PROGRESS_MAX_PERCENT'); + expect($manager)->toContain('MINIO_MC_COMMAND_TIMEOUT_SECONDS'); + expect($manager)->toContain('MINIO_MC_DOWNLOAD_TIMEOUT_SECONDS'); + expect($manager)->toContain('proc_terminate($process'); + expect($manager)->toContain("'connect_timeout' => self::MINIO_S3_CONNECT_TIMEOUT_SECONDS"); + expect($manager)->toContain("'retries' => 0"); + expect($manager)->toContain('&& $forceStorageScan;'); + expect($manager)->not->toContain('$isPrimary || $forceStorageScan'); + expect($manager)->toContain('sanitizePublicLastStatus'); + expect($manager)->toContain('MinIO primary object-scan timeouts do not indicate primary availability failure.'); + expect($manager)->toContain('minioProvisionProgress($status)'); + expect($manager)->toContain('MinIO replica is syncing. Copied'); + expect($manager)->toContain("['mb', '--with-lock', '--ignore-existing', 'target/' . \$bucket]"); + expect($manager)->toContain('repairMinioTargetBucketObjectLockIfEmpty($configDir, $target, $bucket, $message)'); + expect($manager)->toContain('minioBucketHasObjects($target, $bucket)'); + expect($manager)->toContain("'skip_storage_scan' => true"); + expect($manager)->toContain('lastStatusReplicationPercent($host, 5.0)'); + expect($manager)->toContain('completeReadyMinioProvisionOperation'); + expect($manager)->toContain('MinIO replication target is caught up.'); + expect($manager)->toContain('shouldAdvanceActiveProvisionDuringRefresh'); + expect($manager)->toContain('provisionHost((string)$host[\'kind\'], (int)$host[\'id\'])'); + expect($manager)->toContain('stale targets do not keep a healthy current target below 100%'); + expect($manager)->toContain("'replicate',\n 'status',\n 'source/' . \$bucket"); + expect($manager)->toContain("'replicate',\n 'status',\n '--json',\n 'source/' . \$bucket"); + expect($manager)->toContain('private function minioBucketStats('); expect($manager)->toContain("'minio' => ["); expect($routes)->toContain("/superuser/replication/minio"); expect($openapi)->toContain('enum: [database, redis, minio]'); @@ -246,8 +570,9 @@ it('provisions Redis replicas after a connectivity-only preflight and reports sy expect($content)->toContain("executeRaw(['CONFIG', 'REWRITE'])"); expect($content)->toContain('Redis replication was configured; waiting for the replica to catch up.'); expect($content)->toContain('$onlySyncBlockers'); - expect($content)->toContain("'status' => 'running'"); - expect($content)->toContain('Redis replica is syncing from the primary.'); + expect($content)->toContain('redisProvisionProgress'); + expect($content)->toContain('Redis replication is configured and syncing in the background.'); + expect($content)->toContain('Redis replication is configured, but the replica is waiting for the primary link.'); }); it('keeps Redis promotion caught-up, durable, and metadata-safe', function (): void { @@ -285,10 +610,58 @@ it('keeps replication operation progress schema idempotent for existing installs it('creates the generated replication user on the primary during provisioning', function (): void { $content = file_get_contents(app_path('classes/replication_manager.php')); - expect($content)->toContain('ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword)'); + expect($content)->toContain('$grantHosts = $this->databaseReplicationGrantHosts($host, $targetStatus);'); + expect($content)->toContain('ensureDatabaseReplicationUser($primary, $replicationUser, $replicationPassword, $grantHosts)'); + expect($content)->toContain('databaseDeniedAccountHostsFromText'); + expect($content)->toContain('Access denied for user'); + expect($content)->toContain('foreach ($grantHosts as $grantHost)'); + expect($content)->toContain('shouldRepairDatabaseReplicationAccess'); + expect($content)->toContain('repairDatabaseReplicationAccess'); + expect($content)->toContain('shouldRepairDatabaseReplicationThreads'); + expect($content)->toContain('repairDatabaseReplicationThreads'); + expect($content)->toContain('refreshDatabaseReplicationConnection'); + expect($content)->toContain("CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, MASTER_USER = '%s', MASTER_PASSWORD = '%s', MASTER_USE_GTID = slave_pos"); + expect($content)->toContain("CHANGE REPLICATION SOURCE TO SOURCE_HOST = '%s', SOURCE_PORT = %d, SOURCE_USER = '%s', SOURCE_PASSWORD = '%s', SOURCE_AUTO_POSITION = 1"); + expect($content)->toContain('restartDatabaseReplicationThreads'); + expect($content)->toContain('databaseOnlyReplicationThreadBlockers'); + expect($content)->toContain('databaseAccountHostGrantCandidates'); + expect($content)->toContain('START SLAVE SQL_THREAD'); expect($content)->toContain('GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO'); }); +it('extracts host-specific MariaDB replication account denials', function (): void { + $extract = new ReflectionMethod(replication_manager::class, 'databaseDeniedAccountHostsFromText'); + $extract->setAccessible(true); + $normalize = new ReflectionMethod(replication_manager::class, 'normalizeDatabaseAccountHost'); + $normalize->setAccessible(true); + $candidates = new ReflectionMethod(replication_manager::class, 'databaseAccountHostGrantCandidates'); + $candidates->setAccessible(true); + + expect($extract->invoke(null, "Access denied for user 'replication'@'10.0.1.13' (using password: YES)")) + ->toBe(['10.0.1.13']); + expect($extract->invoke(null, "Access denied for user 'replication'@'fd9c:738d:4130::d' (using password: YES)")) + ->toBe(['fd9c:738d:4130::d']); + expect($normalize->invoke(null, '10.0.1.13'))->toBe('10.0.1.13'); + expect($normalize->invoke(null, 'bad host;drop'))->toBeNull(); + expect($candidates->invoke(null, '10.0.1.13'))->toBe(['10.0.1.13', '10.0.1.%']); + expect($candidates->invoke(null, 'fd9c:738d:4130::d'))->toBe(['fd9c:738d:4130::d', 'fd9c:738d:4130::%']); +}); + +it('identifies stopped database replication threads as a restartable status', function (): void { + $onlyThreadBlockers = new ReflectionMethod(replication_manager::class, 'databaseOnlyReplicationThreadBlockers'); + $onlyThreadBlockers->setAccessible(true); + + expect($onlyThreadBlockers->invoke(null, [ + 'Database replication IO and SQL threads must both be running.', + 'Database replication IO thread is not running.', + 'Database replication SQL thread is not running.', + ]))->toBeTrue(); + expect($onlyThreadBlockers->invoke(null, [ + 'Database replication IO thread is not running.', + "error reconnecting to master 'replication@23.88.23.183:5432'", + ]))->toBeFalse(); +}); + it('supports MariaDB prerequisites without requiring Oracle MySQL variables', function (): void { $blockers = replication_manager::databasePrerequisiteBlockers([ 'server_version' => '11.8.6-MariaDB-ubu2404', diff --git a/services/nginx/app/tests/Unit/Replication/SuperuserReplicationRouteWiringTest.php b/services/nginx/app/tests/Unit/Replication/SuperuserReplicationRouteWiringTest.php index aa881f21..9f5d225d 100644 --- a/services/nginx/app/tests/Unit/Replication/SuperuserReplicationRouteWiringTest.php +++ b/services/nginx/app/tests/Unit/Replication/SuperuserReplicationRouteWiringTest.php @@ -13,6 +13,7 @@ it('registers superuser replication endpoints and permissions', function (): voi expect($content)->toContain('/superuser/replication/{kind}/{id}/test'); expect($content)->toContain('/superuser/replication/{kind}/{id}/provision'); expect($content)->toContain('/superuser/replication/{kind}/{id}/promote'); + expect($content)->toContain("\$this->patch('/superuser/replication/{kind}/{id}'"); expect($content)->toContain("requirePermission('superuser_replication_view')"); expect($content)->toContain("requirePermission('superuser_replication_manage')"); expect($content)->toContain("requirePermission('superuser_replication_promote')"); @@ -27,9 +28,11 @@ it('documents replication management in openapi', function (): void { expect($content)->toContain('operationId: generateSuperuserReplicationComposeTemplate'); expect($content)->toContain('operationId: testSuperuserReplicationCredentials'); expect($content)->toContain('operationId: addSuperuserMinioReplicationHost'); + expect($content)->toContain('operationId: renameSuperuserReplicationHost'); expect($content)->toContain('enum: [database, redis, minio]'); expect($content)->toContain('space_headroom_percent'); expect($content)->toContain('SuperuserReplicationStatus'); expect($content)->toContain('SuperuserReplicationHostCreateRequest'); + expect($content)->toContain('SuperuserReplicationHostRenameRequest'); expect($content)->toContain('SuperuserReplicationComposeTemplateRequest'); }); diff --git a/services/nginx/app/tests/Unit/Scanner/ModuleScannerRouteTest.php b/services/nginx/app/tests/Unit/Scanner/ModuleScannerRouteTest.php new file mode 100644 index 00000000..4fa6380e --- /dev/null +++ b/services/nginx/app/tests/Unit/Scanner/ModuleScannerRouteTest.php @@ -0,0 +1,10 @@ +not->toBeFalse(); + expect($route)->toContain("'reason' => 'no_license_plate_detected'"); + expect($route)->toContain('], 200);'); + expect($route)->not->toContain("throw new Exception('License plate extraction failed.')"); +}); diff --git a/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayRouteWiringTest.php b/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayRouteWiringTest.php index e07c507e..6330fc22 100644 --- a/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayRouteWiringTest.php +++ b/services/nginx/app/tests/Unit/Selfserve/EdgeGatewayRouteWiringTest.php @@ -28,6 +28,8 @@ it('registers the v2 operator-facing edge gateway routes', function (): void { it('registers PHP edge agent routes for operations and legacy relay command polling', function (): void { $route = file_get_contents(app_path('routes/edgeGatewaysRoute.php')); + $manager = file_get_contents(app_path('classes/edge_gateway_manager.php')); + $agent = file_get_contents(app_path('resources/edge-gateway-agent/agent.php')); expect($route)->toContain("'/edge-agent/install-token/verify'"); expect($route)->toContain("'/edge-agent/install-token/status'"); @@ -57,6 +59,9 @@ it('registers PHP edge agent routes for operations and legacy relay command poll expect($route)->toContain("'/edge-agent/internal/browser-streams/validate'"); expect($route)->toContain("'/edge-agent/internal/shell-sessions/opened'"); expect($route)->toContain('echo $exception->getMessage()'); + expect($manager)->toContain("\$gatewayPayload['broker_url'] = \$this->buildBrokerPublicUrl();"); + expect($agent)->toContain('applyBrokerUrlFromControlPlaneResponse'); + expect($agent)->toContain("Updated broker URL from control plane heartbeat response."); expect($route)->not->toContain("'/edge-agent/gateways/{id}/shell-actions/poll'"); expect($route)->not->toContain("'/edge-agent/gateways/{id}/shell-sessions/{sessionId}/events'"); }); diff --git a/services/nginx/nginx.conf b/services/nginx/nginx.conf index 672601ff..c10fe680 100644 --- a/services/nginx/nginx.conf +++ b/services/nginx/nginx.conf @@ -68,13 +68,13 @@ http { # Location block for PHP files location ^~ / { add_header Access-Control-Allow-Methods "GET, POST, OPTIONS, PUT, DELETE"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; add_header Access-Control-Allow-Credentials true; if ($request_method = 'OPTIONS') { add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Methods "GET, POST, OPTIONS, PUT, DELETE"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; return 204; } @@ -125,13 +125,13 @@ http { # Location block for PHP files location ^~ / { add_header Access-Control-Allow-Methods "GET, POST, OPTIONS, PUT, DELETE"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; add_header Access-Control-Allow-Credentials true; if ($request_method = 'OPTIONS') { add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Methods "GET, POST, OPTIONS, PUT, DELETE"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; return 204; } @@ -190,4 +190,4 @@ http { # Restrict access to the server, if the -} \ No newline at end of file +} diff --git a/services/nginx/nginx.dev.conf b/services/nginx/nginx.dev.conf index 258901bb..1608de11 100644 --- a/services/nginx/nginx.dev.conf +++ b/services/nginx/nginx.dev.conf @@ -50,12 +50,12 @@ http { # Main application location location / { add_header Access-Control-Allow-Methods "GET, POST, OPTIONS, PUT, DELETE"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; add_header Access-Control-Allow-Credentials true; if ($request_method = 'OPTIONS') { add_header Access-Control-Allow-Origin *; add_header Access-Control-Allow-Methods "GET, POST, OPTIONS, PUT, DELETE"; - add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number"; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, X-Requested-With, X-Customer-Number, X-Release-Trace, X-Release-Channel, X-Frontend-Version"; return 204; } diff --git a/services/traefik/dynamic.yml b/services/traefik/dynamic.yml index 77e6cf4c..3323585f 100644 --- a/services/traefik/dynamic.yml +++ b/services/traefik/dynamic.yml @@ -104,6 +104,9 @@ http: - Authorization - Content-Type - X-Customer-Number + - X-Release-Trace + - X-Release-Channel + - X-Frontend-Version api-ratelimit: rateLimit: average: 100