From 0deeb7f121ed547066ec78a0b716a7b939429a08 Mon Sep 17 00:00:00 2001 From: Jeppe Bundgaard Date: Thu, 12 Mar 2026 14:10:06 +0100 Subject: [PATCH] Add scripts to clone live DB into debug DB for local development --- scripts/clone-live-to-debug-db.ps1 | 131 +++++++++++++++++++++++++++++ scripts/clone-live-to-debug-db.sh | 96 +++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 scripts/clone-live-to-debug-db.ps1 create mode 100644 scripts/clone-live-to-debug-db.sh diff --git a/scripts/clone-live-to-debug-db.ps1 b/scripts/clone-live-to-debug-db.ps1 new file mode 100644 index 00000000..65d4edd7 --- /dev/null +++ b/scripts/clone-live-to-debug-db.ps1 @@ -0,0 +1,131 @@ +param( + [string]$SourceHost = "", + [int]$SourcePort = 3306, + [string]$SourceDatabase = "", + [string]$SourceUser = "", + [string]$SourcePassword = "", + [string]$TargetService = "mysql-debug", + [string]$TargetDatabase = "", + [string]$TargetUser = "", + [string]$TargetPassword = "", + [switch]$Force +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Read-DotEnv { + param([string]$Path) + + $values = @{} + if (-not (Test-Path $Path)) { return $values } + + foreach ($line in Get-Content $Path) { + $trimmed = $line.Trim() + if ($trimmed -eq "" -or $trimmed.StartsWith("#")) { continue } + $idx = $trimmed.IndexOf("=") + if ($idx -lt 1) { continue } + $key = $trimmed.Substring(0, $idx).Trim() + $value = $trimmed.Substring($idx + 1).Trim() + if (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'"))) { + $value = $value.Substring(1, $value.Length - 2) + } + $values[$key] = $value + } + return $values +} + +function Get-EnvOrDotEnv { + param( + [hashtable]$DotEnv, + [string]$Key + ) + $envValue = [Environment]::GetEnvironmentVariable($Key) + if (-not [string]::IsNullOrWhiteSpace($envValue)) { return $envValue } + if ($DotEnv.ContainsKey($Key) -and -not [string]::IsNullOrWhiteSpace([string]$DotEnv[$Key])) { + return [string]$DotEnv[$Key] + } + return "" +} + +$RootDir = Split-Path -Parent $PSScriptRoot +Set-Location $RootDir + +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + throw "Docker is required but was not found in PATH." +} + +$dotEnv = Read-DotEnv -Path (Join-Path $RootDir ".env") + +if ([string]::IsNullOrWhiteSpace($SourceHost)) { $SourceHost = Get-EnvOrDotEnv -DotEnv $dotEnv -Key "CONFIG_DB_HOST" } +if ([string]::IsNullOrWhiteSpace($SourceDatabase)) { $SourceDatabase = Get-EnvOrDotEnv -DotEnv $dotEnv -Key "CONFIG_DB_DATABASE" } +if ([string]::IsNullOrWhiteSpace($SourceUser)) { $SourceUser = Get-EnvOrDotEnv -DotEnv $dotEnv -Key "CONFIG_DB_USER" } +if ([string]::IsNullOrWhiteSpace($SourcePassword)) { $SourcePassword = Get-EnvOrDotEnv -DotEnv $dotEnv -Key "CONFIG_DB_PASSWORD" } + +if ([string]::IsNullOrWhiteSpace($TargetDatabase)) { $TargetDatabase = Get-EnvOrDotEnv -DotEnv $dotEnv -Key "CONFIG_DB_DEBUG_DATABASE" } +if ([string]::IsNullOrWhiteSpace($TargetUser)) { $TargetUser = Get-EnvOrDotEnv -DotEnv $dotEnv -Key "CONFIG_DB_DEBUG_USER" } +if ([string]::IsNullOrWhiteSpace($TargetPassword)) { $TargetPassword = Get-EnvOrDotEnv -DotEnv $dotEnv -Key "CONFIG_DB_DEBUG_PASSWORD" } + +if ([string]::IsNullOrWhiteSpace($SourceHost) -or [string]::IsNullOrWhiteSpace($SourceDatabase) -or [string]::IsNullOrWhiteSpace($SourceUser) -or [string]::IsNullOrWhiteSpace($SourcePassword)) { + throw "Missing live DB credentials. Expected CONFIG_DB_HOST/CONFIG_DB_DATABASE/CONFIG_DB_USER/CONFIG_DB_PASSWORD." +} +if ([string]::IsNullOrWhiteSpace($TargetDatabase) -or [string]::IsNullOrWhiteSpace($TargetUser) -or [string]::IsNullOrWhiteSpace($TargetPassword)) { + throw "Missing debug DB credentials. Expected CONFIG_DB_DEBUG_DATABASE/CONFIG_DB_DEBUG_USER/CONFIG_DB_DEBUG_PASSWORD." +} + +if (-not $Force) { + Write-Host "This will overwrite debug DB '$TargetDatabase' in service '$TargetService' with live DB '$SourceDatabase' from '$SourceHost'." -ForegroundColor Yellow + $confirmation = Read-Host "Type CLONE to continue" + if ($confirmation -ne "CLONE") { throw "Cancelled." } +} + +Write-Host "Starting debug DB service ($TargetService)..." -ForegroundColor Cyan +& docker compose up -d $TargetService + +Write-Host "Waiting for debug DB to accept connections..." -ForegroundColor Cyan +$maxAttempts = 60 +for ($i = 1; $i -le $maxAttempts; $i++) { + & docker compose exec -T $TargetService sh -lc "MYSQL_PWD='$TargetPassword' mysqladmin -u '$TargetUser' ping --silent" *> $null + if ($LASTEXITCODE -eq 0) { break } + if ($i -eq $maxAttempts) { throw "Timed out waiting for $TargetService to become ready." } + Start-Sleep -Seconds 2 +} + +$dumpDir = Join-Path $RootDir ".tmp-db-clone" +$dumpFile = Join-Path $dumpDir "live-to-debug.sql" +$resetFile = Join-Path $dumpDir "reset-debug.sql" +if (-not (Test-Path $dumpDir)) { New-Item -Path $dumpDir -ItemType Directory | Out-Null } +if (Test-Path $dumpFile) { Remove-Item -Path $dumpFile -Force } +if (Test-Path $resetFile) { Remove-Item -Path $resetFile -Force } + +Write-Host "Creating live dump..." -ForegroundColor Cyan +$dumpMount = $dumpDir.Replace('\', '/') +$dumpCmd = "exec mysqldump --compression-algorithms=zstd --single-transaction --quick --set-gtid-purged=OFF -h '$SourceHost' -P '$SourcePort' -u '$SourceUser' '$SourceDatabase' > /dump/live-to-debug.sql" +& docker run --rm -e "MYSQL_PWD=$SourcePassword" -v "${dumpMount}:/dump" mysql:8.4 sh -lc $dumpCmd +if ($LASTEXITCODE -ne 0 -or -not (Test-Path $dumpFile)) { throw "Database dump failed." } + +Write-Host "Resolving docker network for target service..." -ForegroundColor Cyan +$targetContainerId = (& docker compose ps -q $TargetService | Select-Object -First 1) +if ([string]::IsNullOrWhiteSpace($targetContainerId)) { throw "Could not resolve container id for service '$TargetService'." } +$targetNetwork = (& docker inspect $targetContainerId --format "{{range `$k,`$v := .NetworkSettings.Networks}}{{`$k}}{{end}}" | Out-String).Trim() +if ([string]::IsNullOrWhiteSpace($targetNetwork)) { throw "Could not resolve docker network for service '$TargetService'." } + +Write-Host "Resetting debug database '$TargetDatabase'..." -ForegroundColor Cyan +Set-Content -Path $resetFile -Value "DROP DATABASE IF EXISTS $TargetDatabase; CREATE DATABASE $TargetDatabase;" -NoNewline +$resetCmd = "MYSQL_PWD='$TargetPassword' mysql -h '$TargetService' -u '$TargetUser' < /dump/reset-debug.sql" +& docker run --rm ` + --network $targetNetwork ` + -v "${dumpMount}:/dump" ` + mysql:8.4 sh -lc $resetCmd +if ($LASTEXITCODE -ne 0) { throw "Failed to reset debug database." } + +Write-Host "Importing live dump into debug DB..." -ForegroundColor Cyan +& docker run --rm ` + --network $targetNetwork ` + -v "${dumpMount}:/dump" ` + mysql:8.4 sh -lc "MYSQL_PWD='$TargetPassword' mysql -h '$TargetService' -u '$TargetUser' '$TargetDatabase' < /dump/live-to-debug.sql" +if ($LASTEXITCODE -ne 0) { throw "Import into debug database failed." } + +Remove-Item -Path $dumpFile -Force -ErrorAction SilentlyContinue +Remove-Item -Path $resetFile -Force -ErrorAction SilentlyContinue +Write-Host "Done. Debug DB '$TargetDatabase' now mirrors live '$SourceDatabase'." -ForegroundColor Green diff --git a/scripts/clone-live-to-debug-db.sh b/scripts/clone-live-to-debug-db.sh new file mode 100644 index 00000000..2292b219 --- /dev/null +++ b/scripts/clone-live-to-debug-db.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +SOURCE_HOST="${SOURCE_HOST:-}" +SOURCE_PORT="${SOURCE_PORT:-3306}" +SOURCE_DATABASE="${SOURCE_DATABASE:-}" +SOURCE_USER="${SOURCE_USER:-}" +SOURCE_PASSWORD="${SOURCE_PASSWORD:-}" + +TARGET_SERVICE="${TARGET_SERVICE:-mysql-debug}" +TARGET_DATABASE="${TARGET_DATABASE:-}" +TARGET_USER="${TARGET_USER:-}" +TARGET_PASSWORD="${TARGET_PASSWORD:-}" + +FORCE="${FORCE:-0}" + +if [[ -f ".env" ]]; then + # shellcheck disable=SC2046 + export $(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' .env | sed 's/\r$//') +fi + +SOURCE_HOST="${SOURCE_HOST:-${CONFIG_DB_HOST:-}}" +SOURCE_DATABASE="${SOURCE_DATABASE:-${CONFIG_DB_DATABASE:-}}" +SOURCE_USER="${SOURCE_USER:-${CONFIG_DB_USER:-}}" +SOURCE_PASSWORD="${SOURCE_PASSWORD:-${CONFIG_DB_PASSWORD:-}}" + +TARGET_DATABASE="${TARGET_DATABASE:-${CONFIG_DB_DEBUG_DATABASE:-}}" +TARGET_USER="${TARGET_USER:-${CONFIG_DB_DEBUG_USER:-}}" +TARGET_PASSWORD="${TARGET_PASSWORD:-${CONFIG_DB_DEBUG_PASSWORD:-}}" + +if [[ -z "${SOURCE_HOST}" || -z "${SOURCE_DATABASE}" || -z "${SOURCE_USER}" || -z "${SOURCE_PASSWORD}" ]]; then + echo "Missing live DB credentials. Expected CONFIG_DB_HOST/CONFIG_DB_DATABASE/CONFIG_DB_USER/CONFIG_DB_PASSWORD." + exit 1 +fi +if [[ -z "${TARGET_DATABASE}" || -z "${TARGET_USER}" || -z "${TARGET_PASSWORD}" ]]; then + echo "Missing debug DB credentials. Expected CONFIG_DB_DEBUG_DATABASE/CONFIG_DB_DEBUG_USER/CONFIG_DB_DEBUG_PASSWORD." + exit 1 +fi + +if ! command -v docker >/dev/null 2>&1; then + echo "Docker is required but was not found in PATH." + exit 1 +fi + +if [[ "${FORCE}" != "1" ]]; then + echo "This will overwrite debug DB '${TARGET_DATABASE}' in service '${TARGET_SERVICE}' with live DB '${SOURCE_DATABASE}' from '${SOURCE_HOST}'." + read -r -p "Type CLONE to continue: " confirmation + if [[ "${confirmation}" != "CLONE" ]]; then + echo "Cancelled." + exit 1 + fi +fi + +echo "Starting debug DB service (${TARGET_SERVICE})..." +docker compose up -d "${TARGET_SERVICE}" + +echo "Waiting for debug DB to accept connections..." +for i in $(seq 1 60); do + if docker compose exec -T "${TARGET_SERVICE}" sh -lc "MYSQL_PWD='${TARGET_PASSWORD}' mysqladmin -u '${TARGET_USER}' ping --silent" >/dev/null 2>&1; then + break + fi + if [[ "$i" -eq 60 ]]; then + echo "Timed out waiting for ${TARGET_SERVICE} to become ready." + exit 1 + fi + sleep 2 +done + +DUMP_DIR="${ROOT_DIR}/.tmp-db-clone" +DUMP_FILE="${DUMP_DIR}/live-to-debug.sql" +mkdir -p "${DUMP_DIR}" +rm -f "${DUMP_FILE}" + +echo "Creating live dump..." +docker run --rm \ + -e "MYSQL_PWD=${SOURCE_PASSWORD}" \ + -v "${DUMP_DIR}:/dump" \ + mysql:8.4 \ + sh -lc "exec mysqldump --compression-algorithms=zstd --single-transaction --quick --set-gtid-purged=OFF -h '${SOURCE_HOST}' -P '${SOURCE_PORT}' -u '${SOURCE_USER}' '${SOURCE_DATABASE}' > /dump/live-to-debug.sql" + +if [[ ! -f "${DUMP_FILE}" ]]; then + echo "Database dump failed." + exit 1 +fi + +echo "Resetting debug database '${TARGET_DATABASE}'..." +docker compose exec -T "${TARGET_SERVICE}" sh -lc "MYSQL_PWD='${TARGET_PASSWORD}' mysql -u '${TARGET_USER}' -e \"DROP DATABASE IF EXISTS ${TARGET_DATABASE}; CREATE DATABASE ${TARGET_DATABASE};\"" + +echo "Importing live dump into debug DB..." +cat "${DUMP_FILE}" | docker compose exec -T "${TARGET_SERVICE}" sh -lc "MYSQL_PWD='${TARGET_PASSWORD}' mysql -u '${TARGET_USER}' '${TARGET_DATABASE}'" + +rm -f "${DUMP_FILE}" +echo "Done. Debug DB '${TARGET_DATABASE}' now mirrors live '${SOURCE_DATABASE}'."