Skip to content

Commit eedf436

Browse files
Rub21Rub21
committed
Update script for minute replication
1 parent 5103747 commit eedf436

File tree

1 file changed

+62
-24
lines changed

1 file changed

+62
-24
lines changed

‎images/replication-job/start.sh‎

Lines changed: 62 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,21 @@ set -euo pipefail
44
# ============================================================================
55
# OSM Replication Script with osmdbt and S3
66
# ============================================================================
7-
# This script manages OSM replication using osmdbt tools:
8-
# - Executes osmdbt-get-log and osmdbt-create-diff every minute
7+
# This script manages OSM replication using osmdbt tools following the
8+
# official osmdbt workflow (https://github.com/openstreetmap/osmdbt):
9+
#
10+
# Initialization:
11+
# - Runs osmdbt-enable-replication to set up replication slot
12+
# - Requires PostgreSQL with wal_level=logical, max_replication_slots >= 1,
13+
# and a user with REPLICATION attribute
14+
#
15+
# Replication Cycle (every minute):
16+
# 1. osmdbt-catchup - Process old log files if any (from crashes)
17+
# 2. osmdbt-get-log - Fetch new changes from PostgreSQL logical replication
18+
# 3. osmdbt-catchup - Update database to new log file
19+
# 4. osmdbt-create-diff - Generate .osc.gz and state.txt files
20+
#
21+
# Additional Features:
922
# - Uploads .osc.gz and state.txt files to S3 after integrity verification
1023
# * General state.txt (in root directory) - controls replication sequence
1124
# * Specific state.txt files (e.g., 870.state.txt for 870.osc.gz)
@@ -69,31 +82,33 @@ run_dir: ${runDirectory}
6982
EOF
7083

7184
# ============================================================================
72-
# Function: Initialize replication slot and environment
85+
# Function: Enable osmdbt replication (creates slot if needed)
7386
# ============================================================================
74-
function ensure_replication_slot_exists() {
75-
export PGPASSWORD="${POSTGRES_PASSWORD}"
87+
function enable_osmdbt_replication() {
88+
echo "$(date +%F_%H:%M:%S): Enabling osmdbt replication..."
7689

90+
# Check if replication is already enabled by checking for the slot
91+
export PGPASSWORD="${POSTGRES_PASSWORD}"
7792
local exists=$(psql -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}" \
7893
-U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -t -A -c \
7994
"SELECT count(*) FROM pg_replication_slots WHERE slot_name='$REPLICATION_SLOT';" 2>/dev/null | tr -d '[:space:]')
8095

8196
if [ "$exists" = "1" ]; then
82-
echo "$(date +%F_%H:%M:%S): Replication slot '$REPLICATION_SLOT' already exists."
97+
echo "$(date +%F_%H:%M:%S): Replication slot '$REPLICATION_SLOT' already exists. Replication should be enabled."
8398
return 0
8499
fi
85100

86-
echo "$(date +%F_%H:%M:%S): Creating replication slot '$REPLICATION_SLOT' with plugin '$REPLICATION_PLUGIN'..."
87-
if psql -h "${POSTGRES_HOST}" -p "${POSTGRES_PORT}" \
88-
-U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -c \
89-
"SELECT * FROM pg_create_logical_replication_slot('$REPLICATION_SLOT', '$REPLICATION_PLUGIN');" >/dev/null 2>&1; then
90-
echo "$(date +%F_%H:%M:%S): Successfully created replication slot '$REPLICATION_SLOT'."
101+
# Use osmdbt-enable-replication to set up replication properly
102+
echo "$(date +%F_%H:%M:%S): Running osmdbt-enable-replication..."
103+
if /osmdbt/build/src/osmdbt-enable-replication -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-enable-replication.log"; then
104+
echo "$(date +%F_%H:%M:%S): Successfully enabled osmdbt replication."
91105
return 0
106+
else
107+
local error_msg="ERROR: Failed to enable osmdbt replication. Check PostgreSQL configuration (wal_level=logical, max_replication_slots >= 1, user with REPLICATION attribute)."
108+
echo "$(date +%F_%H:%M:%S): $error_msg"
109+
send_slack_message "🚨 ${ENVIROMENT:-production}: $error_msg"
110+
return 1
92111
fi
93-
94-
local error_msg="ERROR: Failed to create replication slot '$REPLICATION_SLOT'. The '$REPLICATION_PLUGIN' plugin may not be installed."
95-
echo "$error_msg"
96-
return 1
97112
}
98113

99114
# ============================================================================
@@ -401,26 +416,50 @@ function verify_sequence_continuity() {
401416
}
402417

403418
# ============================================================================
404-
# Function: Execute osmdbt replication cycle
419+
# Function: Execute osmdbt replication cycle (following official osmdbt workflow)
420+
# ============================================================================
421+
# Official workflow per osmdbt documentation:
422+
# 1. osmdbt-catchup (process old log files if any)
423+
# 2. osmdbt-get-log (get new changes)
424+
# 3. osmdbt-catchup (update database to new log file)
425+
# 4. osmdbt-create-diff (create OSM change files)
405426
# ============================================================================
406427
function execute_replication_cycle() {
407-
echo "$(date +%F_%H:%M:%S): Starting replication cycle..."
428+
echo "$(date +%F_%H:%M:%S): Starting replication cycle (following osmdbt official workflow)..."
408429

409430
# Clean up any existing lock files
410431
find "$workingDirectory" -name "replicate.lock" -delete
411432
find "$runDirectory" -name "*.lock" -delete
412433

413-
# Execute osmdbt-get-log
414-
echo "$(date +%F_%H:%M:%S): Running osmdbt-get-log..."
434+
# Step 1: Catch up old log files (if there are complete log files left over from a crash)
435+
echo "$(date +%F_%H:%M:%S): Step 1: Running osmdbt-catchup (processing old log files if any)..."
436+
if ! /osmdbt/build/src/osmdbt-catchup -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-catchup.log"; then
437+
local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-catchup (step 1) failed"
438+
echo "$(date +%F_%H:%M:%S): $error_msg"
439+
send_slack_message "$error_msg"
440+
return 1
441+
fi
442+
443+
# Step 2: Get new log file with changes
444+
echo "$(date +%F_%H:%M:%S): Step 2: Running osmdbt-get-log (fetching new changes)..."
415445
if ! /osmdbt/build/src/osmdbt-get-log -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-get-log.log"; then
416446
local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-get-log failed"
417447
echo "$(date +%F_%H:%M:%S): $error_msg"
418448
send_slack_message "$error_msg"
419449
return 1
420450
fi
421451

422-
# Execute osmdbt-create-diff
423-
echo "$(date +%F_%H:%M:%S): Running osmdbt-create-diff..."
452+
# Step 3: Catch up database to new log file
453+
echo "$(date +%F_%H:%M:%S): Step 3: Running osmdbt-catchup (updating database to new log file)..."
454+
if ! /osmdbt/build/src/osmdbt-catchup -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-catchup.log"; then
455+
local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-catchup (step 3) failed"
456+
echo "$(date +%F_%H:%M:%S): $error_msg"
457+
send_slack_message "$error_msg"
458+
return 1
459+
fi
460+
461+
# Step 4: Create OSM diff files from log files
462+
echo "$(date +%F_%H:%M:%S): Step 4: Running osmdbt-create-diff (creating OSM change files)..."
424463
if ! /osmdbt/build/src/osmdbt-create-diff -c "$osmdbtConfig" 2>&1 | tee -a "${logDirectory}/osmdbt-create-diff.log"; then
425464
local error_msg="🚨 ${ENVIROMENT:-production}: osmdbt-create-diff failed"
426465
echo "$(date +%F_%H:%M:%S): $error_msg"
@@ -501,8 +540,8 @@ function main() {
501540
exit 1
502541
fi
503542

504-
# Ensure replication slot exists
505-
if ! ensure_replication_slot_exists; then
543+
# Enable osmdbt replication (creates slot and sets up replication)
544+
if ! enable_osmdbt_replication; then
506545
exit 1
507546
fi
508547

@@ -543,5 +582,4 @@ function main() {
543582
done
544583
}
545584

546-
# Run main function
547585
main

0 commit comments

Comments
 (0)