From ca8688f692cc57c332f834288a637f3aaee78705 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 10:17:05 -0600 Subject: [PATCH 01/13] fix docker name --- backup_database.sh | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index ec6a957..01e71c4 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -24,14 +24,20 @@ if [[ (-z ${DB_NAME}) || (-z ${DB_USERNAME}) ]] ; then exit 1 fi +if [ -z ${S3_BUCKET} ]; then + echo "S3_BUCKET is not set" + echo "[ERROR]-S3_BUCKET-not-configured" + exit 1 +fi + # currently the backup directory is hard coded BACKUP_DIR=/home/ubuntu/buildingsync-website-backups mkdir -p ${BACKUP_DIR} # db_password is set from the environment variables in docker-compose. The docker stack must # be running for this command to work. -echo "docker exec $(docker ps -f "name=buildingsyncwebsite_db-postgres" --format "{{.ID}}") pg_dump -U ${DB_USERNAME} -Fc ${DB_NAME} > $(file_name)" -docker exec $(docker ps -f "name=buildingsyncwebsite_db-postgres" --format "{{.ID}}") pg_dump -U ${DB_USERNAME} -Fc ${DB_NAME} > $(file_name) +echo "docker exec $(docker ps -f "name=buildingsync-website-db-postgres-1" --format "{{.ID}}") pg_dump -U ${DB_USERNAME} -Fc ${DB_NAME} > $(file_name)" +docker exec $(docker ps -f "name=buildingsync-website-db-postgres-1" --format "{{.ID}}") pg_dump -U ${DB_USERNAME} -Fc ${DB_NAME} > $(file_name) # Backup the media directory (uploads, especially buildingsync). In docker-land this is # just a container volume, so create a new container with the volume attached and tar it up. @@ -41,3 +47,32 @@ docker run --rm -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ # Delete files older than 30 days. find ${BACKUP_DIR} -mtime +30 -type f -name '*.dump' -delete find ${BACKUP_DIR} -mtime +30 -type f -name '*.tgz' -delete + +# upload to s3 + +for file in $BACKUP_DIR/*.dump +do + echo "Backing up $file to $S3_BUCKET/$RUN_DATE/" + if [ ! -s $file ]; then + # the file is empty, send an error + echo "[ERROR]-PostgreSQL-backup-file-was-empty-or-missing" + else + # can't pass spaces to slack notifications, for now + aws s3 cp $file $S3_BUCKET/$RUN_DATE/ + echo "[SUCCESS]-PostgreSQL-uploaded-to-$S3_BUCKET/$RUN_DATE/$(basename $file)" + fi +done + +for file in $BACKUP_DIR/*.tgz +do + echo "Backing up $file $S3_BUCKET/$RUN_DATE/" + + if [ ! -s $file ]; then + # the file is empty, send an error + echo "[ERROR]-Mediadata-backup-file-was-empty-or-missing" + else + # can't pass spaces to slack notifications, for now + aws s3 cp $file $S3_BUCKET/$RUN_DATE/ + echo "[SUCCESSs]-Mediadata-uploaded-to-$S3_BUCKET/$RUN_DATE/$(basename $file)" + fi +done \ No newline at end of file From e931b032980ba811f65ab93d597a99ed30c7ccab Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 10:46:47 -0600 Subject: [PATCH 02/13] rundate --- backup_database.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index 01e71c4..2f0fcf7 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -30,6 +30,9 @@ if [ -z ${S3_BUCKET} ]; then exit 1 fi +# get the run date to save as the s3 folder name +RUN_DATE=$(date +%Y-%m-%d) + # currently the backup directory is hard coded BACKUP_DIR=/home/ubuntu/buildingsync-website-backups mkdir -p ${BACKUP_DIR} @@ -52,20 +55,20 @@ find ${BACKUP_DIR} -mtime +30 -type f -name '*.tgz' -delete for file in $BACKUP_DIR/*.dump do - echo "Backing up $file to $S3_BUCKET/$RUN_DATE/" + echo "Backing up $file to $S3_BUCKET/buildingsync-website/$RUN_DATE/" if [ ! -s $file ]; then # the file is empty, send an error echo "[ERROR]-PostgreSQL-backup-file-was-empty-or-missing" else # can't pass spaces to slack notifications, for now aws s3 cp $file $S3_BUCKET/$RUN_DATE/ - echo "[SUCCESS]-PostgreSQL-uploaded-to-$S3_BUCKET/$RUN_DATE/$(basename $file)" + echo "[SUCCESS]-PostgreSQL-uploaded-to-$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" fi done for file in $BACKUP_DIR/*.tgz do - echo "Backing up $file $S3_BUCKET/$RUN_DATE/" + echo "Backing up $file to $S3_BUCKET/buildingsync-website/$RUN_DATE/" if [ ! -s $file ]; then # the file is empty, send an error @@ -73,6 +76,6 @@ do else # can't pass spaces to slack notifications, for now aws s3 cp $file $S3_BUCKET/$RUN_DATE/ - echo "[SUCCESSs]-Mediadata-uploaded-to-$S3_BUCKET/$RUN_DATE/$(basename $file)" + echo "[SUCCESSs]-Mediadata-uploaded-to-$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" fi done \ No newline at end of file From 14c7c5ebef2062eef5815951e52ece6a595c7cdc Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 10:49:15 -0600 Subject: [PATCH 03/13] filename --- backup_database.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index 2f0fcf7..b62f461 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -30,13 +30,21 @@ if [ -z ${S3_BUCKET} ]; then exit 1 fi -# get the run date to save as the s3 folder name -RUN_DATE=$(date +%Y-%m-%d) - # currently the backup directory is hard coded BACKUP_DIR=/home/ubuntu/buildingsync-website-backups mkdir -p ${BACKUP_DIR} +# get the run date to save as the s3 folder name +RUN_DATE=$(date +%Y-%m-%d) + +function file_name(){ + echo ${BACKUP_DIR}/${DB_NAME}_$(date '+%Y%m%d_%H%M%S').dump +} + +function media_file_name(){ + echo ${BACKUP_DIR}/${DB_NAME}_media_$(date '+%Y%m%d_%H%M%S').tgz +} + # db_password is set from the environment variables in docker-compose. The docker stack must # be running for this command to work. echo "docker exec $(docker ps -f "name=buildingsync-website-db-postgres-1" --format "{{.ID}}") pg_dump -U ${DB_USERNAME} -Fc ${DB_NAME} > $(file_name)" From db94b8bf068b07d4bd624c22dee3c6c20ce14fcc Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:02:51 -0600 Subject: [PATCH 04/13] filename --- backup_database.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index b62f461..4b297e3 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -11,14 +11,6 @@ DB_NAME=$1 DB_USERNAME=$2 -function file_name(){ - echo ${BACKUP_DIR}/${DB_NAME}_$(date '+%Y%m%d_%H%M%S').dump -} - -function media_file_name(){ - echo /backup/dir/${DB_NAME}_media_$(date '+%Y%m%d_%H%M%S').tgz -} - if [[ (-z ${DB_NAME}) || (-z ${DB_USERNAME}) ]] ; then echo "Expecting command to be of form ./backup_database.sh " exit 1 From 1a1205a7739a63da24138b0319fbe92c323f83e5 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:06:19 -0600 Subject: [PATCH 05/13] fix s3 name --- backup_database.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index 4b297e3..8aa5fd7 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -52,7 +52,6 @@ find ${BACKUP_DIR} -mtime +30 -type f -name '*.dump' -delete find ${BACKUP_DIR} -mtime +30 -type f -name '*.tgz' -delete # upload to s3 - for file in $BACKUP_DIR/*.dump do echo "Backing up $file to $S3_BUCKET/buildingsync-website/$RUN_DATE/" @@ -61,8 +60,8 @@ do echo "[ERROR]-PostgreSQL-backup-file-was-empty-or-missing" else # can't pass spaces to slack notifications, for now - aws s3 cp $file $S3_BUCKET/$RUN_DATE/ - echo "[SUCCESS]-PostgreSQL-uploaded-to-$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" + aws s3 cp $file s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/ + echo "[SUCCESS]-PostgreSQL-uploaded-to-s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" fi done @@ -75,7 +74,7 @@ do echo "[ERROR]-Mediadata-backup-file-was-empty-or-missing" else # can't pass spaces to slack notifications, for now - aws s3 cp $file $S3_BUCKET/$RUN_DATE/ - echo "[SUCCESSs]-Mediadata-uploaded-to-$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" + aws s3 cp $file s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/ + echo "[SUCCESS]-Mediadata-uploaded-to-s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" fi done \ No newline at end of file From 8566c654d34e3230d93e97e294717bfbaa0d5dde Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:09:19 -0600 Subject: [PATCH 06/13] backup media name --- backup_database.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index 8aa5fd7..15d285c 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -44,8 +44,10 @@ docker exec $(docker ps -f "name=buildingsync-website-db-postgres-1" --format "{ # Backup the media directory (uploads, especially buildingsync). In docker-land this is # just a container volume, so create a new container with the volume attached and tar it up. -echo "docker run --rm -it -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf $(media_file_name) /backup/media" -docker run --rm -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf $(media_file_name) /backup/media +MEDIA_BACKUP_FILE=$(media_file_name) +MEDIA_BACKUP_BASENAME=$(basename "$MEDIA_BACKUP_FILE") +echo "docker run --rm -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media" +docker run --rm -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media # Delete files older than 30 days. find ${BACKUP_DIR} -mtime +30 -type f -name '*.dump' -delete From de1598f723e7a385e3e14d879bb21e16ab9916e0 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:15:02 -0600 Subject: [PATCH 07/13] source buildignsync env vars --- backup_database.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backup_database.sh b/backup_database.sh index 15d285c..7d10f06 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -1,5 +1,7 @@ #!/bin/bash + + # This backup script creates nightly database and media file backups of the BuildingSync website when running # in a docker container. The name of the container running the database is hardcoded to look for # *db-postgres*. This may cause an issue if several docker applications are running on the same @@ -8,8 +10,17 @@ # To create nightly backups, add the following to your crontab # 0 0 * * * /srv/buildingsync-website/backup_database.sh >> /home/ubuntu/buildingsync-website-backups/cron.log 2>&1 +# Test with the crontab entry -- should run every minute +# * * * * * /srv/buildingsync-website/backup_database.sh buildingsync-website buildingsync-website >> /home/ubuntu/buildingsync-website-backups/cron.log 2>&1 + +# Source environment variables if the file exists, will read in s3_bucket and PWs as needed +if [ -f /etc/profile.d/buildingsync.sh ]; then + source /etc/profile.d/buildingsync.sh +fi + DB_NAME=$1 DB_USERNAME=$2 +S3_BUCKET=${S3_BUCKET:-"nrel-aws-buildings-backups"} if [[ (-z ${DB_NAME}) || (-z ${DB_USERNAME}) ]] ; then echo "Expecting command to be of form ./backup_database.sh " From fd200fedc92f7f7b1c5c87216047922a45edbb0e Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:19:16 -0600 Subject: [PATCH 08/13] user and group passed --- backup_database.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index 7d10f06..584514a 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -57,8 +57,11 @@ docker exec $(docker ps -f "name=buildingsync-website-db-postgres-1" --format "{ # just a container volume, so create a new container with the volume attached and tar it up. MEDIA_BACKUP_FILE=$(media_file_name) MEDIA_BACKUP_BASENAME=$(basename "$MEDIA_BACKUP_FILE") -echo "docker run --rm -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media" -docker run --rm -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media +# Run as the current user to avoid ownership issues, ubuntu:ubuntu +CURRENT_UID=$(id -u) +CURRENT_GID=$(id -g) +echo "docker run --rm --user $CURRENT_UID:$CURRENT_GID -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media" +docker run --rm --user $CURRENT_UID:$CURRENT_GID -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media # Delete files older than 30 days. find ${BACKUP_DIR} -mtime +30 -type f -name '*.dump' -delete From 8b74a0d71b6a584cf7065fd924fd3dcb4ff84fef Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:20:27 -0600 Subject: [PATCH 09/13] more recent alpine version --- backup_database.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index 584514a..e2bbfb5 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -60,8 +60,8 @@ MEDIA_BACKUP_BASENAME=$(basename "$MEDIA_BACKUP_FILE") # Run as the current user to avoid ownership issues, ubuntu:ubuntu CURRENT_UID=$(id -u) CURRENT_GID=$(id -g) -echo "docker run --rm --user $CURRENT_UID:$CURRENT_GID -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media" -docker run --rm --user $CURRENT_UID:$CURRENT_GID -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.8 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media +echo "docker run --rm --user $CURRENT_UID:$CURRENT_GID -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.19 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media" +docker run --rm --user $CURRENT_UID:$CURRENT_GID -v buildingsync_media:/backup/media -v $BACKUP_DIR:/backup/dir/ alpine:3.19 tar zcvf /backup/dir/$MEDIA_BACKUP_BASENAME /backup/media # Delete files older than 30 days. find ${BACKUP_DIR} -mtime +30 -type f -name '*.dump' -delete From fb9394d031fa22c260da632f4cf1caae52aba021 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:26:22 -0600 Subject: [PATCH 10/13] add path for aws --- backup_database.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/backup_database.sh b/backup_database.sh index e2bbfb5..eb065a1 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -1,6 +1,7 @@ #!/bin/bash - +# Set PATH for cron compatibility - cron has a very limited PATH +export PATH="/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin:$PATH" # This backup script creates nightly database and media file backups of the BuildingSync website when running # in a docker container. The name of the container running the database is hardcoded to look for @@ -33,6 +34,17 @@ if [ -z ${S3_BUCKET} ]; then exit 1 fi +# Check if AWS CLI is installed +if ! command -v aws &> /dev/null; then + echo "AWS CLI is not installed or not in PATH" + echo "[ERROR]-AWS-CLI-not-found" + echo "To install AWS CLI:" + echo " curl 'https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip' -o 'awscliv2.zip'" + echo " unzip awscliv2.zip" + echo " sudo ./aws/install" + exit 1 +fi + # currently the backup directory is hard coded BACKUP_DIR=/home/ubuntu/buildingsync-website-backups mkdir -p ${BACKUP_DIR} From d952c4c294dae4c30ce31e57f31f2a5716b69d9a Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:32:36 -0600 Subject: [PATCH 11/13] fix path --- backup_database.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index eb065a1..2b6379d 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Set PATH for cron compatibility - cron has a very limited PATH -export PATH="/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin:$PATH" +# Set PATH for cron compatibility - cron has a very limited PATH, .local contains AWS CLI +export PATH="/usr/local/bin:/home/ubuntu/.local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin:$PATH" # This backup script creates nightly database and media file backups of the BuildingSync website when running # in a docker container. The name of the container running the database is hardcoded to look for From 03235517873b69d5ce822ea3aa68fd2070a40351 Mon Sep 17 00:00:00 2001 From: Nicholas Long <1907354+nllong@users.noreply.github.com> Date: Mon, 29 Sep 2025 11:36:59 -0600 Subject: [PATCH 12/13] Update backup_database.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- backup_database.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backup_database.sh b/backup_database.sh index 2b6379d..2627b53 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -80,29 +80,29 @@ find ${BACKUP_DIR} -mtime +30 -type f -name '*.dump' -delete find ${BACKUP_DIR} -mtime +30 -type f -name '*.tgz' -delete # upload to s3 -for file in $BACKUP_DIR/*.dump +for file in "$BACKUP_DIR"/*.dump do echo "Backing up $file to $S3_BUCKET/buildingsync-website/$RUN_DATE/" - if [ ! -s $file ]; then + if [ ! -s "$file" ]; then # the file is empty, send an error echo "[ERROR]-PostgreSQL-backup-file-was-empty-or-missing" else # can't pass spaces to slack notifications, for now - aws s3 cp $file s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/ - echo "[SUCCESS]-PostgreSQL-uploaded-to-s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" + aws s3 cp "$file" "s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/" + echo "[SUCCESS]-PostgreSQL-uploaded-to-s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename "$file")" fi done -for file in $BACKUP_DIR/*.tgz +for file in "$BACKUP_DIR"/*.tgz do echo "Backing up $file to $S3_BUCKET/buildingsync-website/$RUN_DATE/" - if [ ! -s $file ]; then + if [ ! -s "$file" ]; then # the file is empty, send an error echo "[ERROR]-Mediadata-backup-file-was-empty-or-missing" else # can't pass spaces to slack notifications, for now - aws s3 cp $file s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/ - echo "[SUCCESS]-Mediadata-uploaded-to-s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename $file)" + aws s3 cp "$file" "s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/" + echo "[SUCCESS]-Mediadata-uploaded-to-s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename "$file")" fi done \ No newline at end of file From 555b58655cd1abf5a68b061c8c566abf3e265586 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 29 Sep 2025 11:41:36 -0600 Subject: [PATCH 13/13] fix precommit --- backup_database.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup_database.sh b/backup_database.sh index 2627b53..891b414 100755 --- a/backup_database.sh +++ b/backup_database.sh @@ -105,4 +105,4 @@ do aws s3 cp "$file" "s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/" echo "[SUCCESS]-Mediadata-uploaded-to-s3://$S3_BUCKET/buildingsync-website/$RUN_DATE/$(basename "$file")" fi -done \ No newline at end of file +done