Add remote borg backup support (#4804)

Signed-off-by: Tim Diels <tim@diels.me>
Signed-off-by: Simon L. <szaimen@e.mail.de>
Co-authored-by: Simon L. <szaimen@e.mail.de>
This commit is contained in:
Tim Diels 2024-11-07 22:19:56 +01:00 committed by GitHub
parent 34a264d945
commit 3eeda1ea91
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 402 additions and 159 deletions

View file

@ -34,19 +34,23 @@ for volume in "${DEFAULT_VOLUMES[@]}"; do
done
# Check if target is mountpoint
if ! mountpoint -q /mnt/borgbackup; then
echo "/mnt/borgbackup is not a mountpoint which is not allowed."
if [ -z "$BORG_REMOTE_REPO" ] && ! mountpoint -q "$MOUNT_DIR"; then
echo "$MOUNT_DIR is not a mountpoint which is not allowed."
exit 1
fi
# Check if target is empty
if [ "$BORG_MODE" != backup ] && [ "$BORG_MODE" != test ] && ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then
echo "The repository is empty. Cannot perform check or restore."
# Check if repo is uninitialized
if [ "$BORG_MODE" != backup ] && [ "$BORG_MODE" != test ] && ! borg info > /dev/null; then
if [ -n "$BORG_REMOTE_REPO" ]; then
echo "The repository is uninitialized or cannot connect to remote. Cannot perform check or restore."
else
echo "The repository is uninitialized. Cannot perform check or restore."
fi
exit 1
fi
# Do not continue if this file exists (needed for simple external blocking)
if [ -f "$BORG_BACKUP_DIRECTORY/aio-lockfile" ]; then
if [ -z "$BORG_REMOTE_REPO" ] && [ -f "$BORG_BACKUP_DIRECTORY/aio-lockfile" ]; then
echo "Not continuing because aio-lockfile exists it seems like a script is externally running which is locking the backup archive."
echo "If this should not be the case, you can fix this by deleting the 'aio-lockfile' file from the backup archive directory."
exit 1
@ -57,6 +61,13 @@ if [ "$BORG_MODE" = backup ] || [ "$BORG_MODE" = restore ]; then
touch "/nextcloud_aio_volumes/nextcloud_aio_database_dump/backup-is-running"
fi
if [ -n "$BORG_REMOTE_REPO" ] && ! [ -f "$BORGBACKUP_KEY" ]; then
echo "First run, creating borg ssh key"
ssh-keygen -f "$BORGBACKUP_KEY" -N ""
echo "You should configure the remote to accept this public key"
fi
echo "Your public ssh key for borgbackup is: $(cat "$BORGBACKUP_KEY.pub")"
# Do the backup
if [ "$BORG_MODE" = backup ]; then
@ -100,15 +111,22 @@ if [ "$BORG_MODE" = backup ]; then
exit 1
fi
# Create backup folder
mkdir -p "$BORG_BACKUP_DIRECTORY"
if [ -z "$BORG_REMOTE_REPO" ]; then
# Create backup folder
mkdir -p "$BORG_BACKUP_DIRECTORY"
fi
# Initialize the repository if the target is empty
if ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then
# Initialize the repository if can't get info from target
if ! borg info > /dev/null; then
# Don't initialize if already initialized
if [ -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config" ]; then
echo "No borg config file was found in the targeted directory."
echo "This might happen if the targeted directory is located on an external drive and the drive not connected anymore. You should check this."
if [ -n "$BORG_REMOTE_REPO" ]; then
echo "Borg could not get info from the remote repo."
echo "This might be a failure to connect to the remote server. See the above borg info output for details."
else
echo "Borg could not get info from the targeted directory."
echo "This might happen if the targeted directory is located on an external drive and the drive not connected anymore. You should check this."
fi
echo "If you instead want to initialize a new backup repository, you may delete the 'borg.config' file that is stored in the mastercontainer volume manually, which will allow you to initialize a new borg repository in the chosen directory:"
echo "sudo docker exec nextcloud-aio-mastercontainer rm /mnt/docker-aio-config/data/borg.config"
exit 1
@ -116,28 +134,44 @@ if [ "$BORG_MODE" = backup ]; then
echo "Initializing repository..."
NEW_REPOSITORY=1
if ! borg init --debug --encryption=repokey-blake2 "$BORG_BACKUP_DIRECTORY"; then
if ! borg init --debug --encryption=repokey-blake2; then
echo "Could not initialize borg repository."
rm -f "$BORG_BACKUP_DIRECTORY/config"
if [ -z "$BORG_REMOTE_REPO" ]; then
# Originally we checked for presence of the config file instead of calling `borg info`. Likely `borg info`
# will error on a partially initialized repo, so this line is probably no longer necessary
rm -f "$BORG_BACKUP_DIRECTORY/config"
fi
exit 1
fi
borg config "$BORG_BACKUP_DIRECTORY" additional_free_space 2G
# Fix too large Borg cache
# https://borgbackup.readthedocs.io/en/stable/faq.html#the-borg-cache-eats-way-too-much-disk-space-what-can-i-do
BORG_ID="$(borg config "$BORG_BACKUP_DIRECTORY" id)"
rm -r "/root/.cache/borg/$BORG_ID/chunks.archive.d"
touch "/root/.cache/borg/$BORG_ID/chunks.archive.d"
if [ -z "$BORG_REMOTE_REPO" ]; then
# borg config only works for local repos; it's up to the remote to ensure the disk isn't full
borg config :: additional_free_space 2G
# Make a backup from the borg config file
if ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then
echo "The borg config file wasn't created. Something is wrong."
# Fix too large Borg cache
# https://borgbackup.readthedocs.io/en/stable/faq.html#the-borg-cache-eats-way-too-much-disk-space-what-can-i-do
BORG_ID="$(borg config :: id)"
rm -r "/root/.cache/borg/$BORG_ID/chunks.archive.d"
touch "/root/.cache/borg/$BORG_ID/chunks.archive.d"
fi
if ! borg info > /dev/null; then
echo "Borg can't get info from the repo it created. Something is wrong."
exit 1
fi
rm -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config"
if ! cp "$BORG_BACKUP_DIRECTORY/config" "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config"; then
echo "Could not copy config file to second place. Cannot perform backup."
exit 1
if [ -n "$BORG_REMOTE_REPO" ]; then
# `borg config` does not support remote repos so instead create a dummy file and rely on the remote to avoid
# corruption of the config file (which contains the encryption key). We don't actually use the contents of
# this file anywhere, so a touch is all we need so we remember we already initialized the repo.
touch "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config"
else
# Make a backup from the borg config file
if ! cp "$BORG_BACKUP_DIRECTORY/config" "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/borg.config"; then
echo "Could not copy config file to second place. Cannot perform backup."
exit 1
fi
fi
echo "Repository successfully initialized."
@ -167,9 +201,9 @@ if [ "$BORG_MODE" = backup ]; then
# Create the backup
echo "Starting the backup..."
get_start_time
if ! borg create "${BORG_OPTS[@]}" "${BORG_EXCLUDE[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-nextcloud-aio" "/nextcloud_aio_volumes/"; then
if ! borg create "${BORG_OPTS[@]}" "${BORG_EXCLUDE[@]}" "::$CURRENT_DATE-nextcloud-aio" "/nextcloud_aio_volumes/" --exclude-from /borg_excludes; then
echo "Deleting the failed backup archive..."
borg delete --stats "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-nextcloud-aio"
borg delete --stats "::$CURRENT_DATE-nextcloud-aio"
echo "Backup failed!"
echo "You might want to check the backup integrity via the AIO interface."
if [ "$NEW_REPOSITORY" = 1 ]; then
@ -188,14 +222,14 @@ if [ "$BORG_MODE" = backup ]; then
# Prune archives
echo "Pruning the archives..."
if ! borg prune --stats --glob-archives '*_*-nextcloud-aio' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then
if ! borg prune --stats --glob-archives '*_*-nextcloud-aio' "${BORG_PRUNE_OPTS[@]}"; then
echo "Failed to prune archives!"
exit 1
fi
# Compact archives
echo "Compacting the archives..."
if ! borg compact "$BORG_BACKUP_DIRECTORY"; then
if ! borg compact; then
echo "Failed to compact archives!"
exit 1
fi
@ -212,19 +246,19 @@ if [ "$BORG_MODE" = backup ]; then
fi
done
echo "Starting the backup for additional volumes..."
if ! borg create "${BORG_OPTS[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-docker-volumes" "/docker_volumes/"; then
if ! borg create "${BORG_OPTS[@]}" "::$CURRENT_DATE-additional-docker-volumes" "/docker_volumes/"; then
echo "Deleting the failed backup archive..."
borg delete --stats "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-docker-volumes"
borg delete --stats "::$CURRENT_DATE-additional-docker-volumes"
echo "Backup of additional docker-volumes failed!"
exit 1
fi
echo "Pruning additional volumes..."
if ! borg prune --stats --glob-archives '*_*-additional-docker-volumes' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then
if ! borg prune --stats --glob-archives '*_*-additional-docker-volumes' "${BORG_PRUNE_OPTS[@]}"; then
echo "Failed to prune additional docker-volumes archives!"
exit 1
fi
echo "Compacting additional volumes..."
if ! borg compact "$BORG_BACKUP_DIRECTORY"; then
if ! borg compact; then
echo "Failed to compact additional docker-volume archives!"
exit 1
fi
@ -242,19 +276,19 @@ if [ "$BORG_MODE" = backup ]; then
EXCLUDE_DIRS+=(--exclude "/host_mounts/$directory/")
done
echo "Starting the backup for additional host mounts..."
if ! borg create "${BORG_OPTS[@]}" "${EXCLUDE_DIRS[@]}" "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-host-mounts" "/host_mounts/"; then
if ! borg create "${BORG_OPTS[@]}" "${EXCLUDE_DIRS[@]}" "::$CURRENT_DATE-additional-host-mounts" "/host_mounts/"; then
echo "Deleting the failed backup archive..."
borg delete --stats "$BORG_BACKUP_DIRECTORY::$CURRENT_DATE-additional-host-mounts"
borg delete --stats "::$CURRENT_DATE-additional-host-mounts"
echo "Backup of additional host-mounts failed!"
exit 1
fi
echo "Pruning additional host mounts..."
if ! borg prune --stats --glob-archives '*_*-additional-host-mounts' "${BORG_PRUNE_OPTS[@]}" "$BORG_BACKUP_DIRECTORY"; then
if ! borg prune --stats --glob-archives '*_*-additional-host-mounts' "${BORG_PRUNE_OPTS[@]}"; then
echo "Failed to prune additional host-mount archives!"
exit 1
fi
echo "Compacting additional host mounts..."
if ! borg compact "$BORG_BACKUP_DIRECTORY"; then
if ! borg compact; then
echo "Failed to compact additional host-mount archives!"
exit 1
fi
@ -276,18 +310,13 @@ fi
if [ "$BORG_MODE" = restore ]; then
get_start_time
# Perform the restore
# Pick archive to restore
if [ -n "$SELECTED_RESTORE_TIME" ]; then
SELECTED_ARCHIVE="$(borg list "$BORG_BACKUP_DIRECTORY" | grep "nextcloud-aio" | grep "$SELECTED_RESTORE_TIME" | awk -F " " '{print $1}' | head -1)"
SELECTED_ARCHIVE="$(borg list | grep "nextcloud-aio" | grep "$SELECTED_RESTORE_TIME" | awk -F " " '{print $1}' | head -1)"
else
SELECTED_ARCHIVE="$(borg list "$BORG_BACKUP_DIRECTORY" | grep "nextcloud-aio" | awk -F " " '{print $1}' | sort -r | head -1)"
SELECTED_ARCHIVE="$(borg list | grep "nextcloud-aio" | awk -F " " '{print $1}' | sort -r | head -1)"
fi
echo "Restoring '$SELECTED_ARCHIVE'..."
mkdir -p /tmp/borg
if ! borg mount "$BORG_BACKUP_DIRECTORY::$SELECTED_ARCHIVE" /tmp/borg; then
echo "Could not mount the backup!"
exit 1
fi
# Save Additional Backup dirs
if [ -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories" ]; then
@ -299,27 +328,12 @@ if [ "$BORG_MODE" = restore ]; then
DAILY_BACKUPTIME="$(cat /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_time)"
fi
# Restore everything except the configuration file
if ! rsync --stats --archive --human-readable -vv --delete \
--exclude "nextcloud_aio_apache/caddy/**" \
--exclude "nextcloud_aio_mastercontainer/caddy/**" \
--exclude "nextcloud_aio_nextcloud/data/nextcloud.log*" \
--exclude "nextcloud_aio_nextcloud/data/audit.log" \
--exclude "nextcloud_aio_mastercontainer/certs/**" \
--exclude "nextcloud_aio_mastercontainer/data/configuration.json" \
--exclude "nextcloud_aio_mastercontainer/data/daily_backup_running" \
--exclude "nextcloud_aio_mastercontainer/data/session_date_file" \
--exclude "nextcloud_aio_mastercontainer/session/**" \
/tmp/borg/nextcloud_aio_volumes/ /nextcloud_aio_volumes/; then
RESTORE_FAILED=1
echo "Something failed while restoring from backup."
fi
# Save current aio password
AIO_PASSWORD="$(jq '.password' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
# Save current path
# Save current backup location vars
BORG_LOCATION="$(jq '.borg_backup_host_location' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
REMOTE_REPO="$(jq '.borg_remote_repo' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
# Save current nextcloud datadir
if grep -q '"nextcloud_datadir":' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json; then
@ -328,21 +342,111 @@ if [ "$BORG_MODE" = restore ]; then
NEXTCLOUD_DATADIR='""'
fi
# Restore the configuration file
if ! rsync --archive --human-readable -vv \
/tmp/borg/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json \
/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json; then
RESTORE_FAILED=1
echo "Something failed while restoring the configuration.json."
if [ -z "$BORG_REMOTE_REPO" ]; then
mkdir -p /tmp/borg
if ! borg mount "::$SELECTED_ARCHIVE" /tmp/borg; then
echo "Could not mount the backup!"
exit 1
fi
# Restore everything except the configuration file
#
# These exclude patterns need to be kept in sync with the borg_excludes file and the find excludes in this file,
# which use a different syntax (patterns appear in 3 places in total)
if ! rsync --stats --archive --human-readable -vv --delete \
--exclude "nextcloud_aio_apache/caddy/**" \
--exclude "nextcloud_aio_mastercontainer/caddy/**" \
--exclude "nextcloud_aio_nextcloud/data/nextcloud.log*" \
--exclude "nextcloud_aio_nextcloud/data/audit.log" \
--exclude "nextcloud_aio_mastercontainer/certs/**" \
--exclude "nextcloud_aio_mastercontainer/data/configuration.json" \
--exclude "nextcloud_aio_mastercontainer/data/daily_backup_running" \
--exclude "nextcloud_aio_mastercontainer/data/session_date_file" \
--exclude "nextcloud_aio_mastercontainer/session/**" \
/tmp/borg/nextcloud_aio_volumes/ /nextcloud_aio_volumes/; then
RESTORE_FAILED=1
echo "Something failed while restoring from backup."
fi
# Restore the configuration file
if ! rsync --archive --human-readable -vv \
/tmp/borg/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json \
/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json; then
RESTORE_FAILED=1
echo "Something failed while restoring the configuration.json."
fi
if ! umount /tmp/borg; then
echo "Failed to unmount the borg archive but should still be able to restore successfully"
fi
else
# Restore nearly everything
#
# borg mount is really slow for remote repos (did not check whether it's slow for local repos too),
# using extract to /tmp would require temporarily storing a second copy of the data.
# So instead extract directly on top of the destination with exclude patterns for the config, but
# then we do still need to delete local files which are not present in the archive.
#
# Older backups may still contain files we've since excluded, so we have to exclude on extract as well.
cd / # borg extract has no destination arg and extracts to CWD
if ! borg extract "::$SELECTED_ARCHIVE" --progress --exclude-from /borg_excludes --pattern '+nextcloud_aio_volumes/**'
then
RESTORE_FAILED=1
echo "Failed to extract backup archive."
else
# Delete files/dirs present locally, but not in the backup archive, excluding conf files
# https://unix.stackexchange.com/a/759341
# This comm does not support -z, but I doubt any file names would have \n in them
echo "Deleting local files which do not exist in the backup"
# These find patterns need to be kept in sync with the borg_excludes file and the rsync excludes in this
# file, which use a different syntax (patterns appear in 3 places in total)
if ! find nextcloud_aio_volumes \
-not \( \
-path nextcloud_aio_volumes/nextcloud_aio_apache/caddy \
-o -path "nextcloud_aio_volumes/nextcloud_aio_apache/caddy/*" \
-o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/caddy \
-o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/caddy/*" \
-o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/certs \
-o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/certs/*" \
-o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/session \
-o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/session/*" \
-o -path "nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/nextcloud.log*" \
-o -path nextcloud_aio_volumes/nextcloud_aio_nextcloud/data/audit.log \
-o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_running \
-o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/session_date_file \
-o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/id_borg*" \
\) \
| LC_ALL=C sort \
| LC_ALL=C comm -23 - \
<(borg list "::$SELECTED_ARCHIVE" --short --exclude-from /borg_excludes --pattern '+nextcloud_aio_volumes/**' | LC_ALL=C sort) \
> /tmp/local_files_not_in_backup
then
RESTORE_FAILED=1
echo "Failed to delete local files not in backup archive."
else
# More robust than e.g. xargs as I got a ~"args line too long" error while testing that, but it's slower
# https://stackoverflow.com/a/21848934
while IFS= read -r file
do rm -vrf -- "$file" || DELETE_FAILED=1
done < /tmp/local_files_not_in_backup
if [ "$DELETE_FAILED" = 1 ]; then
RESTORE_FAILED=1
echo "Failed to delete (some) local files not in backup archive."
fi
fi
fi
fi
# Set backup-mode to restore since it was a restore
CONTENTS="$(jq '."backup-mode" = "restore"' /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
echo -E "${CONTENTS}" > /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json
# Reset the backup path to the currently used one
# Reset the backup location vars to the currently used one
CONTENTS="$(jq ".borg_backup_host_location = $BORG_LOCATION" /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
echo -E "${CONTENTS}" > /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json
CONTENTS="$(jq ".borg_remote_repo = $REMOTE_REPO" /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
echo -E "${CONTENTS}" > /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json
# Reset the AIO password to the currently used one
CONTENTS="$(jq ".password = $AIO_PASSWORD" /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/configuration.json)"
@ -366,8 +470,6 @@ if [ "$BORG_MODE" = restore ]; then
chmod 770 "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_time"
fi
umount /tmp/borg
if [ "$RESTORE_FAILED" = 1 ]; then
exit 1
fi
@ -394,7 +496,7 @@ if [ "$BORG_MODE" = check ]; then
echo "Checking the backup integrity..."
# Perform the check
if ! borg check -v --verify-data "$BORG_BACKUP_DIRECTORY"; then
if ! borg check -v --verify-data; then
echo "Some errors were found while checking the backup integrity!"
echo "Check the AIO interface for advices on how to proceed now!"
exit 1
@ -412,7 +514,7 @@ if [ "$BORG_MODE" = "check-repair" ]; then
echo "Checking the backup integrity and repairing it..."
# Perform the check-repair
if ! echo YES | borg check -v --repair "$BORG_BACKUP_DIRECTORY"; then
if ! echo YES | borg check -v --repair; then
echo "Some errors were found while checking and repairing the backup integrity!"
exit 1
fi
@ -425,19 +527,29 @@ fi
# Do the backup test
if [ "$BORG_MODE" = test ]; then
if ! [ -d "$BORG_BACKUP_DIRECTORY" ]; then
echo "No 'borg' directory in the given backup directory found!"
echo "Only the files/folders below have been found in the given directory."
ls -a "$MOUNT_DIR"
echo "Please adjust the directory so that the borg archive is positioned in a folder named 'borg' inside the given directory!"
exit 1
elif ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then
echo "A 'borg' directory was found but could not find the borg archive."
echo "Only the files/folders below have been found in the borg directory."
ls -a "$BORG_BACKUP_DIRECTORY"
echo "The archive and most importantly the config file must be positioned directly in the 'borg' subfolder."
exit 1
elif ! borg list "$BORG_BACKUP_DIRECTORY"; then
if [ -n "$BORG_REMOTE_REPO" ]; then
if ! borg info > /dev/null; then
echo "Borg could not get info from the remote repo."
echo "See the above borg info output for details."
exit 1
fi
else
if ! [ -d "$BORG_BACKUP_DIRECTORY" ]; then
echo "No 'borg' directory in the given backup directory found!"
echo "Only the files/folders below have been found in the given directory."
ls -a "$MOUNT_DIR"
echo "Please adjust the directory so that the borg archive is positioned in a folder named 'borg' inside the given directory!"
exit 1
elif ! [ -f "$BORG_BACKUP_DIRECTORY/config" ]; then
echo "A 'borg' directory was found but could not find the borg archive."
echo "Only the files/folders below have been found in the borg directory."
ls -a "$BORG_BACKUP_DIRECTORY"
echo "The archive and most importantly the config file must be positioned directly in the 'borg' subfolder."
exit 1
fi
fi
if ! borg list; then
echo "The entered path seems to be valid but could not open the backup archive."
echo "Most likely the entered password was wrong so please adjust it accordingly!"
exit 1