#!/bin/bash - 
# {{{ Header ===================================================================
    #
    #          FILE: /hana/scripts/backup.sh
    # 
    #         USAGE: ./backup.sh
    #
    #   DESCRIPTION: Script for backup of Hana databases:
    #                - performs 2 backup of the Hana databases
    #                  1. Starts backup procedure of complete HANA databases like it
    #                     is performed by Hana Studio
    #                  2. Performs a backup of individual databases (not implemented here yet)
    #                - copies backup files to isystems cloud
    #                - uses a configuration file for the settings: backup.conf
    #                  NFS-Share basierend auf der MAC-Adresse/Servername?
    #                - normally there is no need to change this script, everything
    #                  needed can be set in backup.conf
    #                - this script is written under COMMON LICENSE (by-SA)
    #                - usage and changes are subject to own risk
    #
    #       OPTIONS: ---
    #
    #  REQUIREMENTS: - settings in file ./backup.conf
    #
    #          BUGS: - when smbBACKUP=1, tgz is created on smbMOUNTPOINT
    #                - tgz not cleaned up, using crontab for cleanup
    #                - BCKH2 user has no permission to reclaim log (LOG ADMIN) and defragment (RESOURCE ADMIN) 
    #                - it should be possible to "extract here" without the need of using the mounting tool in sles
    #         NOTES: ---
    #        AUTHOR: Klaus Joerissen (KJ)
    #                Steffen Kamphoff (SK)
    #  ORGANIZATION: iSystems Consulting e.K.
    #       CREATED:      02.08.16 23:39
    #      REVISION: 3.01 05.08.20 16:22
	#      REVISION: 3.02 03.06.21 18:14
	#      REVISION: 3.03 08.09.21 13:37
    #      REVISION: 3.04 30.09.22 13:37
    #      REVISION="3.05 12.05.23 13:37"
           REVISION="3.06 22.07.23 13:37"
    #     IMPORTANT: backup.conf changed from rev. 2.14 to 2.15 to 2.16
    #
    # {{{   CHANGES: ===============================================================
        # REVISION 
		#          3.06  - adding 'ALTER SYSTEM RECLAIM COLUMN LOB SPAC' and 'ALTER SYSTEM RECLAIM ROW LOB SPACE' to cleanup process
		#          3.05  - using 'systemids' which are active, excluding inactive
		#          3.04  - "NOW" is MINUTE now, not HOUR
		#                - using 'systemids' now (from sql select), not 'systemid' (from config)
		#                - check for backup id before and after taking a backup
		#                - select last backup id based on number of backups, not time
		#                - fix defragment datavolume 
		#                - fix reclaim log
		#                - no need to compress when both isysCLOUD and smbBACKUP = 0
		#                - create tgz in HANA_backup_dir, copy it to smbMOUNTPOINT, remove it from HANA_backup_dir
		#                - logging updates
		#          3.03  - added cleanup task placeholder
		#          3.02  - add some single quotation sign to solve $i issue in backup name
        #          3.01  - add [[]] in part "send email"
        #          3.00  - changed settings for Hana 2
        #          2.22  - added if-clause for mail sending without authentification
        #          2.21  - added checks for existance of backup directories again
        #          2.20  - added support for more than one backup per day
        #          2.19  - using compression for backup files to backupshare
        #                - using rsync for upload to isystems cloud
        #          2.18  - added maxAGE and Schema/Iggy variables in automatic installation procedure
        #          2.17  - set cluster check to the beginning of the script
        #          2.16  - added cloud location
        #          2.15  - added cluster functionality (part 1)
        #          2.14  - added automatic setting of backup path
        #          2.13  - changed order of reclaim logs and delete backup catalog
        #          2.12  - added flag for backup to smb share
        #          2.11  - added logwatch, changed smbDOMAIN usage
        #          2.10  - some cosmetics
        #          2.01  - added variable for Hana backup directory
        #                - added Revision number to Output
        #          2.00  - added copy backup to isystems cloud
        #          1.00  - added check for last logins of root
        #                - added change of directory permissions for $myTempDir
        #                  if it already exists during first run of the script
        #          0.05  - changed query for backup-ID from 
        #                  'SELECT TOP 1 min(to_bigint("BACKUP_ID")     to
        #                  'SELECT TOP 1 max(to_bigint("BACKUP_ID")
        #                - path for backup.conf changed from fixed to ./backup.conf
        #          0.04  - added listing backup directories and partitions 
        #          0.03  - completely restructured with additional settings file
        #          0.02  - added path variable in creating subdirectory
        #
        #	  TODOS: - Abfrage, ob ueberhaupt SMB-Share genutzt werden soll
        #                - Nutzung/Alternative NFS-Share anstelle von SMB
        #                - bei Copy zur iSystems Cloud:
        #                  - Ueberpruefung Quota / explizite Meldungen, wenn man sich den
        #                    entsprechenden Grenzen naehert
    # }}} ==========================================================================
# }}}

# {{{ Load settings from config file and clearing old log ======================
    source /hana/scripts/backup.conf    # for running as a cron job an absolute
                                        # path is necessary here!!!

    echo "Backupscript Revision $REVISION " >$jobcontrol
    NOTIFY_ERR_FLAG=""
# }}}

# {{{ Checking environment =====================================================
    # Check for logging directory
    # logdir= # Ableitung aus $jobcontrol     // toDo
    # if [ ! -d "$logdir" ] ; then
    #        mkdir -p "$logdir"
    #        chmod -R 777 "$logdir"
    # fi
# }}}

# {{{ Preparations for backup ==================================================
    # Check if running in a cluster and beeing master or slave
    # the backup only runs on the master
    if [ $MyCLUSTER -eq 1 -o $MyCLUSTER -eq 2 ] ; then
        master_check="$(ip a show dev eth0 |grep "inet " |wc -l)"
        if [ $master_check -eq 1 ] ; then
            echo $(date +"%Y%m%dT%H%M%S") "I am just the slave, backup only runs on the master!" >>$jobcontrol
            if [ $SEND_EMAIL -eq 1 ] ; then
                mymac=$(ip link show dev eth0 | grep -oE 'link/ether ([a-f0-9]{2}:){5}[a-f0-9]{2}' | cut -d' ' -f2 | sed -e 's/://g' )
                SYSident="$mymac - `hostname -f`"
		if [[ $NOTIFY_PWD != "" && $NOTIFY_AUTHGERAFFEL == "" ]] ; then
	            NOTIFY_AUTHGERAFFEL="-S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user="$NOTIFY_SENDER" -S smtp-auth-password="$NOTIFY_PWD" -S ssl-verify=ignore"
                fi
		cat $jobcontrol | grep -v "rows affected" | grep -v "Removing leading" | mailx -v -r $NOTIFY_FROM -s "$NOTIFY_ERR_FLAG $SYSident - $NOTIFY_SUBJECT" -S smtp="$NOTIFY_MAILSERVER:$NOTIFY_PORT" $NOTIFY_AUTHGERAFFEL $NOTIFY_RECEIVER
            fi
	exit
        fi
    fi

    # Mount destination share for backup
    if [ $smbBACKUP -eq 1 ] ; then
		umount $smbMOUNTPOINT >/dev/null 2>&1
		if [ ! -d "$smbMOUNTPOINT" ] ; then
			mkdir -p     "$smbMOUNTPOINT"
			chmod -R 777 "$smbMOUNTPOINT"
		fi
		mount -t cifs -o user=$smbUSER@$smbDOMAIN,password=$smbPASS,vers=$smbVERS //$smbIP/$smbDIR $smbMOUNTPOINT
		ERR=$?
		if [ $ERR != 0 ] ; then
			NOTIFY_ERR_FLAG="ERROR - "
			echo $(date +"%Y%m%dT%H%M%S") ERROR - Backup volume $smbMOUNTPOINT could not be mounted - exit code: $ERR >>$jobcontrol 
		else
			echo $(date +"%Y%m%dT%H%M%S") Backup volume $smbMOUNTPOINT mounted successfully - exit code: $ERR >>$jobcontrol 
		fi
    fi

    # Check for backup temp directory
        if [ ! -d "$MyTempDir" ] ; then
            mkdir "$MyTempDir"
            chmod -R 777 "$MyTempDir"
        else
            chmod -R 777 "$MyTempDir"
        fi

    # Check destination directory for full backups
    NOW="$(date +"%Y%m%d")"
        if [ ! -d "$DEST_fbck" ] ; then
            mkdir -p "$DEST_fbck"
            chmod -R 777 "$DEST_fbck"
        fi

    # Check destination directory for particular databases backups              //// subdirectory hier?
    NOW="$(date +"%Y%m%d")"
        if [ ! -d "$DEST_partdb" ] ; then
            mkdir -p "$DEST_partdb"
            chmod -R 777 "$DEST_partdb"
        fi

    # Get hostname
    HOST="$(hostname)"

    # Set timecode for identifying backups
    NOW="$(date +"%Y%m%d_%H%M%S")"
	# }}}

	# Get hana client if config variable is wrong
	hdbsql1="/hana/shared/NDB/hdbclient/hdbsql" ; if [ -f $hdbsql1 ]; then hdbsql="$hdbsql1" ; fi
	hdbsql2="/usr/sap/hdbclient/hdbsql"         ; if [ -f $hdbsql2 ]; then hdbsql="$hdbsql2" ; fi
	
	# Get active system ids
	systemids=$($hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d SYSTEMDB -x -a -j 'SELECT DISTINCT DATABASE_NAME FROM SYS_DATABASES.M_SERVICES WHERE DATABASE_NAME != '\'''\'' AND ACTIVE_STATUS = '\''YES'\''')
	systemids=`echo "$systemids" | tr -d '"'`
	echo $(date +"%Y%m%dT%H%M%S") Using: $systemids >>$jobcontrol

	# Get inactive system ids
	inactives=$($hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d SYSTEMDB -x -a -j 'SELECT DISTINCT DATABASE_NAME FROM SYS_DATABASES.M_SERVICES WHERE DATABASE_NAME != '\'''\'' AND ACTIVE_STATUS = '\''NO'\''')
	inactives=`echo "$inactives" | tr -d '"'`
	echo $(date +"%Y%m%dT%H%M%S") Inactive: $inactives >>$jobcontrol

# {{{ Loop1: setting backup path, get latest backup-id, perform instance backup
    echo $(date +"%Y%m%dT%H%M%S") "Loop1: setting backup path, get latest backup-id, perform instance backup" >>$jobcontrol
	if [ $maxAGE_fbck -eq 0 ] ; then maxAGE_fbck="1" ; fi
    typeset -i counter ; counter=0 ; for i in $systemids ; do
		ERR=""
        counter=$counter+1
        # {{{ setting backup path
        # Check backup settings on hana server and set backup directory (will be visible in hana studio) // only for debug
        # actual_backup_datapath="$($hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'SELECT * FROM M_INIFILE_CONTENTS WHERE "FILE_NAME" = '\''global.ini'\'' and "KEY" LIKE '\''basepath_databackup'\'' ')" >/dev/null 2>&1
        # actual_backup_logpath="$($hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'SELECT * FROM M_INIFILE_CONTENTS WHERE "FILE_NAME" = '\''global.ini'\'' and "KEY" LIKE '\''basepath_logbackup'\'' ')" >/dev/null 2>&1
        # echo $(date +"%Y%m%dT%H%M%S") $i: "Actual backup datapath: $actual_backup_datapath" >>$jobcontrol
        # echo $(date +"%Y%m%dT%H%M%S") $i: "Actual backup logpath:  $actual_backup_logpath" >>$jobcontrol

        # Set backup path to given path in backup.conf (will be visible in hana studio)
        $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i 'ALTER SYSTEM ALTER CONFIGURATION ('\''global.ini'\'', '\''SYSTEM'\'') SET ('\''persistence'\'', '\''basepath_databackup'\'') = '\'''$HANA_backup_dir'/data'\'' WITH RECONFIGURE' >/dev/null 2>&1
        $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i 'ALTER SYSTEM ALTER CONFIGURATION ('\''global.ini'\'', '\''SYSTEM'\'') SET ('\''persistence'\'', '\''basepath_logbackup'\'')  = '\'''$HANA_backup_dir'/log'\''  WITH RECONFIGURE' >/dev/null 2>&1
        # }}}

        # {{{ Perform Backup HANA System complete backup (like from Hana Studio) =======
        echo $(date +"%Y%m%dT%H%M%S") $i: Backup HANA System starting >>$jobcontrol
		maxAGE="" ; if [ $maxAGE_fbck -gt 1 ] ; then (( maxAGE=$maxAGE_fbck-1 )) ; else maxAGE=1 ; fi ;
		ID[$counter]="$($hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'WITH BACKUPS AS (SELECT *, ROW_NUMBER() OVER (ORDER BY "SYS_START_TIME" DESC) AS "ROWNUM" FROM (SELECT "SYS_START_TIME", "BACKUP_ID" FROM "SYS"."M_BACKUP_CATALOG" WHERE "ENTRY_TYPE_NAME" = '\''complete data backup'\'' AND "STATE_NAME" = '\''successful'\'')) SELECT "BACKUP_ID" FROM BACKUPS where ROWNUM='$maxAGE';')"
                                          $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS  -d $i -x -a -j 'BACKUP CATALOG DELETE ALL BEFORE BACKUP_ID '${ID[$counter]}' COMPLETE;ALTER SYSTEM RECLAIM LOG;ALTER SYSTEM RECLAIM DATAVOLUME 120 DEFRAGMENT;' >/dev/null 2>&1
        echo $(date +"%Y%m%dT%H%M%S") $i:"$hdbsql -i $instance -n $MyHOST -u $MyUSER -p ******** -d $i -x -a -j 'BACKUP DATA USING FILE ('\'$NOW'_'$i'_COMPLETE_DATA_BACKUP'\'') '" >>$jobcontrol
                                          $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS  -d $i -x -a -j 'BACKUP DATA USING FILE ('\'$NOW'_'$i'_COMPLETE_DATA_BACKUP'\'') '
        ERR=$?
            if [ $ERR != 0 ] ; then
                NOTIFY_ERR_FLAG="ERROR - "
                echo $(date +"%Y%m%dT%H%M%S") $i: ERROR - Backup HANA System exit code: $ERR >>$jobcontrol
            else
                echo $(date +"%Y%m%dT%H%M%S") $i: Backup HANA System completed successfully - exit code: $ERR >>$jobcontrol
            fi
		# 447: backup could not be completed: [2000008] Error during asynchronous file transfer (io_getevents), rc=28: No space left on device
        # }}}
		ID[$counter]="$($hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'WITH BACKUPS AS (SELECT *, ROW_NUMBER() OVER (ORDER BY "SYS_START_TIME" DESC) AS "ROWNUM" FROM (SELECT "SYS_START_TIME", "BACKUP_ID" FROM "SYS"."M_BACKUP_CATALOG" WHERE "ENTRY_TYPE_NAME" = '\''complete data backup'\'' AND "STATE_NAME" = '\''successful'\'')) SELECT "BACKUP_ID" FROM BACKUPS where ROWNUM='$maxAGE_fbck';')"
										  $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'BACKUP CATALOG DELETE ALL BEFORE BACKUP_ID '${ID[$counter]}' COMPLETE' >/dev/null 2>&1
	done
    typeset -i counter ; counter=0 ; for i in $systemids ; do
		ERR=""
        counter=$counter+1
        # {{{ SAP HANA Backup catalog clean & LOG clean ================================
        echo $(date +"%Y%m%dT%H%M%S") $i: Clean process starting >>$jobcontrol
		ID[$counter]="$($hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'WITH BACKUPS AS (SELECT *, ROW_NUMBER() OVER (ORDER BY "SYS_START_TIME" DESC) AS "ROWNUM" FROM (SELECT "SYS_START_TIME", "BACKUP_ID" FROM "SYS"."M_BACKUP_CATALOG" WHERE "ENTRY_TYPE_NAME" = '\''complete data backup'\'' AND "STATE_NAME" = '\''successful'\'')) SELECT "BACKUP_ID" FROM BACKUPS where ROWNUM='$maxAGE_fbck';')"
        $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'BACKUP CATALOG DELETE ALL BEFORE BACKUP_ID '${ID[$counter]}' COMPLETE' >/dev/null 2>&1
        ERR=$?
        if [ $ERR != 0 ] ; then
            NOTIFY_ERR_FLAG="ERROR - "
            echo $(date +"%Y%m%dT%H%M%S") $i: ERROR - Clean process - exit code: $ERR >>$jobcontrol
        else  
            echo $(date +"%Y%m%dT%H%M%S") $i: Clean process completed successfully - exit code: $ERR >>$jobcontrol
        fi
        ERR=""
		$hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'ALTER SYSTEM RECLAIM LOG' #>/dev/null 2>&1
        ERR=$?
        if [ $ERR != 0 ] ; then
            NOTIFY_ERR_FLAG="ERROR - "
            echo $(date +"%Y%m%dT%H%M%S") $i: ERROR - Reclaim process - exit code: $ERR >>$jobcontrol
        else  
            echo $(date +"%Y%m%dT%H%M%S") $i: Reclaim process completed successfully - exit code: $ERR >>$jobcontrol
        fi
		ERR=""
        $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'ALTER SYSTEM RECLAIM COLUMN LOB SPACE' #>/dev/null 2>&1
        ERR=$?
        if [ $ERR != 0 ] ; then
            NOTIFY_ERR_FLAG="ERROR - "
            echo $(date +"%Y%m%dT%H%M%S") $i: ERROR - Column Lob Space Reclaim process - exit code: $ERR >>$jobcontrol
        else  
            echo $(date +"%Y%m%dT%H%M%S") $i: Column Lob Space Reclaim process completed successfully - exit code: $ERR >>$jobcontrol
        fi
		ERR=""
        $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'ALTER SYSTEM RECLAIM ROW LOB SPACE' #>/dev/null 2>&1
        ERR=$?
        if [ $ERR != 0 ] ; then
            NOTIFY_ERR_FLAG="ERROR - "
            echo $(date +"%Y%m%dT%H%M%S") $i: ERROR - Row Lob Space Reclaim process - exit code: $ERR >>$jobcontrol
        else  
            echo $(date +"%Y%m%dT%H%M%S") $i: Row Lob Space Reclaim process completed successfully - exit code: $ERR >>$jobcontrol
        fi
		ERR=""
        $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS -d $i -x -a -j 'ALTER SYSTEM RECLAIM DATAVOLUME 120 DEFRAGMENT' #>/dev/null 2>&1
        ERR=$?
        if [ $ERR != 0 ] ; then
            NOTIFY_ERR_FLAG="ERROR - "
            echo $(date +"%Y%m%dT%H%M%S") $i: ERROR - Defragment process - exit code: $ERR >>$jobcontrol
        else  
            echo $(date +"%Y%m%dT%H%M%S") $i: Defragment process completed successfully - exit code: $ERR >>$jobcontrol
        fi
		ERR=""
        # }}}
    done
# }}}

# {{{ Loop2: perform backup of selected databases =============================
	echo $(date +"%Y%m%dT%H%M%S") "Loop2: perform export of selected databases is NOT working yet" >>$jobcontrol
# This section is NOT working yet
    # {{{ Backup particular databases and delete their old backups
    # echo $(date +"%Y%m%dT%H%M%S") "Loop 2: Backup particular databases and delete their old backups" >>$jobcontrol
    # typeset -i counter
    # counter=0
    # for i in $systemids ; do
    #     counter=$counter+1
    # echo $(date +"%Y%m%dT%H%M%S") Backup particular databases starting >>$jobcontrol
    # for db in $DBS
    # do
    #     dbc=$(echo $db | sed 's/"//g')
    #     skipdb=-1
    #     if [ "$IGGY" != "" ];
    #     then
    #         for i in $IGGY
    #         do
    #             [ "$dbc" == "$i" ] && skipdb=1 || :
    #         done
    #     fi

    #     if [ "$skipdb" == "-1" ] ; then
    #         FILE="$DEST_partdb/$dbc.$HOST.$NOW"
    #         # do all inone job in pipe,
    #         # and pipe it out to gz file in backup dir :)
    #         $hdbsql -i $instance -n $MyHOST -u $MyUSER -p $MyPASS 'export '$db'."*" as binary into '\'$MyTempDir\'' with replace threads 10'  && tar cfz "$FILE.tgz" $MyTempDir && rm -R $MyTempDir/*
    #         ERR=$?
    #     echo "Errorcode Backup particular databases:" $ERR >>$jobcontrol
    #         if [ $ERR != 0 ] ; then
    #     NOTIFY_ERR_FLAG="ERROR - "
    #         echo $(date +"%Y%m%dT%H%M%S") ERROR - Backup particular database: $db - exit code: $ERR >>$jobcontrol
    #         else  
    #     echo $(date +"%Y%m%dT%H%M%S") Backup particular database: $db successful - exit code: $ERR >>$jobcontrol
    #         fi
    #     fi
    # done
    # # }}}


    # # {{{ Delete old particular database backups
    # if [ $DELETE_OLD_BACKUPS -eq 1 ] ; then
    #     echo $(date +"%Y%m%dT%H%M%S") Deleting old database backups starting >>$jobcontrol
    #     find "$DEST_partdb" -noleaf -maxdepth 1 -mtime +"$maxAGE_partdb" -type d | $GREP -v "^$DEST_partdb$" | while read DIR; do
    #         rm -rf "$DIR"  >/dev/null 2>&1
    #         ERR=$?
    #         echo $(date +"%Y%m%dT%H%M%S") Deleting old database backup: $DIR,  $db - exit code: $ERR >>$jobcontrol
    #         if [ $ERR != 0 ] ; then
    #             NOTIFY_ERR_FLAG="ERROR - "
    #             echo $(date +"%Y%m%dT%H%M%S") ERROR - Deleting old database backup: $db - exit code: $ERR >>$jobcontrol
    #         else  
    #             echo $(date +"%Y%m%dT%H%M%S") Deleting old database backup: $db successful - exit code: $ERR >>$jobcontrol
    #         fi
    #     done
    # fi
    # }}}
# }}}

# {{{ Loop3: Compressing files, move to smb-share and upload to isystems cloud 
	cd /tmp
    echo $(date +"%Y%m%dT%H%M%S") "Loop3: Compressing backup files, move them to smb share and upload them to isystems cloud" >>$jobcontrol
    typeset -i counter ; counter=0 ; for i in $systemids ; do
	ERR=""
	if [[ $isysCLOUD -eq 1 || $smbBACKUP -eq 1 ]] ; then
		cd $HANA_backup_dir
        counter=$counter+1
		ERR=""
        # {{{ Compressing backup files for copy to smb share and/or isystems cloud
        echo $(date +"%Y%m%dT%H%M%S") $i: Compressing backup files started >>$jobcontrol
        if [ "SYSTEMDB" == "$i" ] ; then
			rm $HANA_backup_dir/*-SYSTEMDB-*.tgz               >/dev/null 2>&1
			rm $HANA_backup_dir/data/SYSTEMDB/*-SYSTEMDB-*.tgz >/dev/null 2>&1
            ERR=""
			tar cfz data/SYSTEMDB/$NOW-SYSTEMDB-$(hostname -f).tgz data/SYSTEMDB/$NOW*
        else
			rm $HANA_backup_dir/*-DB_$i-*.tgz                  >/dev/null 2>&1
			rm $HANA_backup_dir/data/DB_$i/*-DB_$i-*.tgz       >/dev/null 2>&1
            ERR=""
			tar cfz data/DB_$i/$NOW-DB_$i-$(hostname -f).tgz       data/DB_$i/$NOW*
        fi
		cd /tmp
		# }}}
		
        # {{{ Upload backup files to iSystems Cloud
        if [ $isysCLOUD -eq 1 ] ; then
            echo $(date +"%Y%m%dT%H%M%S") $i: Upload backup files to iSystems cloud starting >>$jobcontrol
            # Preparing environment upload if not already done
            ssh -p 55667 $isysUSER@$isysLOCATION 'mkdir -p ~/Hana-Backups/backups_fbck ~/Hana-Backups/backups_db' >/dev/null 2>&1
            echo $isysmaxAGE > $DEST_fbck/isysmaxAGE.conf
            scp -P 55667 $DEST_fbck/isysmaxAGE.conf $isysUSER@$isysLOCATION:~/Hana-Backups/ >>$jobcontrol
            rm $DEST_fbck/isysmaxAGE.conf
            # Starting upload of compressed full backup file
			ERR=""
            if [ "SYSTEMDB" == "$i" ] ; then
                rsync -e "ssh -p 55667" --partial $HANA_backup_dir/data/SYSTEMDB/$NOW-SYSTEMDB-$(hostname -f).tgz $isysUSER@$isysLOCATION:~/Hana-Backups/backups_fbck/ >>$jobcontrol
            else
                rsync -e "ssh -p 55667" --partial $HANA_backup_dir/data/DB_$i/$NOW-DB_$i-$(hostname -f).tgz       $isysUSER@$isysLOCATION:~/Hana-Backups/backups_fbck/ >>$jobcontrol
            fi
			ERR=$?
            if [ $ERR != 0 ] ; then
                NOTIFY_ERR_FLAG="ERROR - "
                echo $(date +"%Y%m%dT%H%M%S") $i: ERROR - Upload full backup files to iSystems cloud - exit code: $ERR >>$jobcontrol
            else  
                echo $(date +"%Y%m%dT%H%M%S") $i: Upload full backup files to iSystems cloud successful - exit code: $ERR >>$jobcontrol
            fi
        fi
		# }}}
		
		# {{{ Copying backup files from default backup location to destination directory ===
        if [ $smbBACKUP -eq 1 ] ; then
			echo $(date +"%Y%m%dT%H%M%S") $i: Copying archive to SMB started >>$jobcontrol
            if [ "SYSTEMDB" == "$i" ] ; then
                cp $HANA_backup_dir/data/SYSTEMDB/$NOW-SYSTEMDB-$(hostname -f).tgz $DEST_fbck/ && echo $(date +"%Y%m%dT%H%M%S") $i: Successfully copied archive file to "//$smbIP/$smbDIR/$NOW-SYSTEMDB-$(hostname -f).tgz" >>$jobcontrol
            else
                cp $HANA_backup_dir/data/DB_$i/$NOW-DB_$i-$(hostname -f).tgz       $DEST_fbck/ && echo $(date +"%Y%m%dT%H%M%S") $i: Successfully copied archive file to "//$smbIP/$smbDIR/$NOW-DB_$i-$(hostname -f).tgz" >>$jobcontrol
            fi    
		fi
		# {{{ Delete Archives
		if [[ $isysCLOUD -eq 1 || $smbBACKUP -eq 1 ]] ; then
			if [ "SYSTEMDB" == "$i" ] ; then
				rm $HANA_backup_dir/data/SYSTEMDB/$NOW-SYSTEMDB-$(hostname -f).tgz && echo $(date +"%Y%m%dT%H%M%S") $i: Successfully deleted archive file "$HANA_backup_dir/data/SYSTEMDB/$NOW-SYSTEMDB-$(hostname -f).tgz" >>$jobcontrol
				rm $HANA_backup_dir/data/SYSTEMDB/*.tgz >/dev/null 2>&1
			else
				rm $HANA_backup_dir/data/DB_$i/$NOW-DB_$i-$(hostname -f).tgz       && echo $(date +"%Y%m%dT%H%M%S") $i: Successfully deleted archive file "$HANA_backup_dir/data/DB_$i/$NOW-DB_$i-$(hostname -f).tgz" >>$jobcontrol
				rm $HANA_backup_dir/data/DB_$i/*.tgz >/dev/null 2>&1
			fi 
		fi
	else 
		echo $(date +"%Y%m%dT%H%M%S") $i: Compressing backup files skipped >>$jobcontrol
	fi
    done
# }}}

# {{{ Getting content of backup folders on SMB-share and in isystems cloud =====
	if [[ $isysCLOUD -eq 1 || $smbBACKUP -eq 1 ]] ; then
		FILTERYEAR="$(date +"%Y")"
		echo $(date +"%Y%m%dT%H%M%S") Content of full backup folder on SMB-Share: >>$jobcontrol
		ls -lah $DEST_fbck | grep $FILTERYEAR >>$jobcontrol
	
		echo $(date +"%Y%m%dT%H%M%S") Content of particular database folder on SMB-Share: >>$jobcontrol
		ls -lah $DEST_partdb |grep $FILTERYEAR >>$jobcontrol
	
		echo $(date +"%Y%m%dT%H%M%S") Content of full backup folder in iSystems cloud: >>$jobcontrol
		ssh -p 55667 $isysUSER@$isysLOCATION "ls -lah ~/Hana-Backups/backups_fbck/ |grep $FILTERYEAR" >>$jobcontrol
	
		# # Starting upload of particular backup files
		# rsync -e "ssh -p 55667 -c blowfish" --partial $DEST_partdb/* $isysUSER@$isysLOCATION:~/Hana-Backups/backups_db/ >>$jobcontrol
		# if [ $ERR != 0 ] ; then
		#     NOTIFY_ERR_FLAG="ERROR - "
		#     echo $(date +"%Y%m%dT%H%M%S") ERROR - Upload particular backup files to iSystems cloud - exit code: $ERR >>$jobcontrol
		# else  
		#     echo $(date +"%Y%m%dT%H%M%S") Upload particular backup files to iSystems cloud successful - exit code: $ERR >>$jobcontrol
		# fi
	
		# # Getting content of backup location after upload
		# echo ""Content of particular db backup folder in iSystems cloud: >>$jobcontrol
		# ssh -p 55667 $isysUSER@$isysLOCATION 'ls -lahR ~/Hana-Backups/backups_db/ |grep 201' >>$jobcontrol
	
		# echo $(date +"%Y%m%dT%H%M%S") Upload backup files to iSystems cloud successful - exit code: $ERR >>$jobcontrol
		# fi  ???
	fi
# }}}

# {{{ Unmounting SMB-share =====================================================
	ERR=""
    if [ $smbBACKUP -eq 1 ] ; then
        umount $smbMOUNTPOINT >>$jobcontrol
		ERR=$?
        if [ $ERR != 0 ] ; then
            NOTIFY_ERR_FLAG="ERROR - "
            echo $(date +"%Y%m%dT%H%M%S") ERROR - Unmounting SMB-share - exit code: $ERR >>$jobcontrol
        else  
            echo $(date +"%Y%m%dT%H%M%S") Unmounting SMB-share successful - exit code: $ERR >>$jobcontrol
        fi
    fi
# }}}

# {{{ add usage of cpu and harddisk to report ==================================
    echo "[IOSTAT]"   >>$jobcontrol
    iostat            >>$jobcontrol
    echo "[DISKFREE]" >>$jobcontrol
    df -h             >>$jobcontrol
    echo "[LAST]"     >>$jobcontrol
    last root | head  >>$jobcontrol
    # echo "" >>$jobcontrol
    # logwatch --detail high --range today >>$jobcontrol
# }}}

# {{{ Cleanup ===============================================
	if [ 1 -eq 1 ] ; then
		days=$isysmaxAGE;path=$HANA_backup_dir;#ls -lah $path/*.tgz
		#find $path -name "*.tgz" -type f -mtime -$days -exec ls -d -1 {} \;
		#find $path -name "*.tgz" -type f -mtime +$days -exec ls -d -1 {} \;
		#find $path -name "*.tgz" -type f -mtime +$days -exec rm -f    {} \;
		#ls -lah $path/*.tgz
		#https://www.thegeekstuff.com/2012/11/linux-touch-command/
		#https://www.cyberciti.biz/faq/linux-unix-how-to-find-and-remove-files/
		#log that cleanup procedure starts
		#log what files got successfully deleted
	fi
# }}}

# {{{ Sending email notification ===============================================
    echo ""
    if [ $SEND_EMAIL -eq 1 ] ; then
        mymac=$(ip link show dev eth0 | grep -oE 'link/ether ([a-f0-9]{2}:){5}[a-f0-9]{2}' | cut -d' ' -f2 | sed -e 's/://g' )
        SYSident="$mymac - $NOTIFY_CUSTOMER - `hostname -f`"
        if [[ $NOTIFY_PWD != "" && "$NOTIFY_AUTHGERAFFEL" == "" ]] ; then
            NOTIFY_AUTHGERAFFEL="-S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user="$NOTIFY_SENDER" -S smtp-auth-password="$NOTIFY_PWD" -S ssl-verify=ignore"
        fi
        cat $jobcontrol | grep -v "rows affected" | grep -v "Removing leading" | mailx -v -r $NOTIFY_FROM -s "$NOTIFY_ERR_FLAG $SYSident - $NOTIFY_SUBJECT" -S smtp="$NOTIFY_MAILSERVER:$NOTIFY_PORT" $NOTIFY_AUTHGERAFFEL $NOTIFY_RECEIVER
    fi
# }}}

