"""
StorageConnectorClient base class for Storage Connectors
"""

# python modules
from abc import ABCMeta, abstractmethod
import os
import sys, getpass
from tempfile import mkstemp
import time, re
import hashlib

try:
    import NameServerPy, linecache
except:
    pass

#####################################################################################

class InstallerTracer:
    """Tracer that only traces to stdout"""
    
    trcLvl = 2
    
    def __init__(self, traceLevel):
        """
        Initialization
        @param traceLevel: trace level: DEBUG, INFO, WARNING, ERROR, FATAL
        @type traceLevel: string
        """
        if traceLevel.upper() == "DEBUG":
            self.trcLvl = 1
        if traceLevel.upper() == "INFO":
            self.trcLvl = 2
        if traceLevel.upper() == "WARNING":
            self.trcLvl = 3
        if traceLevel.upper() == "ERROR":
            self.trcLvl = 4
        if traceLevel.upper() == "FATAL":
            self.trcLvl = 5

    def debug(self, message):
        if self.trcLvl <= 1:
            print message

    def info(self, message):
        if self.trcLvl <= 2:
            print message

    def warning(self, message):
        if self.trcLvl <= 3:
            print message

    def error(self, message):
        if self.trcLvl <= 4:
            print message

    def fatal(self, message):
        if self.trcLvl <= 5:
            print message


##############

class StorageConfiguration:
    """Base class for configuration parameters parsing"""

    __metaclass__ = ABCMeta

    _globalConfig = None
    _params = {}

    DATA = "data"
    LOG = "log"
    LOGMIRROR = "logmirror"
    
    @abstractmethod
    def __init__(self):
        """Abstract method: only a concrete configuration handling class must be used"""
        pass

    def reload(self):
        """Reads global.ini into memory."""

        self._params.clear()

        # self.readOptions is overwritten by concrete classes: reload and read options from a asection
        cfgVals = self.readOptions("storage")

        partMatch = re.compile('partition_*_*_')
        for(k,v) in cfgVals:
            if partMatch.match(k):
                keySplit = re.split("__|_",k)
                if(len(keySplit)==5 and (keySplit[3]=="es" or keySplit[3]=="ets")): # keys with _es/_ets contain additional underscore
                    self._params[keySplit[1], keySplit[2]+"_"+keySplit[3], keySplit[4]] = v
                else:
                    self._params[keySplit[1], keySplit[2], keySplit[3]] = v

    def getParamListForKey(self, storagePartition, usageType, wildcards = True):
        """
        Retrieves a list of all defined parameters for the key pair (storagePartition, usageType)
        @param storagePartition: the id of the storage partition
        @type storagePartition: int
        @param usageType: the type of the storage partition
        @type usageType: string
        @param wildacrds: determines whether keys checked for may be only defined by wildcards or not, default = True
        @type wildcards: boolean
        @return: list of parameters
        @rtype: list<string>
        """

        usageType = usageType.lower()
        
        if wildcards:
            spCheckList = [str(storagePartition), "*"]
            utCheckList = [usageType, "*"]
        else:
            spCheckList = [str(storagePartition)]
            utCheckList = [usageType]

        paramList = []
        for (sp, ut, p) in self._params:
            if str(sp) in spCheckList and ut in utCheckList:
                paramList.append(p)
    
        return paramList
    
    def getParameter(self, storagePartition, usageType, param):
        """
        Retrieves the parameter value.
        @param storagePartition: the id of the storage partition
        @type storagePartition: int
        @param usageType: the type of the storage partition
        @type usageType: string
        @param param: parameter name
        @type param: string
        @return: the parameter value
        @rtype: string
        """

        storagePartition = str(storagePartition)
        usageType = usageType.lower()

        if self._params.has_key((storagePartition, usageType, param)):
            return self._params[(storagePartition, usageType, param)]
        elif self._params.has_key(("*", usageType, param)):
            return self._params[("*", usageType, param)]
        elif self._params.has_key((storagePartition, "*", param)):
            return self._params[(storagePartition, "*", param)]
        elif self._params.has_key(("*", "*", param)):
            return self._params[("*", "*", param)]
        else:
            return None

    def hasParameter(self, storagePartition, usageType, param):
        """
        Checks if a parameter exists.
        @param storagePartition: the id of the storage partition
        @param usageType: the type of the storage partition
        @param param: parameter name
        @return: True or False
        @rtype: boolean
        """
        return self.getParameter(storagePartition, usageType, param) != None
        
    def hasKey(self, storagePartition, usageType, wildcards = True):
        """
        Checks if a key par exists in the configuration file.
        @param storagePartition: the id of the storage partition
        @param usageType: the type of the storage partition
        @return: True or False
        @rtype: boolean
        """
        pList = self.getParamListForKey(storagePartition, usageType, wildcards)
        return len(pList) > 0
        

class StorageConfigurationHDB(StorageConfiguration):
    """Wrapper for HDB configuration file access."""
        
    def __init__(self):
        """Initialization"""

        import ConfigMgrPy
        self._globalConfig = ConfigMgrPy.LayeredConfiguration('global.ini', ConfigMgrPy.READONLY)
        self.reload()

    def readOptions(self, section):
        """
        Reads options of a section of the global.ini
        @param section: the section name
        @type section: string
        @return: a dict of options and their values
        @rtype: dict<string, string>
        """

        self._globalConfig.reload()
        return self._globalConfig.getValues(section)



class StorageConfigurationInstallation(StorageConfiguration):
    """Parses global.ini by itself for running the Storage Connector outside HANA (during installation)"""

    __path = None

    def __init__(self, configFilePath):
        """
        Initialization
        @param configFilePath: path to directory containing global.ini
        @type configFilePath: string
        """
        self.__path = configFilePath
        import ConfigParser
        self._globalConfig = ConfigParser.ConfigParser()
        self._globalConfig.read(os.path.join(self.__path, "global.ini"))
        self.reload()
        
    def readOptions(self, section):
        """
        Reads options of a section of the global.ini
        @param section: the section name
        @type section: string
        @return: a dict of options and their values
        @rtype: dict<string, string>
        """

        self._globalConfig.read(os.path.join(self.__path, "global.ini"))
        return self._globalConfig.items(section)


#####################################################################################


class Helper(object):
    """Some helper functions that are useful for the client."""
    
    @staticmethod
    def _ping(host, tracer = None):
        """
        Pings a host
        @param tracer: a tracer object
        @type tracer: InstallerTracer | ServiceClientPy.Tracer
        @return: info about success
        @rtype: boolean
        """

        (code, _) = Helper._runOsCommand("ping %s -c1" % host, tracer)
        return code == 0

    @staticmethod
    def _runOsCommand(command, tracer = None):
        """
        Runs an OS command
        @param command: the command to be issued
        @type command: string
        @param tracer: a tracer object
        @type tracer: InstallerTracer | ServiceClientPy.Tracer
        @return: output of OS command: (code, command-output)
        @rtype: (int, string)
        """

        if tracer != None:
            tracer.debug("run OS command `%s`" % command)
        
        try:
            # #########################################################################################
            # using system facility directly to minimize risk of memory allocation (and whatever stuff)
            # for forked processes that may wait endlessly for resources of the parent process
            tmpFileOut = mkstemp(suffix='.out', prefix='fc')
            tmpFileErr = mkstemp(suffix='.err', prefix='fc')
            retCode = os.system("%s > %s 2> %s" % (command, tmpFileOut[1], tmpFileErr[1]))
            with open("%s" % tmpFileOut[1], 'r') as file:
                output = file.read()
            with open("%s" % tmpFileErr[1], 'r') as file:
                errors = file.read()
            os.close(tmpFileOut[0])
            os.close(tmpFileErr[0])
            os.remove(tmpFileOut[1])
            os.remove(tmpFileErr[1])
            # #########################################################################################
            
            if tracer != None:
                tracer.debug("  => return code: %s" % retCode)
                tracer.debug("  => stdout:      %s" % output)
                tracer.debug("  => stderr:      %s" % errors)

            if retCode != 0:
                return (retCode, "OS operation finished unsuccessfully: %s" % errors)
            
            return (retCode, output)
        except Exception as e:
            print e
            return (-1, 'Error while executing OS command')

    @staticmethod
    def _run2PipedOsCommand(command1, command2, tracer = None):
        """
        Runs two piped OS commands
        @param command1: the command to be issued before pipe
        @type command1: string
        @param command2: the command to be issued after pipe
        @type command2: string
        @param tracer: a tracer object
        @type tracer: InstallerTracer | ServiceClientPy.Tracer
        @return: output of OS commands: (code, command-output)
        @rtype: (int, string)
        """

        if tracer != None:
            tracer.debug("run OS command `%s | %s`" % (command1, command2))
        
        try:
            # #########################################################################################
            # using system facility directly to minimize risk of memory allocation (and whatever stuff)
            # for forked processes that may wait endlessly for resources of the parent process
            tmpFileOut = mkstemp(suffix='.out', prefix='fc')
            tmpFileErr = mkstemp(suffix='.err', prefix='fc')
            retCode = os.system("%s | %s > %s 2> %s" % (command1, command2, tmpFileOut[1], tmpFileErr[1]));
            if retCode == 256:
                retCode = 0
            with open("%s" % tmpFileOut[1], 'r') as file:
                output = file.read()
            with open("%s" % tmpFileErr[1], 'r') as file:
                errors = file.read()
            os.close(tmpFileOut[0])
            os.close(tmpFileErr[0])
            os.remove(tmpFileOut[1])
            os.remove(tmpFileErr[1])
            # #########################################################################################
            
            if tracer != None:
                tracer.debug("  => return code: %s" % retCode)
                tracer.debug("  => stdout:      %s" % output)
                tracer.debug("  => stderr:      %s" % errors)

            if retCode != 0:
                return (retCode, "OS operation finished unsuccessfully: %s" % errors)
            
            return (retCode, output)
        except Exception as e:
            print e
            return (-1, 'Error while executing OS command')


    @staticmethod
    def asBool(value):
        """
        Translates common values of string, int and boolean to boolean
        @param value: the value
        @type value: arbitrary
        @return: translated value
        @rtype: boolean
        """

        if type(value) == str and value.lower() in ["yes", "on", "true", "1"]:
            return True
        elif type(value) == int and int(value) > 0:
            return True
        else:
            return False

    @staticmethod
    def _generateUniqueHostKey(hostname):
        key = hashlib.md5(hostname).hexdigest()[:16]  # SCSI-3 PR keys are only 64bit long
        i = 1
        while key[0] == "0":
            filler = ''.join(['#' for _ in range(i)])
            key = hashlib.md5(hostname + filler).hexdigest()[:16]  # SCSI-3 PR keys are only 64bit long
            i = i+1

        return key

#####################################################################################


class StorageConnectorClient(object):
    """
    Abstract base class for a storage connector client defining the expected interface and containing some basic functions
    """
    __metaclass__ = ABCMeta

    _cfg = None
    tracer = None
    sid = None

    def __init__(self, method = None, configLocation = None, sid = None, mnt_data = None, mnt_log = None, tracelevel = "INFO", partition = 1):
        """
        Initializes the Storage Connector Client.
          If method != None: the HDB environment will be used - all remaining parameters can be left untouched
          Else: the installer environment will be used

        @param method: the method to call, attach or detach
        @type method:  string
        @param configLocation: path to global.ini directory
        @type configLocation: string
        @param sid: the SID of the system
        @type sid: string
        @param mnt_data: basepath to data
        @type mnt_data: string
        @param mnt_log: basepath to log
        @type mnt_log: string
        @param tracelevel: the trace level
        @type tracelevel: string
        @param partition: partition number
        @type partition: int
        """

        try:
            import ServiceClient
            self.tracer = ServiceClient.Tracer("ha_%s" % self.__class__.__name__)
        except:
            self.tracer = InstallerTracer(tracelevel)

        if method == None:
            import ConfigMgrPy
            self._cfg = StorageConfigurationHDB()
            self.sid = ConfigMgrPy.sapgparam("SAPSYSTEMNAME")
        else:
            self._cfg = StorageConfigurationInstallation(configLocation)
            self.sid = sid
    
        self.tracer.debug("tracer 'ha_%s' initialized" % self.__class__.__name__)

        usageData = "data"
        usageLog = "log"
        if not partition is None:
            if int(partition) >= 1024 and int(partition) < 2048: #override usage type if partition is a DT persistence (e.g. for hdbmount)
                usageData = "data_es"
                usageLog  = "log_es"
            if int(partition) >= 2048 and int(partition) < 3072: #override usage type if partition is a ETS persistence (e.g. for hdbmount)
                usageData = "data_ets"
                usageLog  = "log_ets"

        if method == "listpartitions":
            print self.getAvailableStoragePartitions()
            return

        storages = [{"partition" : partition, "usage_type" : usageData, "path" : mnt_data}, {"partition" : partition, "usage_type" : usageLog, "path" : mnt_log}]
        if method == "attach":
            self.attach(storages)
        elif method == "detach":
            self.detach(storages)
        elif method == "unmount":
            self.unmount(storages)
        elif method == "setsidowner":
            self.setsidowner(storages)

    def term(self):
        """Terminates the storage connector client"""
        
        self.tracer.debug("terminating %s..." % self.__class__.__name__)


    #########################################


    @abstractmethod
    def about(self):
        """
        Returns some static information about the storage connector client, it must have the format
            return {"provider_company" :     "",
                    "provider_name" :        "",
                    "provider_description" : ""
                    "provider_version" :     ""
                   }
        @return: a dict containing information about the storage connector client
        @rtype: {string : string}
        """
        pass
    
    @staticmethod
    def sudoers():
        """
        Gives information about the necessary sudo permission to be set. Example:
        
            return "ALL=NOPASSWD: /bin/mkdir, /bin/chown, /sbin/multipath, /usr/bin/sg_persist, /bin/mount, /bin/umount"
        
        Permissions should be set as restricted as possible.
        @return: the string for /etc/sudoers
        @rtype: string
        """
        return ""

    def attach(self, storages):
        """
        Attaches storages on this host
        @param storages: storage descriptions {partition, usage_type, path}
        @type storages: a list of dicts: [{"partition" : ..., "usage_type" : ..., "path": ...}, ...]
        @return: information about success
        @rtype: int
        """
        return 0
    
    def detach(self, storages):
        """
        Detaches storages from this host
        @param storages: storage descriptions {partition, usage_type, path}
        @type storages: a list of dicts: [{"partition" : ..., "usage_type" : ..., "path": ...}, ...]
        @return: information about success
        @rtype: int
        """
        return 0

    def stonith(self, hostname):
        """
        Runs SHOOT THE OTHER NODE ON THE HEAD for <hostname>
        @param hostname: the failing host to be STONITHed
        @return: information about success
        @rtype: int
        """
        return 0

    def info(self, paths):
        """
        Collects information about currently attached devices
        @param paths: a list of paths the are assumed to be mounted
        @type paths: list of strings
        @return: a dict containing information about the device mounted at the paths
        @rtype: {string : string}
        """
        pass


    def getAvailableStoragePartitions(self):
        availablePartitions = []

        for (k, v) in self._cfg.readOptions("storage"):
            if k.startswith("partition_"):
                sp = k.split("partition_")[1].split("_")[0]
                if sp not in availablePartitions and sp != "*":
                    availablePartitions.append(sp)
        
        return availablePartitions
    
    def setsidowner(self, storages):
        self.tracer.info("%s.setsidowner method called" % self.__class__.__name__)

        if os.geteuid() != 0:
            msg = "called 'setsidowner' as user '%s' instead of root" % getpass.getuser()
            print msg
            self.tracer.fatal(msg)
            raise Exception(msg)

        me = "%sadm" % self.sid.lower()

        # loop over storages given by HDB
        for storage in storages:

            # extract necessary information
            path = os.path.realpath(storage.get("path"))

            # check current mounts
            mountList = self._isMounted(path)

            # abort if path is not mounted at all as this is a necessity
            if len(mountList) == 0:
                msg = "path '%s' is not mounted" % path
                print msg
                self.tracer.fatal(msg)
                raise Exception(msg)

            (code, output) = Helper._runOsCommand("chown -R %s %s" % (me, path), self.tracer)
            if code != 0:
                msg = "could not set '%s' as owner of path '%s': '%s'" % (me, path, output)
                print msg
                self.tracer.fatal(msg)
                raise Exception(msg)

            sapgroup = "sapsys"
            (code, output) = Helper._runOsCommand("chgrp -R %s %s" % (sapgroup, path), self.tracer)
            if code != 0:
                msg = "could not set group '%s' as owner of path '%s': '%s'" % (sapgroup, path, output)
                print msg
                self.tracer.fatal(msg)
                raise Exception(msg)

        return 0

    def unmount(self, storages):
        self.tracer.info("%s.unmount method called" % self.__class__.__name__)

        # loop over storages given by HDB
        for storage in storages:

            # check current mounts
            path = os.path.realpath(storage.get("path"))
            if not os.path.isdir(path):
                continue

            msg = "unmounting path '%s'" % path
            print msg
            self.tracer.info(msg)

            mountList = self._isMounted(path)
            # if no device is attached at all, end processing
            if len(mountList) != 0:
                # try to unmount device and wait up to 1 minute
                return self._forcedUnmount('a device', path, 12)

        return 0

    def extractSingleDevices(self, result, wwid):
#       360080e500017c3bc00000fab4fa0fabf dm-0 VENDOR,PRODUCTNAME ;)
#       size=64G features='0' hwhandler='1 rdac' wp=rw
#       |-+- policy='round-robin 0' prio=-1 status=active
#       | `- 1:0:0:0  sdb 8:16   active undef running
#       `-+- policy='round-robin 0' prio=-1 status=enabled
#         `- 2:0:0:0  sdn 8:208  active undef running

        deviceName = ""
        lines = result.split("\n")
        for l in lines:
            if wwid in l:
                idx = 0
                ls = l.split()
                for p in ls:
                    if p.startswith("dm-"):
                        deviceName = ls[idx]
                    idx = idx + 1
                break

        if len(deviceName) == 0:
            raise Exception("device with wwid '%s' not found" %  wwid)

        msg = "found '%s' as internal multipath device name for wwid '%s'" % (deviceName, wwid)
        self.tracer.info(msg)

        (code, output) = Helper._runOsCommand("ls /sys/block/%s/slaves" % deviceName, self.tracer)
        if code != 0:
            msg = "no single path devices found for wwid '%s'" % wwid
            print msg
            self.tracer.fatal(msg)
            raise Exception(msg)

        singleDevices = output.split()
        singleDevices = ["/dev/%s" % n for n in singleDevices]

        msg = "  with single devices: %s" % singleDevices
        self.tracer.info(msg)

        return singleDevices

    def handlePRUnitAttention(self, device):
        """handles <PR in: unit attention>"""
        retry = 0
        while True:
            if retry > self.retries:
                break

            (code, output) = Helper._runOsCommand("sudo /usr/bin/sg_persist -r %s" % device, self.tracer)
            if code == 0 and "PR generation=" in output:
                return

            time.sleep(self.interval)
            retry = retry+1

        msg = "unable to handle UNIT ATTENTION or device not ready after %s seconds" % (self.interval * self.retries)
        self.tracer.warning(msg)

    def flushMultipath(self, prType):
        if prType is not 6:
            return

        self.tracer.info("flush multipath devices")
        # if a device is polled, it will not be flushed, therefore, run flush several times 
        for i in range(1, 4):
            Helper._runOsCommand("sudo /sbin/multipath -F", self.tracer)
            time.sleep(0.1)

    def getPRType(self, defaultPrType = 6):
        prType = defaultPrType
        if self._cfg.hasParameter("*","*","prtype"):
            prType = int(self._cfg.getParameter("*","*","prtype"))

        if prType not in [5, 6]:
            raise Exception("unsupported prout-type '%s' for persistent reservation" % prType)

        if prType == 5:
            prTypeReservationText = "Write Exclusive"
        if prType == 6:
            prTypeReservationText = "Exclusive Access"

        self.tracer.info("using --prout-type=%s for persistent reservations" % prType)

        return (prType, prTypeReservationText)

    def attachMount(self, connectionData, device, path):
        # it might take a short time, if a reservation was updated, until the device is ready to be mounted
        # therefore, try several times...
        try:
            mo = connectionData["mountoptions"]
        except:
            mo = ""

        self.tracer.info("using mount options: '%s'" % mo)

        if "xfs" in mo and not self._cfg.hasParameter("*","*","disablexfsrepair"):
            (code, output) = Helper._runOsCommand("sudo /usr/sbin/xfs_repair %s" % device, self.tracer)
            if not code == 0:
                self.tracer.warning("could not clean device %s as filecheck failed with rc=%s [output=%s]" % (device,code,output))

        retry = 0
        while True:
            if retry > self.retries:
                self.tracer.fatal("mount failed after multiple retries")
                raise Exception("mount failed after multiple retries")

            try:
                self.tracer.info("mount '%s' to '%s' with options '%s'" % (device, path, mo))
                self._mount(device, path, mo)
            except:
                pass

            time.sleep(self.interval)
            if self._isMounted(path):
                self.tracer.debug("device was mounted after %s retries" % retry)
                break;

            retry = retry+1
            self._umount(path)

        msg = "attached device '%s' to path '%s'" % (device, path)
        print msg
        self.tracer.info(msg)


    #########################################

    def _generateUniqueHostKey(self, hostname):
        return Helper._generateUniqueHostKey(hostname)

    def _mount(self, device, path, options = ""):
        """
        Mounts a device to a path.
        @param device: a device name
        @type device: string
        @param path: the path the device shall be mounted to
        @type device: string
        @param options: additional options for the mount command
        @type: string
        @return: information about success
        @rtype: boolean
        @raise Exception: if device is not mountable for any reason
        """
        self.tracer.debug("_mount(%s, %s)" % (device, path))
        (code, output) = Helper._runOsCommand("sudo mount %s %s %s" % (options, device, path), self.tracer)
        if not code == 0:
            raise Exception("unable to mount device `%s` to path `%s` - Error %s: %s" % (device, path, code, output))
        
        return True

    def _umount(self, device_or_path, lazy = False):
        """
        Unmounts a device or a path regarding to its entry in `cat /proc/mounts`.
        @param device_or_path: a device name or path that shall be unmounted
        @type device_or_path: string
        @param lazy: specifies if the device shall be unmounted lazy or not, default = not
        @type lazy: boolean
        @return: True if successful, False otherwise
        @rtype: boolean 
        """
        lopt = ""
        if lazy:
            lopt = "-l"

        self.tracer.debug("_umount(%s, %s)" % (device_or_path, lazy))

        (code, output) = Helper._runOsCommand("sudo umount %s %s" % (lopt, device_or_path), self.tracer)
        if not code == 0:
            return False
        else:
            return True
    
    def _isMounted(self, device_or_path):
        """
        Returns a set of mappings from device to path for a specified device or path (multiple mounts are possible)
        @param device_or_path: a device name or path
        @type device_or_path: string
        @return: a set of tuples of current mounts (device, path)
        @rtype: set of (string, string) tuples 
        """
        self.tracer.debug("_isMounted(%s)" % device_or_path)
        (code, output) = Helper._run2PipedOsCommand("cat /proc/mounts", "grep -w %s" % device_or_path)
        if not code == 0:
            raise Exception("could not lookup mounted devices (Error %s): %s" % (code, output))

        mounts = set()
        for line in output.split("\n"):
            try:
                mounts.add( (line.split()[0], line.split()[1]) )
            except:
                pass

        self.tracer.debug("=> mountpoints: %s" % str(mounts))

        return mounts

    def _lsof_and_kill(self, path):
        """
        Kills all processes with SIGKILL blocking the specified path (based on lsof)
        @param path: the argument to search for in lsof
        @type path: string
        @rtype: void
        @raise Exception: if lsof was not runnable
        """
        self.tracer.debug("_lsof_and_kill(%s)" % path)
        (code, output) = Helper._run2PipedOsCommand("sudo lsof", "grep -w %s" % path)
        if not code == 0:
            raise Exception("could not run `lsof` (Error %s): %s" % (code, output))

        nspid = os.getpid()
        if len(output) > 0:
            lines = output.split("\n")
            pids = []
            for l in lines:
                try:
                    pids.append((l.split()[1], l))
                except:
                    pass

            for pid in pids:
                try:
                    os.kill(int(pid[0]),0) # ignore temporary pids created for previous lsof-command
                except OSError:
                    continue

                self.tracer.info("found process with pid=%s still active on path '%s', trying to kill it (nspid=%s)" % (pid[0], path, nspid))
                self.tracer.info("-- Corresponding lsof entry: %s" % pid[1])
                if not str(pid[0]) == str(nspid): #prevent self-kill during system shutdown
                    (c, o) = Helper._runOsCommand("sudo kill -9 %s" % pid[0], self.tracer)
                    self.tracer.info("kill -9 %s; rc=%s; stdout/stderr=%s" % (pid[0], c, o))
                    print "sent KILL signal to pid %s" % pid[0]
                else:
                    self.tracer.info("The nameserver itself is blocking the mount point -> skipping")
        else:
            self.tracer.info("no processes with lsof found")


    def _getConnectionDataForLun(self, storagePart, usageType):
        """
        Reads the HDB configuration for a specified storage and returns a dict of relevant parameters.
        @param storagePart: the partition id
        @type storagePart: integer
        @param usageType: the type of the partition
        @type usageType: string
        @return: a string of defined configuration parameters
        @rtype: {string : string}
        """
        pMap = {}
        pList = self._cfg.getParamListForKey(storagePart, usageType)

        for param in pList:
            pMap[param] = self._cfg.getParameter(storagePart, usageType, param)
        
        return pMap

    def _checkAndCreatePath(self, path):
        """Looks up OS real path and creates new mountpoint on demand.
        @param path: the path to check and translate
        @type path: string
        @raise Exception: if mkdir command fails
        @return: the (created) real path
        @rtype: string"""
        
        path = os.path.realpath(path)
        # check if path already exists on this machine
        # this happens either when a host is added to a landscape or a standby first time takes over a host
        if not os.path.isdir(path):
            # create path
            (code, output) = Helper._runOsCommand("mkdir -p %s" % path, self.tracer)
            if not code == 0:
                msg = "unable to create directory %s: Code %s: %s" % (path, code, output)
                print msg
                self.tracer.error(msg)
                raise Exception(msg)

        return path

    def _setUpSysTracing(self):
        try:
            sys.settrace(self._traceit)
        except:
            pass  # don't care if something goes wrong

        return 0


    _threadInfo = "initializing Storage Connector"

    def _traceit(self, frame, event, arg):
        """Profiling function for supporting M_THREADS view to show current state of the Storage Connector.
        
        IT SHOULD NOT BE OVERWRITTEN
        
        """

        if not "NameServerPy" in sys.modules:
            return self._traceit

        # setup TNSClient
        if 'ns' not in globals():
            global ns
            ns = None
    
        if ns == None:
            NameServerPy.init()
            ns = NameServerPy.TNSClient()

        # actual tracing
        if event == "line":
            lineno = frame.f_lineno
    
            if frame.f_globals.has_key("__file__"):
                filename = frame.f_globals["__file__"]
                if (filename.endswith(".pyc") or
                    filename.endswith(".pyo")):
                    filename = filename[:-1]
        
                name = frame.f_globals["__name__"]
                line = linecache.getline(filename, lineno)
                
                currentLine = "%s:%s: %s" % (name, lineno, line.strip())

                if self.__class__.__name__ in name:
                    self._threadInfo = currentLine
                    traceit = currentLine
                else:
                    traceit = "%s   >>>   %s" % (self._threadInfo, currentLine)

                ns.setHaProviderThreadDetail(traceit)
                return self._traceit

        ns.setHaProviderThreadDetail(self._threadInfo)
        return self._traceit
    
    # constant wait time after each check of the unmount status
    WAIT_SECONDS_AFTER_KILL = 5

    def _forcedUnmount(self, device, path, max_retries):
        msg = "unmounting device '%s' from '%s'" % (device, path)
        self.tracer.info(msg)

        retries = 0
        unmount_done = False

        # retry mount check after hard kill has been issued
        while retries < max_retries:
            # try to unmount device
            while self._umount(path):
                pass

            # check again, if umount was successful
            checkMounts = self._isMounted(path)
            if len(checkMounts) > 0:

                # the kill signal is only send one time
                if retries == 0:
                    self.tracer.warning("try to 'kill -9' processes blocking the mountpoint %s" % (path))
                    self._lsof_and_kill(path)

                # give kernel some time
                time.sleep(self.WAIT_SECONDS_AFTER_KILL)
                retries += 1
            else:
                unmount_done = True
                break

        # forced unmount not possible even after full wait cycle
        if unmount_done is False:
            msg = "device on path '%s'busy, will not unmount" % (path)
            print msg
            self.tracer.warning(msg)
            return 1 # unmount failed

        if retries == 0:
            # unmount succeeded immediately
            msg = "detached device '%s' from path '%s'" % (device, path)
        else:
            # forced unmount done but it took some a while
            retry_time = retries * self.WAIT_SECONDS_AFTER_KILL
            msg = "detached device '%s' from path '%s' after '%d' seconds" % (device, path, retry_time)

        print msg
        self.tracer.info(msg)
        return 0

    def _reloadMultipathDaemon(self):
        if os.path.isfile("/usr/bin/systemctl"): # systemd
            reloadCommand = "sudo /usr/bin/systemctl reload-or-restart multipathd"
        elif os.path.isfile("/etc/init.d/multipathd"): # System V init
            reloadCommand = "sudo /etc/init.d/multipathd force-reload"
        else:
            msg = "service wrapper for multipathd service not found"
            print msg
            self.tracer.error(msg)
            raise Exception(msg)

        (code, output) = Helper._runOsCommand(reloadCommand, self.tracer)
        if code != 0:
            msg = "could not reload multipath topology with '%s': %s" % (reloadCommand, output)
            print msg
            self.tracer.error(msg)
            raise Exception(msg)

    def unmountEverything(self, mountList, path):
        for (dev, mountpoint) in mountList:
            # try to unmount device and wait up to 3 minutes
            self._forcedUnmount(dev, mountpoint, 36)

        # this will create the mountpoint if necessary
        path = self._checkAndCreatePath(path)

        # cleanup even more - mounts will remain in the system, but devices won't be accessible
        # although it isn't necessary to unmount, it might cause lots of confusion
        p = path.split(os.sep)
        i = 0
        for i in range(0, len(p)):
            if p[i].startswith("mnt"):
                break

        basepath = os.sep.join(p[:i])

        for mnt in os.listdir(basepath):
            self.tracer.info("unmounting obsolete mount point '%s' from previous failovers" % os.path.join(basepath, mnt))
            self._umount(os.path.join(basepath, mnt))    # don't care about success