##### RELEASE NOTES ##### # Version 1.0 # - initial version # Version 0.9 # - all dev versions, no differentiation between versions ######################### from client import StorageConnectorClient, Helper import hashlib import os, time class fcClientLVM1(StorageConnectorClient): """ Implementation of the Fiber Channel Storage Connector client for HANA. It requires multipathing to be set up with Linux dm-multipath and the sg3_utils package to be installed, due to the usage of the sg_persist tool. Note: sufficient rights must be granted without the need for typing the password, this can be achieved by: echo "adm ALL=NOPASSWD: /sbin/multipath, /sbin/multipathd, /etc/init.d/multipathd, /usr/bin/sg_persist, /bin/mount, /bin/umount, /bin/kill, /usr/bin/lsof Note: if using the --storage_cfg parameter during installation (or during addhost with already enabled Storage Connector API, the installer will take care of setting /etc/sudoers correctly) """ apiVersion = 2 # prout-type to fence the devices prType = 6 # exclusive access, registrants only # the persistent reservation key used by all nodes of an instance prKey = None interval = 1 retries = 20 def __init__(self, *args, **kwargs): # delegate construction to base class super(fcClientLVM1, self).__init__(*args, **kwargs) def about(self): return {"provider_company" : "SAP", "provider_name" : "fcClientLVM1", "provider_description" : "Generic Fiber Channel Storage Connector. For any Fiber Channel storages that fulfill the usual Linux kernel interfaces as well as support the SCSI-3 standard.", "provider_version" : "1.0"} @staticmethod def sudoers(): """Gives information about the necessary sudo permission to be set.""" return """ALL=NOPASSWD: /sbin/multipath, /sbin/multipathd, /etc/init.d/multipathd, /usr/bin/sg_persist, /bin/mount, /bin/umount, /bin/kill, /usr/bin/lsof, /sbin/vgchange, /sbin/vgscan""" def extractSingleDevices(self, result, wwid): # 360080e500017c3bc00000fab4fa0fabf dm-0 VENDOR,PRODUCTNAME ;) # size=64G features='0' hwhandler='1 rdac' wp=rw # |-+- policy='round-robin 0' prio=-1 status=active # | `- 1:0:0:0 sdb 8:16 active undef running # `-+- policy='round-robin 0' prio=-1 status=enabled # `- 2:0:0:0 sdn 8:208 active undef running lines = result.split("\n") for l in lines: if wwid in l: idx = 0 ls = l.split() for p in ls: if p.startswith("dm-"): deviceName = ls[idx] idx = idx + 1 break msg = "found '%s' as internal multipath device name for wwid '%s'" % (deviceName, wwid) self.tracer.info(msg) (code, output) = Helper._runOsCommand("ls /sys/block/%s/slaves" % deviceName, self.tracer) if code != 0: msg = "no single path devices found for wwid '%s'" % wwid print msg self.tracer.fatal(msg) raise Exception(msg) singleDevices = output.split() singleDevices = ["/dev/%s" % n for n in singleDevices] msg = " with single devices: %s" % singleDevices self.tracer.info(msg) return singleDevices def handlePRUnitAttention(self, device): """handles """ retry = 0 while True: if retry > self.retries: break (code, output) = Helper._runOsCommand("sudo /usr/bin/sg_persist -r %s" % device, self.tracer) if code == 0 and "PR generation=" in output: return time.sleep(self.interval) retry = retry+1 msg = "unable to handle UNIT ATTENTION or device not ready after %s seconds" % (self.interval * self.retries) self.tracer.warning(msg) # request the storage server to activate a LUN for this host def attach(self, storages): """Attaches storages on this host.""" self.tracer.debug("%s.attach method called" % self.__class__.__name__) self.prKey = hashlib.md5(self.sid).hexdigest()[:16] # SCSI-3 PR keys are only 64bit long self.hookAttachBegin(globals(), locals()) firstDev = True # loop over storages given by HDB for storage in storages: self.hookAttachBeginDevice(globals(), locals()) self.tracer.info("trying to attach for partition %s, usage type %s on path %s" % (storage.get("partition"), storage.get("usage_type"), storage.get("path"))) # retrieve all parameters for this storage (unique key => (partition, usageType) ) if not self._cfg.hasParameter(storage.get("partition"), storage.get("usage_type"), "lvmname"): msg = "no storage with key (partiton, usageType) = (%s, %s) configured" % (storage.get("partition"), storage.get("usage_type")) print msg self.tracer.fatal(msg) raise Exception(msg) ################################################################################################### connectionData = self._getConnectionDataForLun(storage.get("partition"), storage.get("usage_type")) try: self.prType = connectionData["prtype"] except: pass if not int(self.prType) in [5, 6]: raise Exception("unsupported prout-type '%s' for persistent reservation" % self.prType) if int(self.prType) == 5: self.prTypeReservationText = "Write Exclusive" if int(self.prType) == 6: self.prTypeReservationText = "Exclusive Access" self.tracer.info("using --prout-type=%s for persisten reservations" % self.prType) if firstDev: # some migration stuff when switching from prType 6 to 5: (code, output) = Helper._runOsCommand("sudo /sbin/multipath -r", self.tracer) if "reject" in output.lower() or int(self.prType) == 6: (code, output) = Helper._runOsCommand("sudo /etc/init.d/multipathd force-reload", self.tracer) if code != 0: msg = "could not reload multipath topology: %s" % output print msg self.tracer.error(msg) raise Exception(msg) firstDev = False # set variables for better code readability try: lvmname = connectionData["lvmname"] except: raise Exception("lvmname not set in global.ini") (c ,o) = Helper._runOsCommand("sudo /sbin/vgscan", self.tracer) if c != 0: msg = "vgscan failed for device: %s" % o print msg self.tracer.fatal(msg) raise Exception(msg) time.sleep(1) vg = lvmname.rsplit("-", 1)[0] (c ,o) = Helper._runOsCommand("sudo /sbin/vgchange -ay %s" % vg, self.tracer) if c != 0: msg = "vgchange failed for volume group '%s': %s" % (vg, o) print msg self.tracer.fatal(msg) raise Exception(msg) ############ devices = {} device = "/dev/mapper/%s" % lvmname found = False for i in range(1, 4): if os.path.exists(device): found = True break; time.sleep(1) if not found: raise Exception("LVM device '%s' not found" % device) linkIsCreated = True dmdev = os.path.realpath(device).split("/")[-1] if not dmdev.startswith("dm-"): linkIsCreated = False for i in range(1, 11): # poll up to 10 seconds, because /dev/disk/by-id is updated ASYNC only by the kernel! (c, o) = Helper._run2PipedOsCommand("ls -ls /dev/disk/by-id", "grep -w %s" % lvmname, self.tracer) dmdev = o.split("/")[-1] time.sleep(1) if dmdev.startswith("dm-"): linkIsCreated = True break; if not linkIsCreated: msg = "unable to resolve '%s' to a 'dm-X' name" % lvmname print msg self.tracer.fatal(msg) raise Exception(msg) dmdev = dmdev.strip() self.tracer.info("device is %s" % dmdev) (code, output) = Helper._runOsCommand("ls /sys/block/%s/slaves" % dmdev, self.tracer) for dm in output.split(): (code2, output2) = Helper._runOsCommand("sudo /sbin/multipath -l", self.tracer) for l in output2.split("\n"): if " %s " % dm in l: try: wwid = l.split("(")[1].split(")")[0] # with alias except: wwid = l.split()[0] #without alias devices[wwid] = "/dev/mapper/%s" % lvmname for (wwid, dev) in devices.items(): self.tracer.info("%s will be attached" % dev) ################################################################################################### self.hookAttachPrePRClear(globals(), locals()) (code, output) = Helper._runOsCommand("sudo /sbin/multipath -l", self.tracer) singleDevices = [] for (wwid, dev) in devices.items(): singleDevices.extend(self.extractSingleDevices(output, wwid)) success = False for d in singleDevices: self.handlePRUnitAttention(d) # register PR key on the device, for removing existing reservations (c1, _) = Helper._runOsCommand("sudo /usr/bin/sg_persist --out --register --param-sark=%s %s" % (self.prKey, d), self.tracer) # clear all PR keys (c2, _) = Helper._runOsCommand("sudo /usr/bin/sg_persist --out --clear --param-rk=%s %s" % (self.prKey, d), self.tracer) success = success or c2 == 0 if success == False: msg = "unable to find available device for writing SCSI-3 persistent reservation" print msg self.tracer.fatal(msg) raise Exception(msg) for d in singleDevices: self.handlePRUnitAttention(d) # check PR key (code, output) = Helper._runOsCommand("sudo /usr/bin/sg_persist -i -k %s" % d, self.tracer) if code != 0 or "there are NO registered reservation keys" not in output: msg = "unable to find PR key on device '%s': %s" % (device, output) print msg self.tracer.fatal(msg) raise Exception(msg) ################################################################################################### self.hookAttachPreCleanupUmount(globals(), locals()) # unmount everything mountList = set() mountList.update(self._isMounted(storage.get("path"))) for (wwid, dev) in devices.items(): mountList.update(self._isMounted(device)) for (dev, mountpoint) in mountList: msg = "unmounting device '%s' from '%s'" % (dev, mountpoint) self.tracer.info(msg) # unmount unnecessary mounts while self._umount(mountpoint): pass #check again, if umount was successful checkMounts = self._isMounted(mountpoint) # if not, try to send SIGKILL to running processes if len(checkMounts) > 0: self.tracer.warning("try to 'kill -9' processes blocking the mountpoint") self._lsof_and_kill(mountpoint) # try umount agian while self._umount(mountpoint): pass # check once more # if device is still mounted, end processing and inform about need administrator action checkMounts = self._isMounted(mountpoint) if len(checkMounts) > 0: msg = "unable to unmount path '%s' (mounted to '%s'), reboot may be required" % (storage.get("path"), dev) print msg self.tracer.fatal(msg) raise Exception(msg) # this will create the mountpoint if necessary path = self._checkAndCreatePath(storage.get("path")) # cleanup even more - mounts will remain in the system, but devices won't be accessible # although it isn't necessary to unmount, it might cause lots of confusion p = path.split(os.sep) i = 0 for i in range(0, len(p)): if p[i].startswith("mnt"): break; basepath = os.sep.join(p[:i]) for mnt in os.listdir(basepath): self.tracer.info("unmounting obsolete mount '%s'" % os.path.join(basepath, mnt)) self._umount(os.path.join(basepath, mnt)) # don't care about success ################################################################################################### self.hookAttachPrePRRegister(globals(), locals()) for d in singleDevices: self.handlePRUnitAttention(d) # register key Helper._runOsCommand("sudo /usr/bin/sg_persist --out --register --param-sark=%s %s" % (self.prKey, d), self.tracer) self.handlePRUnitAttention(d) # reserve with prKey Helper._runOsCommand("sudo /usr/bin/sg_persist --out --reserve --param-rk=%s --prout-type=%s %s" % (self.prKey, self.prType, d), self.tracer) self.handlePRUnitAttention(d) # check actual reservation (code, output) = Helper._runOsCommand("sudo /usr/bin/sg_persist -r %s" % d, self.tracer) if code != 0 or self.prTypeReservationText not in output: msg = "reservation of Persistent Reservation key failed for device '%s': %s" % (d, output) print msg self.tracer.fatal(msg) raise Exception(msg) ################################################################################################### vg = lvmname.rsplit("-", 1)[0] (c ,o) = Helper._runOsCommand("sudo /sbin/vgchange -ay %s" % vg, self.tracer) if c != 0: msg = "vgchange failed for device '%s': %s" % (vg, o) print msg self.tracer.fatal(msg) raise Exception(msg) ################################################################################################### self.hookAttachPreMount(globals(), locals()) # it might take a short time, if a reservation was updated, until the device is ready to be mounted # therefore, try several times... try: mo = connectionData["mountoptions"] except: mo = "" self.tracer.info("using mount options: '%s'" % mo) retry = 0 while True: if retry > self.retries: self.tracer.fatal("mount failed after multiple retries") raise Exception("mount failed after multiple retries") try: self.tracer.info("mount '%s' to '%s' with options '%s'" % (device, path, mo)) self._mount(device, path, mo) except: done = False time.sleep(self.interval) if self._isMounted(path): self.tracer.debug("device was mounted after %s retries" % retry) break; retry = retry+1 if not done: self._umount(path) msg = "attached device '%s' to path '%s'" % (device, path) print msg self.tracer.info(msg) self.hookAttachEndDevice(globals(), locals()) self.hookAttachEnd(globals(), locals()) if int(self.prType) == 6: # if a device is polled, it will not be flushed, therefore, run flush several times for i in range(1, 3): Helper._runOsCommand("sudo /sbin/multipath -F", self.tracer) time.sleep(0.1) return 0 def detach(self, storages): self.hookDetachBegin(globals(), locals()) self.tracer.debug("%s.detach method called" % self.__class__.__name__) self.prKey = hashlib.md5(self.sid).hexdigest()[:16] # SCSI-3 PR keys are only 64bit long # helping variable; if at least one storage connect fails, this is set to False # only if all storages are connected successfully, True will be returned success = True # loop over storages given by HDB for storage in storages: self.hookDetachBeginDevice(globals(), locals()) # extract necessary information path = self._checkAndCreatePath(storage.get("path")) ################################################################################################### self.hookDetachPreUmount(globals(), locals()) # check current mounts mountList = self._isMounted(path) # if no device is attached at all, end processing if len(mountList) != 0: # try to unmount device while self._umount(path): pass else: (c ,o) = Helper._runOsCommand("sudo /sbin/vgchange -an %s" % vg, self.tracer) if c != 0: msg = "vgchange failed for device '%s' : %s" % (vg, o) print msg self.tracer.fatal(msg) raise Exception(msg) msg = "no device attached to path '%s'" % path print msg self.tracer.info(msg) continue # check again if it was successful checkMounts = self._isMounted(path) if len(checkMounts) > 0: self.tracer.warning("device on path '%s'busy, trying lazy umount" % path) # send umount again, this time lazy while self._umount(path, True): pass devices = {} # cleanup of the devices => in normal case |mountList|=1 for (d, _) in mountList: # look up current single devices dmdev = os.path.realpath(d).split("/")[-1] device = d vg = device.rsplit("/", 1)[1].rsplit("-", 1)[0] (c ,o) = Helper._runOsCommand("sudo /sbin/vgchange -an %s" % vg, self.tracer) if c != 0: msg = "vgchange failed for device '%s' : %s" % (vg, o) print msg self.tracer.fatal(msg) raise Exception(msg) (code, output) = Helper._runOsCommand("ls /sys/block/%s/slaves" % dmdev, self.tracer) for dm in output.split(): (code2, output2) = Helper._runOsCommand("sudo /sbin/multipath -l", self.tracer) for l in output2.split("\n"): if " %s " % dm in l: try: wwid = l.split("(")[1].split(")")[0] # with alais except: wwid = l.split()[0] #without alias devices[wwid] = d singleDevices = [] for (wwid, dev) in devices.items(): (code, output) = Helper._runOsCommand("sudo /sbin/multipath -l %s" % wwid, self.tracer) singleDevices.extend(self.extractSingleDevices(output, wwid)) self.hookDetachPrePRClear(globals(), locals()) for d in singleDevices: self.handlePRUnitAttention(d) # clear PR; no error handling, since this command might safely fail in case someone else than the reservation owner tries to detach (code, output) = Helper._runOsCommand("sudo /usr/bin/sg_persist --out --clear --param-rk=%s %s" % (self.prKey, d), self.tracer) ################################################################################################### msg = "detached device '%s' from path '%s'" % (device, path) print msg self.tracer.info(msg) self.hookDetachEndDevice(globals(), locals()) connectionData = self._getConnectionDataForLun(storage.get("partition"), storage.get("usage_type")) try: self.prType = connectionData["prtype"] except: pass if int(self.prType) == 6: # if a device is polled, it will not be flushed, therefore, run flush several times for i in range(1, 3): Helper._runOsCommand("sudo /sbin/multipath -F", self.tracer) time.sleep(0.1) self.hookDetachEnd(globals(), locals()) return not success def info(self, paths): self.tracer.debug("%s.info method called" % self.__class__.__name__) mappings = [] # retrieve information for each path given by HDB for path in paths: # determine real OS path without symlinks and retrieve the mounted devices path = os.path.realpath(path) mountList = self._isMounted(path) # no device is mounted if len(mountList) == 0: mappings.append({"path" : path, "OS Device Name" : "not_connected"}) continue if len(mountList) > 1: mappings.append({"path" : path, "OS Device Name" : "found_multiple_mounts_on_path"}) continue devices = {} for ml in mountList: dmdev = os.path.realpath(ml[0]).split("/")[-1] (code, output) = Helper._runOsCommand("ls /sys/block/%s/slaves" % dmdev, self.tracer) for dm in output.split(): (code2, output2) = Helper._runOsCommand("sudo /sbin/multipath -l", self.tracer) for l in output2.split("\n"): if " %s " % dm in l: try: wwid = l.split("(")[1].split(")")[0] # with alais except: wwid = l.split()[0] #without alias devices[wwid] = ml[0] # filesystem type (code, output) = Helper._run2PipedOsCommand("cat /proc/mounts", "grep -w %s" % path) if not code == 0: self.tracer.warning("error running cat /proc/mounts: code %s: %s" % (code, output)) fstype = "?" else: fstype = output.split()[2] wwids= [] for wwid, dev in devices.items(): wwids.append(wwid) # combine all extracted information mappings.append({ "path" : path, "OS Filesystem Type" : fstype, "WWIDs" : str(wwids), }) return mappings def hookBeginAttach(self, globals, locals): pass def hookAttachBegin(self, globals, locals): pass def hookAttachBeginDevice(self, globals, locals): pass def hookAttachPrePRClear(self, globals, locals): pass def hookAttachPreCleanupUmount(self, globals, locals): pass def hookAttachPrePRRegister(self, globals, locals): pass def hookAttachPreMount(self, globals, locals): pass def hookAttachEndDevice(self, globals, locals): pass def hookAttachEnd(self, globals, locals): pass def hookDetachBegin(self, globals, locals): pass def hookDetachBeginDevice(self, globals, locals): pass def hookDetachPreUmount(self, globals, locals): pass def hookDetachPrePRClear(self, globals, locals): pass def hookDetachEndDevice(self, globals, locals): pass def hookDetachEnd(self, globals, locals): pass