#!/usr/bin/python3 -tt
#
#	Resource agent for monitoring Azure Scheduled Events
#
# 	License:	GNU General Public License (GPL)
#	(c) 2018 	Tobias Niekamp, Microsoft Corp.
#				and Linux-HA contributors

import os
import sys
import time
import subprocess
import json
try:
		import urllib2
		from urllib2 import URLError
except ImportError:
		import urllib.request as urllib2
		from urllib.error import URLError
import socket
from collections import defaultdict

OCF_FUNCTIONS_DIR = os.environ.get("OCF_FUNCTIONS_DIR", "%s/lib/heartbeat" % os.environ.get("OCF_ROOT"))
sys.path.append(OCF_FUNCTIONS_DIR)
import ocf

##############################################################################


VERSION = "0.20"
USER_AGENT = "Pacemaker-ResourceAgent/%s %s" % (VERSION, ocf.distro())

attr_lastDocVersion  = "azure-events-az_lastDocVersion"
attr_curNodeState = "azure-events-az_curNodeState"
attr_pendingEventIDs = "azure-events-az_pendingEventIDs"
attr_healthstate = "#health-azure"

default_loglevel = ocf.logging.INFO
default_relevantEventTypes = set(["Reboot", "Redeploy"])

##############################################################################

class attrDict(defaultdict):
	"""
	A wrapper for accessing dict keys like an attribute
	"""
	def __init__(self, data):
		super(attrDict, self).__init__(attrDict)
		for d in data.keys():
			self.__setattr__(d, data[d])

	def __getattr__(self, key):
		try:
			return self[key]
		except KeyError:
			raise AttributeError(key)

	def __setattr__(self, key, value):
		self[key] = value

##############################################################################

class azHelper:
	"""
	Helper class for Azure's metadata API (including Scheduled Events)
	"""
	metadata_host = "http://169.254.169.254/metadata"
	instance_api  = "instance"
	events_api    = "scheduledevents"
	events_api_version = "2020-07-01"
	instance_api_version = "2021-12-13"

	@staticmethod
	def _sendMetadataRequest(endpoint, postData=None, api_version="2019-08-01"):
		"""
		Send a request to Azure's Azure Metadata Service API
		"""

		retryCount = int(ocf.get_parameter("retry_count",3))
		retryWaitTime = int(ocf.get_parameter("retry_wait",20))
		requestTimeout = int(ocf.get_parameter("request_timeout",15))

		url = "%s/%s?api-version=%s" % (azHelper.metadata_host, endpoint, api_version)
		data = ""
		ocf.logger.debug("_sendMetadataRequest: begin; endpoint = %s, postData = %s, retry_count = %s, retry_wait time = %s, request_timeout = %s" % (endpoint, postData, retryCount, retryWaitTime, requestTimeout))
		ocf.logger.debug("_sendMetadataRequest: url = %s" % url)

		if postData and type(postData) != bytes:
			postData = postData.encode()

		req = urllib2.Request(url, postData)
		req.add_header("Metadata", "true")
		req.add_header("User-Agent", USER_AGENT)

		if retryCount > 0:
			ocf.logger.debug("_sendMetadataRequest: retry enabled")

		successful = None
		for retry in range(retryCount+1):
			try:
				resp = urllib2.urlopen(req, timeout=requestTimeout)
			except Exception as e:
				excType = e.__class__.__name__
				if excType == TimeoutError.__name__:
					ocf.logger.warning("Request timed out after %s seconds Error: %s" % (requestTimeout, e))
				if excType == URLError.__name__:
					if hasattr(e, 'reason'):
						ocf.logger.warning("Failed to reach the server: %s" % e.reason)
					elif hasattr(e, 'code'):
						ocf.logger.warning("The server couldn\'t fulfill the request. Error code: %s" % e.code)

				if retryCount > 1 and retry != retryCount:
					ocf.logger.warning("Request failed, retry (%s/%s) wait %s seconds before retry (wait time)" % (retry + 1,retryCount,retryWaitTime))
					time.sleep(retryWaitTime)

			else:
				data = resp.read()
				ocf.logger.debug("_sendMetadataRequest: response = %s" % data)
				successful = 1
				break

		if data:
			data = json.loads(data)

		ocf.logger.debug("_sendMetadataRequest: finished")
		return data

	@staticmethod
	def getInstanceInfo():
		"""
		Fetch details about the current VM from Azure's Azure Metadata Service API
		"""
		ocf.logger.debug("getInstanceInfo: begin")

		jsondata = azHelper._sendMetadataRequest(azHelper.instance_api, None, azHelper.instance_api_version)
		ocf.logger.debug("getInstanceInfo: json = %s" % jsondata)

		if jsondata:
			ocf.logger.debug("getInstanceInfo: finished, returning {}".format(jsondata["compute"]))
			return attrDict(jsondata["compute"])
		else:
			apiCall = "%s/%s?api-version=%s" % (azHelper.metadata_host, azHelper.instance_api, azHelper.instance_api_version)
			ocf.ocf_exit_reason("getInstanceInfo: Unable to get instance info - call: %s" % apiCall)
			sys.exit(ocf.OCF_ERR_GENERIC)

	@staticmethod
	def pullScheduledEvents():
		"""
		Retrieve all currently scheduled events via Azure Metadata Service API
		"""
		ocf.logger.debug("pullScheduledEvents: begin")

		jsondata = azHelper._sendMetadataRequest(azHelper.events_api, None, azHelper.events_api_version)
		ocf.logger.debug("pullScheduledEvents: json = %s" % jsondata)

		if jsondata:
			ocf.logger.debug("pullScheduledEvents: finished")
			return attrDict(jsondata)
		else:
			apiCall = "%s/%s?api-version=%s" % (azHelper.metadata_host, azHelper.events_api, azHelper.events_api_version)
			ocf.ocf_exit_reason("pullScheduledEvents: Unable to get scheduledevents info - call: %s" % apiCall)
			sys.exit(ocf.OCF_ERR_GENERIC)


	@staticmethod
	def forceEvents(eventIDs):
		"""
		Force a set of events to start immediately
		"""
		ocf.logger.debug("forceEvents: begin")

		events = []
		for e in eventIDs:
			events.append({
				"EventId": e,
			})
		postData = {
			"StartRequests" : events
		}
		ocf.logger.info("forceEvents: postData = %s" % postData)
		resp = azHelper._sendMetadataRequest(azHelper.events_api, postData=json.dumps(postData))

		ocf.logger.debug("forceEvents: finished")
		return

##############################################################################

class clusterHelper:
	"""
	Helper functions for Pacemaker control via crm
	"""
	@staticmethod
	def _getLocation(node):
		"""
		Helper function to retrieve local/global attributes
		"""
		if node:
			return ["--node", node]
		else:
			return ["--type", "crm_config"]

	@staticmethod
	def _exec(command, *args):
		"""
		Helper function to execute a UNIX command
		"""
		args = list(args)
		ocf.logger.debug("_exec: begin; command = %s, args = %s" % (command, str(args)))

		def flatten(*n):
			return (str(e) for a in n
				for e in (flatten(*a) if isinstance(a, (tuple, list)) else (str(a),)))
		command = list(flatten([command] + args))
		ocf.logger.debug("_exec: cmd = %s" % " ".join(command))
		try:
			ret = subprocess.check_output(command)
			if type(ret) != str:
				ret = ret.decode()
			ocf.logger.debug("_exec: return = %s" % ret)
			return ret.rstrip()
		except Exception as err:
			ocf.logger.exception(err)
			return None

	@staticmethod
	def setAttr(key, value, node=None):
		"""
		Set the value of a specific global/local attribute in the Pacemaker cluster
		"""
		ocf.logger.debug("setAttr: begin; key = %s, value = %s, node = %s" % (key, value, node))

		if value:
			ret = clusterHelper._exec("crm_attribute",
									  "--name", key,
									  "--update", value,
									  clusterHelper._getLocation(node))
		else:
			ret = clusterHelper._exec("crm_attribute",
									  "--name", key,
									  "--delete",
									  clusterHelper._getLocation(node))

		ocf.logger.debug("setAttr: finished")
		return len(ret) == 0

	@staticmethod
	def getAttr(key, node=None):
		"""
		Retrieve a global/local attribute from the Pacemaker cluster
		"""
		ocf.logger.debug("getAttr: begin; key = %s, node = %s" % (key, node))

		val = clusterHelper._exec("crm_attribute",
								  "--name", key,
								  "--query", "--quiet",
								  "--default", "",
								  clusterHelper._getLocation(node))
		ocf.logger.debug("getAttr: finished")
		if not val:
			return None
		return val if not val.isdigit() else int(val)

	@staticmethod
	def getAllNodes():
		"""
		Get a list of hostnames for all nodes in the Pacemaker cluster
		"""
		ocf.logger.debug("getAllNodes: begin")

		nodes = []
		nodeList = clusterHelper._exec("crm_node", "--list")
		for n in nodeList.split("\n"):
			nodes.append(n.split()[1])
		ocf.logger.debug("getAllNodes: finished; return %s" % str(nodes))

		return nodes

	@staticmethod
	def getHostNameFromAzName(azName):
		"""
		Helper function to get the actual host name from an Azure node name
		"""
		return clusterHelper.getAttr("hostName_%s" % azName)

	@staticmethod
	def removeHoldFromNodes():
		"""
		Remove the ON_HOLD state from all nodes in the Pacemaker cluster
		"""
		ocf.logger.debug("removeHoldFromNodes: begin")

		for n in clusterHelper.getAllNodes():
			if clusterHelper.getAttr(attr_curNodeState, node=n) == "ON_HOLD":
				clusterHelper.setAttr(attr_curNodeState, "AVAILABLE", node=n)
				ocf.logger.info("removeHoldFromNodes: removed ON_HOLD from node %s" % n)

		ocf.logger.debug("removeHoldFromNodes: finished")
		return False

	@staticmethod
	def otherNodesAvailable(exceptNode):
		"""
		Check if there are any nodes (except a given node) in the Pacemaker cluster that have state AVAILABLE
		"""
		ocf.logger.debug("otherNodesAvailable: begin; exceptNode = %s" % exceptNode)

		for n in clusterHelper.getAllNodes():
			state = clusterHelper.getAttr(attr_curNodeState, node=n)
			state = stringToNodeState(state) if state else AVAILABLE
			if state == AVAILABLE and n != exceptNode.hostName:
				ocf.logger.info("otherNodesAvailable: at least %s is available" % n)
				ocf.logger.debug("otherNodesAvailable: finished")
				return True
		ocf.logger.info("otherNodesAvailable: no other nodes are available")
		ocf.logger.debug("otherNodesAvailable: finished")

		return False

	@staticmethod
	def transitionSummary():
		"""
		Get the current Pacemaker transition summary (used to check if all resources are stopped when putting a node standby)
		"""
		# <tniek> Is a global crm_simulate "too much"? Or would it be sufficient it there are no planned transitions for a particular node?
		# # crm_simulate -LS
		# 	Transition Summary:
		# 	 * Promote rsc_SAPHana_HN1_HDB03:0      (Slave -> Master hsr3-db1)
		# 	 * Stop    rsc_SAPHana_HN1_HDB03:1      (hsr3-db0)
		# 	 * Move    rsc_ip_HN1_HDB03     (Started hsr3-db0 -> hsr3-db1)
		# 	 * Start   rsc_nc_HN1_HDB03     (hsr3-db1)
		# # Excepted result when there are no pending actions:
		# 	Transition Summary:
		ocf.logger.debug("transitionSummary: begin")

		summary = clusterHelper._exec("crm_simulate", "-LS")
		if not summary:
			ocf.logger.warning("transitionSummary: could not load transition summary")
			return ""
		if summary.find("Transition Summary:") < 0:
			ocf.logger.debug("transitionSummary: no transactions: %s" % summary)
			return ""
		j=summary.find('Transition Summary:') + len('Transition Summary:')
		l=summary.lower().find('executing cluster transition:')
		ret = list(filter(str.strip, summary[j:l].split("\n")))

		ocf.logger.debug("transitionSummary: finished; return = %s" % str(ret))
		return ret

	@staticmethod
	def listOperationsOnNode(node):
		"""
		Get a list of all current operations for a given node (used to check if any resources are pending)
		"""
		# hsr3-db1:/home/tniek # crm_resource --list-operations -N hsr3-db0
		# rsc_azure-events-az    (ocf::heartbeat:azure-events-az):      Started: rsc_azure-events-az_start_0 (node=hsr3-db0, call=91, rc=0, last-rc-change=Fri Jun  8 22:37:46 2018, exec=115ms): complete
		# rsc_azure-events-az    (ocf::heartbeat:azure-events-az):      Started: rsc_azure-events-az_monitor_10000 (node=hsr3-db0, call=93, rc=0, last-rc-change=Fri Jun  8 22:37:47 2018, exec=197ms): complete
		# rsc_SAPHana_HN1_HDB03   (ocf::suse:SAPHana):    Master: rsc_SAPHana_HN1_HDB03_start_0 (node=hsr3-db0, call=-1, rc=193, last-rc-change=Fri Jun  8 22:37:46 2018, exec=0ms): pending
		# rsc_SAPHanaTopology_HN1_HDB03   (ocf::suse:SAPHanaTopology):    Started: rsc_SAPHanaTopology_HN1_HDB03_start_0 (node=hsr3-db0, call=90, rc=0, last-rc-change=Fri Jun  8 22:37:46 2018, exec=3214ms): complete
		ocf.logger.debug("listOperationsOnNode: begin; node = %s" % node)

		resources = clusterHelper._exec("crm_resource", "--list-operations", "-N", node)
		if len(resources) == 0:
			ret = []
		else:
			ret = resources.split("\n")

		ocf.logger.debug("listOperationsOnNode: finished; return = %s" % str(ret))
		return ret

	@staticmethod
	def noPendingResourcesOnNode(node):
		"""
		Check that there are no pending resources on a given node
		"""
		ocf.logger.debug("noPendingResourcesOnNode: begin; node = %s" % node)

		for r in clusterHelper.listOperationsOnNode(node):
			ocf.logger.debug("noPendingResourcesOnNode: * %s" % r)
			resource = r.split()[-1]
			if resource == "pending":
				ocf.logger.info("noPendingResourcesOnNode: found resource %s that is still pending" % resource)
				ocf.logger.debug("noPendingResourcesOnNode: finished; return = False")
				return False
		ocf.logger.info("noPendingResourcesOnNode: no pending resources on node %s" % node)
		ocf.logger.debug("noPendingResourcesOnNode: finished; return = True")

		return True

	@staticmethod
	def allResourcesStoppedOnNode(node):
		"""
		Check that all resources on a given node are stopped
		"""
		ocf.logger.debug("allResourcesStoppedOnNode: begin; node = %s" % node)

		if clusterHelper.noPendingResourcesOnNode(node):
			if len(clusterHelper.transitionSummary()) == 0:
				ocf.logger.info("allResourcesStoppedOnNode: no pending resources on node %s and empty transition summary" % node)
				ocf.logger.debug("allResourcesStoppedOnNode: finished; return = True")
				return True
			ocf.logger.info("allResourcesStoppedOnNode: transition summary is not empty")
			ocf.logger.debug("allResourcesStoppedOnNode: finished; return = False")
			return False

		ocf.logger.info("allResourcesStoppedOnNode: still pending resources on node %s" % node)
		ocf.logger.debug("allResourcesStoppedOnNode: finished; return = False")
		return False

##############################################################################

AVAILABLE = 0	# Node is online and ready to handle events
STOPPING = 1	# Standby has been triggered, but some resources are still running
IN_EVENT = 2	# All resources are stopped, and event has been initiated via Azure Metadata Service
ON_HOLD = 3		# Node has a pending event that cannot be started there are n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              