| #!/usr/bin/python |
| # |
| |
| # Copyright (C) 2010, 2012 Google Inc. |
| # All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # 1. Redistributions of source code must retain the above copyright notice, |
| # this list of conditions and the following disclaimer. |
| # |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS |
| # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR |
| # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """Tool to merge two or more clusters together. |
| |
| The clusters have to run the same version of Ganeti! |
| |
| """ |
| |
| # pylint: disable=C0103 |
| # C0103: Invalid name cluster-merge |
| |
| import logging |
| import os |
| import optparse |
| import shutil |
| import sys |
| import tempfile |
| |
| from ganeti import cli |
| from ganeti import config |
| from ganeti import constants |
| from ganeti import errors |
| from ganeti import ssh |
| from ganeti import utils |
| from ganeti import pathutils |
| from ganeti import compat |
| |
| |
| _GROUPS_MERGE = "merge" |
| _GROUPS_RENAME = "rename" |
| _CLUSTERMERGE_ECID = "clustermerge-ecid" |
| _RESTART_ALL = "all" |
| _RESTART_UP = "up" |
| _RESTART_NONE = "none" |
| _RESTART_CHOICES = (_RESTART_ALL, _RESTART_UP, _RESTART_NONE) |
| _PARAMS_STRICT = "strict" |
| _PARAMS_WARN = "warn" |
| _PARAMS_CHOICES = (_PARAMS_STRICT, _PARAMS_WARN) |
| |
| |
| PAUSE_PERIOD_OPT = cli.cli_option("-p", "--watcher-pause-period", default=1800, |
| action="store", type="int", |
| dest="pause_period", |
| help=("Amount of time in seconds watcher" |
| " should be suspended from running")) |
| GROUPS_OPT = cli.cli_option("--groups", default=None, metavar="STRATEGY", |
| choices=(_GROUPS_MERGE, _GROUPS_RENAME), |
| dest="groups", |
| help=("How to handle groups that have the" |
| " same name (One of: %s/%s)" % |
| (_GROUPS_MERGE, _GROUPS_RENAME))) |
| PARAMS_OPT = cli.cli_option("--parameter-conflicts", default=_PARAMS_STRICT, |
| metavar="STRATEGY", |
| choices=_PARAMS_CHOICES, |
| dest="params", |
| help=("How to handle params that have" |
| " different values (One of: %s/%s)" % |
| _PARAMS_CHOICES)) |
| |
| RESTART_OPT = cli.cli_option("--restart", default=_RESTART_ALL, |
| metavar="STRATEGY", |
| choices=_RESTART_CHOICES, |
| dest="restart", |
| help=("How to handle restarting instances" |
| " same name (One of: %s/%s/%s)" % |
| _RESTART_CHOICES)) |
| |
| SKIP_STOP_INSTANCES_OPT = \ |
| cli.cli_option("--skip-stop-instances", default=True, action="store_false", |
| dest="stop_instances", |
| help=("Don't stop the instances on the clusters, just check " |
| "that none is running")) |
| |
| |
| def Flatten(unflattened_list): |
| """Flattens a list. |
| |
| @param unflattened_list: A list of unflattened list objects. |
| @return: A flattened list |
| |
| """ |
| flattened_list = [] |
| |
| for item in unflattened_list: |
| if isinstance(item, list): |
| flattened_list.extend(Flatten(item)) |
| else: |
| flattened_list.append(item) |
| return flattened_list |
| |
| |
| class MergerData(object): |
| """Container class to hold data used for merger. |
| |
| """ |
| def __init__(self, cluster, key_path, nodes, instances, master_node, |
| config_path=None): |
| """Initialize the container. |
| |
| @param cluster: The name of the cluster |
| @param key_path: Path to the ssh private key used for authentication |
| @param nodes: List of online nodes in the merging cluster |
| @param instances: List of instances running on merging cluster |
| @param master_node: Name of the master node |
| @param config_path: Path to the merging cluster config |
| |
| """ |
| self.cluster = cluster |
| self.key_path = key_path |
| self.nodes = nodes |
| self.instances = instances |
| self.master_node = master_node |
| self.config_path = config_path |
| |
| |
| class Merger(object): |
| """Handling the merge. |
| |
| """ |
| RUNNING_STATUSES = compat.UniqueFrozenset([ |
| constants.INSTST_RUNNING, |
| constants.INSTST_ERRORUP, |
| ]) |
| |
| def __init__(self, clusters, pause_period, groups, restart, params, |
| stop_instances): |
| """Initialize object with sane defaults and infos required. |
| |
| @param clusters: The list of clusters to merge in |
| @param pause_period: The time watcher shall be disabled for |
| @param groups: How to handle group conflicts |
| @param restart: How to handle instance restart |
| @param stop_instances: Indicates whether the instances must be stopped |
| (True) or if the Merger must only check if no |
| instances are running on the mergee clusters (False) |
| |
| """ |
| self.merger_data = [] |
| self.clusters = clusters |
| self.pause_period = pause_period |
| self.work_dir = tempfile.mkdtemp(suffix="cluster-merger") |
| (self.cluster_name, ) = cli.GetClient().QueryConfigValues(["cluster_name"]) |
| self.ssh_runner = ssh.SshRunner(self.cluster_name) |
| self.groups = groups |
| self.restart = restart |
| self.params = params |
| self.stop_instances = stop_instances |
| if self.restart == _RESTART_UP: |
| raise NotImplementedError |
| |
| def Setup(self): |
| """Sets up our end so we can do the merger. |
| |
| This method is setting us up as a preparation for the merger. |
| It makes the initial contact and gathers information needed. |
| |
| @raise errors.RemoteError: for errors in communication/grabbing |
| |
| """ |
| (remote_path, _, _) = ssh.GetUserFiles("root") |
| |
| if self.cluster_name in self.clusters: |
| raise errors.CommandError("Cannot merge cluster %s with itself" % |
| self.cluster_name) |
| |
| # Fetch remotes private key |
| for cluster in self.clusters: |
| result = self._RunCmd(cluster, "cat %s" % remote_path, batch=False, |
| ask_key=False) |
| if result.failed: |
| raise errors.RemoteError("There was an error while grabbing ssh private" |
| " key from %s. Fail reason: %s; output: %s" % |
| (cluster, result.fail_reason, result.output)) |
| |
| key_path = utils.PathJoin(self.work_dir, cluster) |
| utils.WriteFile(key_path, mode=0600, data=result.stdout) |
| |
| result = self._RunCmd(cluster, "gnt-node list -o name,offline" |
| " --no-headers --separator=,", private_key=key_path) |
| if result.failed: |
| raise errors.RemoteError("Unable to retrieve list of nodes from %s." |
| " Fail reason: %s; output: %s" % |
| (cluster, result.fail_reason, result.output)) |
| nodes_statuses = [line.split(",") for line in result.stdout.splitlines()] |
| nodes = [node_status[0] for node_status in nodes_statuses |
| if node_status[1] == "N"] |
| |
| result = self._RunCmd(cluster, "gnt-instance list -o name --no-headers", |
| private_key=key_path) |
| if result.failed: |
| raise errors.RemoteError("Unable to retrieve list of instances from" |
| " %s. Fail reason: %s; output: %s" % |
| (cluster, result.fail_reason, result.output)) |
| instances = result.stdout.splitlines() |
| |
| path = utils.PathJoin(pathutils.DATA_DIR, "ssconf_%s" % |
| constants.SS_MASTER_NODE) |
| result = self._RunCmd(cluster, "cat %s" % path, private_key=key_path) |
| if result.failed: |
| raise errors.RemoteError("Unable to retrieve the master node name from" |
| " %s. Fail reason: %s; output: %s" % |
| (cluster, result.fail_reason, result.output)) |
| master_node = result.stdout.strip() |
| |
| self.merger_data.append(MergerData(cluster, key_path, nodes, instances, |
| master_node)) |
| |
| def _PrepareAuthorizedKeys(self): |
| """Prepare the authorized_keys on every merging node. |
| |
| This method add our public key to remotes authorized_key for further |
| communication. |
| |
| """ |
| (_, pub_key_file, auth_keys) = ssh.GetUserFiles("root") |
| pub_key = utils.ReadFile(pub_key_file) |
| |
| for data in self.merger_data: |
| for node in data.nodes: |
| result = self._RunCmd(node, ("cat >> %s << '!EOF.'\n%s!EOF.\n" % |
| (auth_keys, pub_key)), |
| private_key=data.key_path, max_attempts=3) |
| |
| if result.failed: |
| raise errors.RemoteError("Unable to add our public key to %s in %s." |
| " Fail reason: %s; output: %s" % |
| (node, data.cluster, result.fail_reason, |
| result.output)) |
| |
| def _RunCmd(self, hostname, command, user="root", use_cluster_key=False, |
| strict_host_check=False, private_key=None, batch=True, |
| ask_key=False, max_attempts=1): |
| """Wrapping SshRunner.Run with default parameters. |
| |
| For explanation of parameters see L{ganeti.ssh.SshRunner.Run}. |
| |
| """ |
| for _ in range(max_attempts): |
| result = self.ssh_runner.Run(hostname=hostname, command=command, |
| user=user, use_cluster_key=use_cluster_key, |
| strict_host_check=strict_host_check, |
| private_key=private_key, batch=batch, |
| ask_key=ask_key) |
| if not result.failed: |
| break |
| |
| return result |
| |
| def _CheckRunningInstances(self): |
| """Checks if on the clusters to be merged there are running instances |
| |
| @rtype: boolean |
| @return: True if there are running instances, False otherwise |
| |
| """ |
| for cluster in self.clusters: |
| result = self._RunCmd(cluster, "gnt-instance list -o status") |
| if self.RUNNING_STATUSES.intersection(result.output.splitlines()): |
| return True |
| |
| return False |
| |
| def _StopMergingInstances(self): |
| """Stop instances on merging clusters. |
| |
| """ |
| for cluster in self.clusters: |
| result = self._RunCmd(cluster, "gnt-instance shutdown --all" |
| " --force-multiple") |
| |
| if result.failed: |
| raise errors.RemoteError("Unable to stop instances on %s." |
| " Fail reason: %s; output: %s" % |
| (cluster, result.fail_reason, result.output)) |
| |
| def _DisableWatcher(self): |
| """Disable watch on all merging clusters, including ourself. |
| |
| """ |
| for cluster in ["localhost"] + self.clusters: |
| result = self._RunCmd(cluster, "gnt-cluster watcher pause %d" % |
| self.pause_period) |
| |
| if result.failed: |
| raise errors.RemoteError("Unable to pause watcher on %s." |
| " Fail reason: %s; output: %s" % |
| (cluster, result.fail_reason, result.output)) |
| |
| def _RemoveMasterIps(self): |
| """Removes the master IPs from the master nodes of each cluster. |
| |
| """ |
| for data in self.merger_data: |
| result = self._RunCmd(data.master_node, |
| "gnt-cluster deactivate-master-ip --yes") |
| |
| if result.failed: |
| raise errors.RemoteError("Unable to remove master IP on %s." |
| " Fail reason: %s; output: %s" % |
| (data.master_node, |
| result.fail_reason, |
| result.output)) |
| |
| def _StopDaemons(self): |
| """Stop all daemons on merging nodes. |
| |
| """ |
| cmd = "%s stop-all" % pathutils.DAEMON_UTIL |
| for data in self.merger_data: |
| for node in data.nodes: |
| result = self._RunCmd(node, cmd, max_attempts=3) |
| |
| if result.failed: |
| raise errors.RemoteError("Unable to stop daemons on %s." |
| " Fail reason: %s; output: %s." % |
| (node, result.fail_reason, result.output)) |
| |
| def _FetchRemoteConfig(self): |
| """Fetches and stores remote cluster config from the master. |
| |
| This step is needed before we can merge the config. |
| |
| """ |
| for data in self.merger_data: |
| result = self._RunCmd(data.cluster, "cat %s" % |
| pathutils.CLUSTER_CONF_FILE) |
| |
| if result.failed: |
| raise errors.RemoteError("Unable to retrieve remote config on %s." |
| " Fail reason: %s; output %s" % |
| (data.cluster, result.fail_reason, |
| result.output)) |
| |
| data.config_path = utils.PathJoin(self.work_dir, "%s_config.data" % |
| data.cluster) |
| utils.WriteFile(data.config_path, data=result.stdout) |
| |
| # R0201: Method could be a function |
| def _KillMasterDaemon(self): # pylint: disable=R0201 |
| """Kills the local master daemon. |
| |
| @raise errors.CommandError: If unable to kill |
| |
| """ |
| result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-master"]) |
| if result.failed: |
| raise errors.CommandError("Unable to stop master daemons." |
| " Fail reason: %s; output: %s" % |
| (result.fail_reason, result.output)) |
| |
| def _MergeConfig(self): |
| """Merges all foreign config into our own config. |
| |
| """ |
| my_config = config.ConfigWriter(offline=True) |
| fake_ec_id = 0 # Needs to be uniq over the whole config merge |
| |
| for data in self.merger_data: |
| other_config = config.ConfigWriter(data.config_path, accept_foreign=True) |
| self._MergeClusterConfigs(my_config, other_config) |
| self._MergeNodeGroups(my_config, other_config) |
| |
| for node in other_config.GetNodeList(): |
| node_info = other_config.GetNodeInfo(node) |
| # Offline the node, it will be reonlined later at node readd |
| node_info.master_candidate = False |
| node_info.drained = False |
| node_info.offline = True |
| my_config.AddNode(node_info, _CLUSTERMERGE_ECID + str(fake_ec_id)) |
| fake_ec_id += 1 |
| |
| for instance in other_config.GetInstanceList(): |
| instance_info = other_config.GetInstanceInfo(instance) |
| |
| # Update the DRBD port assignments |
| # This is a little bit hackish |
| for dsk in instance_info.disks: |
| if dsk.dev_type in constants.DTS_DRBD: |
| port = my_config.AllocatePort() |
| |
| logical_id = list(dsk.logical_id) |
| logical_id[2] = port |
| dsk.logical_id = tuple(logical_id) |
| |
| my_config.AddInstance(instance_info, |
| _CLUSTERMERGE_ECID + str(fake_ec_id)) |
| fake_ec_id += 1 |
| |
| def _MergeClusterConfigs(self, my_config, other_config): |
| """Checks that all relevant cluster parameters are compatible |
| |
| """ |
| my_cluster = my_config.GetClusterInfo() |
| other_cluster = other_config.GetClusterInfo() |
| err_count = 0 |
| |
| # |
| # Generic checks |
| # |
| check_params = [ |
| "beparams", |
| "default_iallocator", |
| "drbd_usermode_helper", |
| "hidden_os", |
| "maintain_node_health", |
| "master_netdev", |
| "ndparams", |
| "nicparams", |
| "primary_ip_family", |
| "tags", |
| "uid_pool", |
| ] |
| check_params_strict = [ |
| "volume_group_name", |
| ] |
| if my_cluster.IsFileStorageEnabled() or \ |
| other_cluster.IsFileStorageEnabled(): |
| check_params_strict.append("file_storage_dir") |
| if my_cluster.IsSharedFileStorageEnabled() or \ |
| other_cluster.IsSharedFileStorageEnabled(): |
| check_params_strict.append("shared_file_storage_dir") |
| check_params.extend(check_params_strict) |
| |
| if self.params == _PARAMS_STRICT: |
| params_strict = True |
| else: |
| params_strict = False |
| |
| for param_name in check_params: |
| my_param = getattr(my_cluster, param_name) |
| other_param = getattr(other_cluster, param_name) |
| if my_param != other_param: |
| logging.error("The value (%s) of the cluster parameter %s on %s" |
| " differs to this cluster's value (%s)", |
| other_param, param_name, other_cluster.cluster_name, |
| my_param) |
| if params_strict or param_name in check_params_strict: |
| err_count += 1 |
| |
| # |
| # Custom checks |
| # |
| |
| # Check default hypervisor |
| my_defhyp = my_cluster.enabled_hypervisors[0] |
| other_defhyp = other_cluster.enabled_hypervisors[0] |
| if my_defhyp != other_defhyp: |
| logging.warning("The default hypervisor (%s) differs on %s, new" |
| " instances will be created with this cluster's" |
| " default hypervisor (%s)", other_defhyp, |
| other_cluster.cluster_name, my_defhyp) |
| |
| if (set(my_cluster.enabled_hypervisors) != |
| set(other_cluster.enabled_hypervisors)): |
| logging.error("The set of enabled hypervisors (%s) on %s differs to" |
| " this cluster's set (%s)", |
| other_cluster.enabled_hypervisors, |
| other_cluster.cluster_name, my_cluster.enabled_hypervisors) |
| err_count += 1 |
| |
| # Check hypervisor params for hypervisors we care about |
| for hyp in my_cluster.enabled_hypervisors: |
| for param in my_cluster.hvparams[hyp]: |
| my_value = my_cluster.hvparams[hyp][param] |
| other_value = other_cluster.hvparams[hyp][param] |
| if my_value != other_value: |
| logging.error("The value (%s) of the %s parameter of the %s" |
| " hypervisor on %s differs to this cluster's parameter" |
| " (%s)", |
| other_value, param, hyp, other_cluster.cluster_name, |
| my_value) |
| if params_strict: |
| err_count += 1 |
| |
| # Check os hypervisor params for hypervisors we care about |
| for os_name in set(my_cluster.os_hvp.keys() + other_cluster.os_hvp.keys()): |
| for hyp in my_cluster.enabled_hypervisors: |
| my_os_hvp = self._GetOsHypervisor(my_cluster, os_name, hyp) |
| other_os_hvp = self._GetOsHypervisor(other_cluster, os_name, hyp) |
| if my_os_hvp != other_os_hvp: |
| logging.error("The OS parameters (%s) for the %s OS for the %s" |
| " hypervisor on %s differs to this cluster's parameters" |
| " (%s)", |
| other_os_hvp, os_name, hyp, other_cluster.cluster_name, |
| my_os_hvp) |
| if params_strict: |
| err_count += 1 |
| |
| # |
| # Warnings |
| # |
| if my_cluster.modify_etc_hosts != other_cluster.modify_etc_hosts: |
| logging.warning("The modify_etc_hosts value (%s) differs on %s," |
| " this cluster's value (%s) will take precedence", |
| other_cluster.modify_etc_hosts, |
| other_cluster.cluster_name, |
| my_cluster.modify_etc_hosts) |
| |
| if my_cluster.modify_ssh_setup != other_cluster.modify_ssh_setup: |
| logging.warning("The modify_ssh_setup value (%s) differs on %s," |
| " this cluster's value (%s) will take precedence", |
| other_cluster.modify_ssh_setup, |
| other_cluster.cluster_name, |
| my_cluster.modify_ssh_setup) |
| |
| # |
| # Actual merging |
| # |
| my_cluster.reserved_lvs = list(set(my_cluster.reserved_lvs + |
| other_cluster.reserved_lvs)) |
| |
| if my_cluster.prealloc_wipe_disks != other_cluster.prealloc_wipe_disks: |
| logging.warning("The prealloc_wipe_disks value (%s) on %s differs to this" |
| " cluster's value (%s). The least permissive value (%s)" |
| " will be used", other_cluster.prealloc_wipe_disks, |
| other_cluster.cluster_name, |
| my_cluster.prealloc_wipe_disks, True) |
| my_cluster.prealloc_wipe_disks = True |
| |
| for os_, osparams in other_cluster.osparams.items(): |
| if os_ not in my_cluster.osparams: |
| my_cluster.osparams[os_] = osparams |
| elif my_cluster.osparams[os_] != osparams: |
| logging.error("The OS parameters (%s) for the %s OS on %s differs to" |
| " this cluster's parameters (%s)", |
| osparams, os_, other_cluster.cluster_name, |
| my_cluster.osparams[os_]) |
| if params_strict: |
| err_count += 1 |
| |
| if err_count: |
| raise errors.ConfigurationError("Cluster config for %s has incompatible" |
| " values, please fix and re-run" % |
| other_cluster.cluster_name) |
| |
| # R0201: Method could be a function |
| def _GetOsHypervisor(self, cluster, os_name, hyp): # pylint: disable=R0201 |
| if os_name in cluster.os_hvp: |
| return cluster.os_hvp[os_name].get(hyp, None) |
| else: |
| return None |
| |
| # R0201: Method could be a function |
| def _MergeNodeGroups(self, my_config, other_config): |
| """Adds foreign node groups |
| |
| ConfigWriter.AddNodeGroup takes care of making sure there are no conflicts. |
| """ |
| # pylint: disable=R0201 |
| logging.info("Node group conflict strategy: %s", self.groups) |
| |
| my_grps = my_config.GetAllNodeGroupsInfo().values() |
| other_grps = other_config.GetAllNodeGroupsInfo().values() |
| |
| # Check for node group naming conflicts: |
| conflicts = [] |
| for other_grp in other_grps: |
| for my_grp in my_grps: |
| if other_grp.name == my_grp.name: |
| conflicts.append(other_grp) |
| |
| if conflicts: |
| conflict_names = utils.CommaJoin([g.name for g in conflicts]) |
| logging.info("Node groups in both local and remote cluster: %s", |
| conflict_names) |
| |
| # User hasn't specified how to handle conflicts |
| if not self.groups: |
| raise errors.CommandError("The following node group(s) are in both" |
| " clusters, and no merge strategy has been" |
| " supplied (see the --groups option): %s" % |
| conflict_names) |
| |
| # User wants to rename conflicts |
| elif self.groups == _GROUPS_RENAME: |
| for grp in conflicts: |
| new_name = "%s-%s" % (grp.name, other_config.GetClusterName()) |
| logging.info("Renaming remote node group from %s to %s" |
| " to resolve conflict", grp.name, new_name) |
| grp.name = new_name |
| |
| # User wants to merge conflicting groups |
| elif self.groups == _GROUPS_MERGE: |
| for other_grp in conflicts: |
| logging.info("Merging local and remote '%s' groups", other_grp.name) |
| for node_name in other_grp.members[:]: |
| node = other_config.GetNodeInfo(node_name) |
| # Access to a protected member of a client class |
| # pylint: disable=W0212 |
| other_config._UnlockedRemoveNodeFromGroup(node) |
| |
| # Access to a protected member of a client class |
| # pylint: disable=W0212 |
| my_grp_uuid = my_config._UnlockedLookupNodeGroup(other_grp.name) |
| |
| # Access to a protected member of a client class |
| # pylint: disable=W0212 |
| my_config._UnlockedAddNodeToGroup(node, my_grp_uuid) |
| node.group = my_grp_uuid |
| # Remove from list of groups to add |
| other_grps.remove(other_grp) |
| |
| for grp in other_grps: |
| #TODO: handle node group conflicts |
| my_config.AddNodeGroup(grp, _CLUSTERMERGE_ECID) |
| |
| # R0201: Method could be a function |
| def _StartMasterDaemon(self, no_vote=False): # pylint: disable=R0201 |
| """Starts the local master daemon. |
| |
| @param no_vote: Should the masterd started without voting? default: False |
| @raise errors.CommandError: If unable to start daemon. |
| |
| """ |
| env = {} |
| if no_vote: |
| env["EXTRA_MASTERD_ARGS"] = "--no-voting --yes-do-it" |
| |
| result = utils.RunCmd([pathutils.DAEMON_UTIL, "start-master"], env=env) |
| if result.failed: |
| raise errors.CommandError("Couldn't start ganeti master." |
| " Fail reason: %s; output: %s" % |
| (result.fail_reason, result.output)) |
| |
| def _ReaddMergedNodesAndRedist(self): |
| """Readds all merging nodes and make sure their config is up-to-date. |
| |
| @raise errors.CommandError: If anything fails. |
| |
| """ |
| for data in self.merger_data: |
| for node in data.nodes: |
| logging.info("Readding node %s", node) |
| result = utils.RunCmd(["gnt-node", "add", "--readd", |
| "--no-ssh-key-check", node]) |
| if result.failed: |
| logging.error("%s failed to be readded. Reason: %s, output: %s", |
| node, result.fail_reason, result.output) |
| |
| result = utils.RunCmd(["gnt-cluster", "redist-conf"]) |
| if result.failed: |
| raise errors.CommandError("Redistribution failed. Fail reason: %s;" |
| " output: %s" % (result.fail_reason, |
| result.output)) |
| |
| # R0201: Method could be a function |
| def _StartupAllInstances(self): # pylint: disable=R0201 |
| """Starts up all instances (locally). |
| |
| @raise errors.CommandError: If unable to start clusters |
| |
| """ |
| result = utils.RunCmd(["gnt-instance", "startup", "--all", |
| "--force-multiple"]) |
| if result.failed: |
| raise errors.CommandError("Unable to start all instances." |
| " Fail reason: %s; output: %s" % |
| (result.fail_reason, result.output)) |
| |
| # R0201: Method could be a function |
| # TODO: make this overridable, for some verify errors |
| def _VerifyCluster(self): # pylint: disable=R0201 |
| """Runs gnt-cluster verify to verify the health. |
| |
| @raise errors.ProgrammError: If cluster fails on verification |
| |
| """ |
| result = utils.RunCmd(["gnt-cluster", "verify"]) |
| if result.failed: |
| raise errors.CommandError("Verification of cluster failed." |
| " Fail reason: %s; output: %s" % |
| (result.fail_reason, result.output)) |
| |
| def Merge(self): |
| """Does the actual merge. |
| |
| It runs all the steps in the right order and updates the user about steps |
| taken. Also it keeps track of rollback_steps to undo everything. |
| |
| """ |
| rbsteps = [] |
| try: |
| logging.info("Pre cluster verification") |
| self._VerifyCluster() |
| |
| logging.info("Prepare authorized_keys") |
| rbsteps.append("Remove our key from authorized_keys on nodes:" |
| " %(nodes)s") |
| self._PrepareAuthorizedKeys() |
| |
| rbsteps.append("Start all instances again on the merging" |
| " clusters: %(clusters)s") |
| if self.stop_instances: |
| logging.info("Stopping merging instances (takes a while)") |
| self._StopMergingInstances() |
| logging.info("Checking that no instances are running on the mergees") |
| instances_running = self._CheckRunningInstances() |
| if instances_running: |
| raise errors.CommandError("Some instances are still running on the" |
| " mergees") |
| logging.info("Disable watcher") |
| self._DisableWatcher() |
| logging.info("Merging config") |
| self._FetchRemoteConfig() |
| logging.info("Removing master IPs on mergee master nodes") |
| self._RemoveMasterIps() |
| logging.info("Stop daemons on merging nodes") |
| self._StopDaemons() |
| |
| logging.info("Stopping master daemon") |
| self._KillMasterDaemon() |
| |
| rbsteps.append("Restore %s from another master candidate" |
| " and restart master daemon" % |
| pathutils.CLUSTER_CONF_FILE) |
| self._MergeConfig() |
| self._StartMasterDaemon(no_vote=True) |
| |
| # Point of no return, delete rbsteps |
| del rbsteps[:] |
| |
| logging.warning("We are at the point of no return. Merge can not easily" |
| " be undone after this point.") |
| logging.info("Readd nodes") |
| self._ReaddMergedNodesAndRedist() |
| |
| logging.info("Merge done, restart master daemon normally") |
| self._KillMasterDaemon() |
| self._StartMasterDaemon() |
| |
| if self.restart == _RESTART_ALL: |
| logging.info("Starting instances again") |
| self._StartupAllInstances() |
| else: |
| logging.info("Not starting instances again") |
| logging.info("Post cluster verification") |
| self._VerifyCluster() |
| except errors.GenericError, e: |
| logging.exception(e) |
| |
| if rbsteps: |
| nodes = Flatten([data.nodes for data in self.merger_data]) |
| info = { |
| "clusters": self.clusters, |
| "nodes": nodes, |
| } |
| logging.critical("In order to rollback do the following:") |
| for step in rbsteps: |
| logging.critical(" * %s", step % info) |
| else: |
| logging.critical("Nothing to rollback.") |
| |
| # TODO: Keep track of steps done for a flawless resume? |
| |
| def Cleanup(self): |
| """Clean up our environment. |
| |
| This cleans up remote private keys and configs and after that |
| deletes the temporary directory. |
| |
| """ |
| shutil.rmtree(self.work_dir) |
| |
| |
| def main(): |
| """Main routine. |
| |
| """ |
| program = os.path.basename(sys.argv[0]) |
| |
| parser = optparse.OptionParser(usage="%%prog [options...] <cluster...>", |
| prog=program) |
| parser.add_option(cli.DEBUG_OPT) |
| parser.add_option(cli.VERBOSE_OPT) |
| parser.add_option(PAUSE_PERIOD_OPT) |
| parser.add_option(GROUPS_OPT) |
| parser.add_option(RESTART_OPT) |
| parser.add_option(PARAMS_OPT) |
| parser.add_option(SKIP_STOP_INSTANCES_OPT) |
| |
| (options, args) = parser.parse_args() |
| |
| utils.SetupToolLogging(options.debug, options.verbose) |
| |
| if not args: |
| parser.error("No clusters specified") |
| |
| cluster_merger = Merger(utils.UniqueSequence(args), options.pause_period, |
| options.groups, options.restart, options.params, |
| options.stop_instances) |
| try: |
| try: |
| cluster_merger.Setup() |
| cluster_merger.Merge() |
| except errors.GenericError, e: |
| logging.exception(e) |
| return constants.EXIT_FAILURE |
| finally: |
| cluster_merger.Cleanup() |
| |
| return constants.EXIT_SUCCESS |
| |
| |
| if __name__ == "__main__": |
| sys.exit(main()) |
