| # |
| # |
| |
| # Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc. |
| # All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # 1. Redistributions of source code must retain the above copyright notice, |
| # this list of conditions and the following disclaimer. |
| # |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS |
| # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
| # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR |
| # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| |
| """Cluster related QA tests. |
| |
| """ |
| |
| import os |
| import re |
| import tempfile |
| import time |
| |
| from ganeti import _constants |
| from ganeti import constants |
| from ganeti import compat |
| from ganeti import utils |
| from ganeti import pathutils |
| |
| import qa_config |
| import qa_daemon |
| import qa_error |
| import qa_instance |
| import qa_job_utils |
| import qa_logging |
| import qa_rapi |
| import qa_utils |
| |
| from qa_utils import AssertEqual, AssertCommand, AssertRedirectedCommand, \ |
| GetCommandOutput, CheckFileUnmodified |
| |
| |
| # Prefix for LVM volumes created by QA code during tests |
| _QA_LV_PREFIX = "qa-" |
| |
| #: cluster verify command |
| _CLUSTER_VERIFY = ["gnt-cluster", "verify"] |
| |
| |
| def _RemoveFileFromAllNodes(filename): |
| """Removes a file from all nodes. |
| |
| """ |
| for node in qa_config.get("nodes"): |
| AssertCommand(["rm", "-f", filename], node=node) |
| |
| |
| def _CheckFileOnAllNodes(filename, content): |
| """Verifies the content of the given file on all nodes. |
| |
| """ |
| cmd = utils.ShellQuoteArgs(["cat", filename]) |
| for node in qa_config.get("nodes"): |
| AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content) |
| |
| |
| def _GetClusterField(field_path): |
| """Get the value of a cluster field. |
| |
| @type field_path: list of strings |
| @param field_path: Names of the groups/fields to navigate to get the desired |
| value, e.g. C{["Default node parameters", "oob_program"]} |
| @return: The effective value of the field (the actual type depends on the |
| chosen field) |
| |
| """ |
| assert isinstance(field_path, list) |
| assert field_path |
| ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"]) |
| for key in field_path: |
| ret = ret[key] |
| return ret |
| |
| |
| # Cluster-verify errors (date, "ERROR", then error code) |
| _CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):") |
| |
| |
| def _GetCVErrorCodes(cvout): |
| errs = set() |
| warns = set() |
| for l in cvout.splitlines(): |
| m = _CVERROR_RE.match(l) |
| if m: |
| etype = m.group(1) |
| ecode = m.group(2) |
| if etype == "ERROR": |
| errs.add(ecode) |
| elif etype == "WARNING": |
| warns.add(ecode) |
| return (errs, warns) |
| |
| |
| def _CheckVerifyErrors(actual, expected, etype): |
| exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected) |
| if not actual.issuperset(exp_codes): |
| missing = exp_codes.difference(actual) |
| raise qa_error.Error("Cluster-verify didn't return these expected" |
| " %ss: %s" % (etype, utils.CommaJoin(missing))) |
| |
| |
| def _CheckVerifyNoWarnings(actual, expected): |
| exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected) |
| excess = actual.intersection(exp_codes) |
| if excess: |
| raise qa_error.Error("Cluster-verify returned these warnings:" |
| " %s" % (utils.CommaJoin(excess))) |
| |
| |
| def AssertClusterVerify(fail=False, errors=None, |
| warnings=None, no_warnings=None): |
| """Run cluster-verify and check the result, ignoring warnings by default. |
| |
| @type fail: bool |
| @param fail: if cluster-verify is expected to fail instead of succeeding. |
| @type errors: list of tuples |
| @param errors: List of CV_XXX errors that are expected; if specified, all the |
| errors listed must appear in cluster-verify output. A non-empty value |
| implies C{fail=True}. |
| @type warnings: list of tuples |
| @param warnings: List of CV_XXX warnings that are expected to be raised; if |
| specified, all the errors listed must appear in cluster-verify output. |
| @type no_warnings: list of tuples |
| @param no_warnings: List of CV_XXX warnings that we expect NOT to be raised. |
| """ |
| cvcmd = "gnt-cluster verify" |
| mnode = qa_config.GetMasterNode() |
| if errors or warnings or no_warnings: |
| with CheckFileUnmodified(mnode.primary, pathutils.CLUSTER_CONF_FILE): |
| cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes", |
| fail=(fail or errors)) |
| print cvout |
| (act_errs, act_warns) = _GetCVErrorCodes(cvout) |
| if errors: |
| _CheckVerifyErrors(act_errs, errors, "error") |
| if warnings: |
| _CheckVerifyErrors(act_warns, warnings, "warning") |
| if no_warnings: |
| _CheckVerifyNoWarnings(act_warns, no_warnings) |
| |
| else: |
| with CheckFileUnmodified(mnode.primary, pathutils.CLUSTER_CONF_FILE): |
| AssertCommand(cvcmd, fail=fail, node=mnode) |
| |
| |
| # data for testing failures due to bad keys/values for disk parameters |
| _FAIL_PARAMS = ["nonexistent:resync-rate=1", |
| "drbd:nonexistent=1", |
| "drbd:resync-rate=invalid", |
| ] |
| |
| |
| def TestClusterInitDisk(): |
| """gnt-cluster init -D""" |
| name = qa_config.get("name") |
| for param in _FAIL_PARAMS: |
| AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True) |
| |
| |
| def TestClusterInit(): |
| """gnt-cluster init""" |
| # If we don't modify the SSH setup by Ganeti, we have to ensure connectivity |
| # before |
| master = qa_config.GetMasterNode() |
| if not qa_config.GetModifySshSetup(): |
| (key_type, _, priv_key_file, pub_key_file, auth_key_path) = \ |
| qa_config.GetSshConfig() |
| AssertCommand("echo -e 'y\n' | ssh-keygen -t %s -f %s -q -N ''" |
| % (key_type, priv_key_file)) |
| AssertCommand("cat %s >> %s" % (pub_key_file, auth_key_path)) |
| for node in qa_config.get("nodes"): |
| if node != master: |
| for key_file in [priv_key_file, pub_key_file]: |
| AssertCommand("scp -oStrictHostKeyChecking=no %s %s:%s" % |
| (key_file, node.primary, key_file)) |
| AssertCommand("ssh %s \'cat %s >> %s\'" |
| % (node.primary, pub_key_file, auth_key_path)) |
| |
| # Initialize cluster |
| enabled_disk_templates = qa_config.GetEnabledDiskTemplates() |
| cmd = [ |
| "gnt-cluster", "init", |
| "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4), |
| "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()), |
| "--enabled-disk-templates=%s" % |
| ",".join(enabled_disk_templates), |
| ] |
| |
| if not qa_config.GetModifySshSetup(): |
| cmd.append("--no-ssh-init") |
| |
| if constants.DT_FILE in enabled_disk_templates: |
| cmd.append( |
| "--file-storage-dir=%s" % |
| qa_config.get("default-file-storage-dir", |
| pathutils.DEFAULT_FILE_STORAGE_DIR)) |
| if constants.DT_SHARED_FILE in enabled_disk_templates: |
| cmd.append( |
| "--shared-file-storage-dir=%s" % |
| qa_config.get("default-shared-file-storage-dir", |
| pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR)) |
| if constants.DT_GLUSTER in enabled_disk_templates: |
| cmd.append( |
| "--gluster-storage-dir=%s" % |
| qa_config.get("default-gluster-storage-dir", |
| pathutils.DEFAULT_GLUSTER_STORAGE_DIR)) |
| |
| for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count", |
| "nic-count"): |
| spec_values = [] |
| for spec_val in ("min", "max", "std"): |
| spec = qa_config.get("ispec_%s_%s" % |
| (spec_type.replace("-", "_"), spec_val), None) |
| if spec is not None: |
| spec_values.append("%s=%d" % (spec_val, spec)) |
| if spec_values: |
| cmd.append("--specs-%s=%s" % (spec_type, ",".join(spec_values))) |
| |
| master = qa_config.GetMasterNode() |
| if master.secondary: |
| cmd.append("--secondary-ip=%s" % master.secondary) |
| |
| if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()): |
| vgname = qa_config.get("vg-name", constants.DEFAULT_VG) |
| if vgname: |
| cmd.append("--vg-name=%s" % vgname) |
| else: |
| raise qa_error.Error("Please specify a volume group if you enable" |
| " lvm-based disk templates in the QA.") |
| |
| master_netdev = qa_config.get("master-netdev", None) |
| if master_netdev: |
| cmd.append("--master-netdev=%s" % master_netdev) |
| |
| nicparams = qa_config.get("default-nicparams", None) |
| if nicparams: |
| cmd.append("--nic-parameters=%s" % |
| ",".join(utils.FormatKeyValue(nicparams))) |
| |
| # Cluster value of the exclusive-storage node parameter |
| e_s = qa_config.get("exclusive-storage") |
| if e_s is not None: |
| cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s]) |
| else: |
| e_s = False |
| qa_config.SetExclusiveStorage(e_s) |
| |
| extra_args = qa_config.get("cluster-init-args") |
| |
| if extra_args: |
| # This option was removed in 2.10, but in order to not break QA of older |
| # branches we remove it from the extra_args if it is in there. |
| opt_drbd_storage = "--no-drbd-storage" |
| if opt_drbd_storage in extra_args: |
| extra_args.remove(opt_drbd_storage) |
| cmd.extend(extra_args) |
| |
| cmd.append(qa_config.get("name")) |
| |
| AssertCommand(cmd) |
| |
| cmd = ["gnt-cluster", "modify"] |
| |
| # hypervisor parameter modifications |
| hvp = qa_config.get("hypervisor-parameters", {}) |
| for k, v in hvp.items(): |
| cmd.extend(["-H", "%s:%s" % (k, v)]) |
| # backend parameter modifications |
| bep = qa_config.get("backend-parameters", "") |
| if bep: |
| cmd.extend(["-B", bep]) |
| |
| if len(cmd) > 2: |
| AssertCommand(cmd) |
| |
| # OS parameters |
| osp = qa_config.get("os-parameters", {}) |
| for k, v in osp.items(): |
| AssertCommand(["gnt-os", "modify", "-O", v, k]) |
| |
| # OS hypervisor parameters |
| os_hvp = qa_config.get("os-hvp", {}) |
| for os_name in os_hvp: |
| for hv, hvp in os_hvp[os_name].items(): |
| AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name]) |
| |
| |
| def TestClusterRename(): |
| """gnt-cluster rename""" |
| cmd = ["gnt-cluster", "rename", "-f"] |
| |
| original_name = qa_config.get("name") |
| rename_target = qa_config.get("rename", None) |
| if rename_target is None: |
| print qa_logging.FormatError('"rename" entry is missing') |
| return |
| |
| for data in [ |
| cmd + [rename_target], |
| _CLUSTER_VERIFY, |
| cmd + [original_name], |
| _CLUSTER_VERIFY, |
| ]: |
| AssertCommand(data) |
| |
| |
| def TestClusterOob(): |
| """out-of-band framework""" |
| oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID() |
| |
| AssertCommand(_CLUSTER_VERIFY) |
| AssertCommand(["gnt-cluster", "modify", "--node-parameters", |
| "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" % |
| utils.NewUUID()]) |
| |
| AssertCommand(_CLUSTER_VERIFY, fail=True) |
| |
| AssertCommand(["touch", oob_path_exists]) |
| AssertCommand(["chmod", "0400", oob_path_exists]) |
| AssertCommand(["gnt-cluster", "copyfile", oob_path_exists]) |
| |
| try: |
| AssertCommand(["gnt-cluster", "modify", "--node-parameters", |
| "oob_program=%s" % oob_path_exists]) |
| |
| AssertCommand(_CLUSTER_VERIFY, fail=True) |
| |
| AssertCommand(["chmod", "0500", oob_path_exists]) |
| AssertCommand(["gnt-cluster", "copyfile", oob_path_exists]) |
| |
| AssertCommand(_CLUSTER_VERIFY) |
| finally: |
| AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists]) |
| |
| AssertCommand(["gnt-cluster", "modify", "--node-parameters", |
| "oob_program="]) |
| |
| |
| def TestClusterEpo(): |
| """gnt-cluster epo""" |
| master = qa_config.GetMasterNode() |
| |
| # Assert that OOB is unavailable for all nodes |
| result_output = GetCommandOutput(master.primary, |
| "gnt-node list --verbose --no-headers -o" |
| " powered") |
| AssertEqual(compat.all(powered == "(unavail)" |
| for powered in result_output.splitlines()), True) |
| |
| # Conflicting |
| AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True) |
| # --all doesn't expect arguments |
| AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True) |
| |
| # Unless --all is given master is not allowed to be in the list |
| AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True) |
| |
| with qa_job_utils.PausedWatcher(): |
| # This shouldn't fail |
| AssertCommand(["gnt-cluster", "epo", "-f", "--all"]) |
| |
| # All instances should have been stopped now |
| result_output = GetCommandOutput(master.primary, |
| "gnt-instance list --no-headers -o status") |
| # ERROR_down because the instance is stopped but not recorded as such |
| AssertEqual(compat.all(status == "ERROR_down" |
| for status in result_output.splitlines()), True) |
| |
| # Now start everything again |
| AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"]) |
| |
| # All instances should have been started now |
| result_output = GetCommandOutput(master.primary, |
| "gnt-instance list --no-headers -o status") |
| AssertEqual(compat.all(status == "running" |
| for status in result_output.splitlines()), True) |
| |
| |
| def TestClusterVerify(): |
| """gnt-cluster verify""" |
| AssertCommand(_CLUSTER_VERIFY) |
| AssertCommand(["gnt-cluster", "verify-disks"]) |
| |
| |
| def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes): |
| """gnt-cluster verify-disks with broken DRBD""" |
| qa_daemon.TestPauseWatcher() |
| |
| try: |
| info = qa_instance.GetInstanceInfo(instance.name) |
| snode = inst_nodes[1] |
| for idx, minor in enumerate(info["drbd-minors"][snode.primary]): |
| if idx % 2 == 0: |
| break_drbd_cmd = \ |
| "(drbdsetup %d down >/dev/null 2>&1;" \ |
| " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \ |
| (minor, minor) |
| else: |
| break_drbd_cmd = \ |
| "(drbdsetup %d detach >/dev/null 2>&1;" \ |
| " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \ |
| (minor, minor) |
| AssertCommand(break_drbd_cmd, node=snode) |
| |
| verify_output = GetCommandOutput(qa_config.GetMasterNode().primary, |
| "gnt-cluster verify-disks") |
| activation_msg = "Activating disks for instance '%s'" % instance.name |
| if activation_msg not in verify_output: |
| raise qa_error.Error("gnt-cluster verify-disks did not activate broken" |
| " DRBD disks:\n%s" % verify_output) |
| |
| verify_output = GetCommandOutput(qa_config.GetMasterNode().primary, |
| "gnt-cluster verify-disks") |
| if activation_msg in verify_output: |
| raise qa_error.Error("gnt-cluster verify-disks wants to activate broken" |
| " DRBD disks on second attempt:\n%s" % verify_output) |
| |
| AssertCommand(_CLUSTER_VERIFY) |
| finally: |
| qa_daemon.TestResumeWatcher() |
| |
| |
| def TestJobqueue(): |
| """gnt-debug test-jobqueue""" |
| AssertCommand(["gnt-debug", "test-jobqueue"]) |
| |
| |
| def TestDelay(node): |
| """gnt-debug delay""" |
| AssertCommand(["gnt-debug", "delay", "1"]) |
| AssertCommand(["gnt-debug", "delay", "--no-master", "1"]) |
| AssertCommand(["gnt-debug", "delay", "--no-master", |
| "-n", node.primary, "1"]) |
| |
| |
| def TestClusterReservedLvs(): |
| """gnt-cluster reserved lvs""" |
| # if no lvm-based templates are supported, skip the test |
| if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): |
| return |
| vgname = qa_config.get("vg-name", constants.DEFAULT_VG) |
| lvname = _QA_LV_PREFIX + "test" |
| lvfullname = "/".join([vgname, lvname]) |
| |
| # Clean cluster |
| AssertClusterVerify() |
| |
| AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", ""]) |
| AssertCommand(["lvcreate", "-L1G", "-n", lvname, vgname]) |
| AssertClusterVerify(fail=False, |
| warnings=[constants.CV_ENODEORPHANLV]) |
| |
| AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", |
| "%s,.*/other-test" % lvfullname]) |
| AssertClusterVerify(no_warnings=[constants.CV_ENODEORPHANLV]) |
| |
| AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", |
| ".*/%s.*" % _QA_LV_PREFIX]) |
| AssertClusterVerify(no_warnings=[constants.CV_ENODEORPHANLV]) |
| |
| AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", ""]) |
| AssertClusterVerify(fail=False, |
| warnings=[constants.CV_ENODEORPHANLV]) |
| |
| AssertCommand(["lvremove", "-f", lvfullname]) |
| AssertClusterVerify() |
| |
| |
| def TestClusterModifyEmpty(): |
| """gnt-cluster modify""" |
| AssertCommand(["gnt-cluster", "modify"], fail=True) |
| |
| |
| def TestClusterModifyDisk(): |
| """gnt-cluster modify -D""" |
| for param in _FAIL_PARAMS: |
| AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True) |
| |
| |
| def _GetOtherEnabledDiskTemplate(undesired_disk_templates, |
| enabled_disk_templates): |
| """Returns one template that is not in the undesired set. |
| |
| @type undesired_disk_templates: list of string |
| @param undesired_disk_templates: a list of disk templates that we want to |
| exclude when drawing one disk template from the list of enabled |
| disk templates |
| @type enabled_disk_templates: list of string |
| @param enabled_disk_templates: list of enabled disk templates (in QA) |
| |
| """ |
| desired_templates = list(set(enabled_disk_templates) |
| - set(undesired_disk_templates)) |
| if desired_templates: |
| template = desired_templates[0] |
| else: |
| # If no desired disk template is available for QA, choose 'diskless' and |
| # hope for the best. |
| template = constants.ST_DISKLESS |
| |
| return template |
| |
| |
| def TestClusterModifyFileBasedStorageDir( |
| file_disk_template, dir_config_key, default_dir, option_name): |
| """Tests gnt-cluster modify wrt to file-based directory options. |
| |
| @type file_disk_template: string |
| @param file_disk_template: file-based disk template |
| @type dir_config_key: string |
| @param dir_config_key: key for the QA config to retrieve the default |
| directory value |
| @type default_dir: string |
| @param default_dir: default directory, if the QA config does not specify |
| it |
| @type option_name: string |
| @param option_name: name of the option of 'gnt-cluster modify' to |
| change the directory |
| |
| """ |
| enabled_disk_templates = qa_config.GetEnabledDiskTemplates() |
| assert file_disk_template in constants.DTS_FILEBASED |
| if not qa_config.IsTemplateSupported(file_disk_template): |
| return |
| |
| # Get some non-file-based disk template to disable file storage |
| other_disk_template = _GetOtherEnabledDiskTemplate( |
| utils.storage.GetDiskTemplatesOfStorageTypes(constants.ST_FILE, |
| constants.ST_SHARED_FILE), |
| enabled_disk_templates |
| ) |
| |
| file_storage_dir = qa_config.get(dir_config_key, default_dir) |
| invalid_file_storage_dir = "/boot/" |
| |
| for fail, cmd in [ |
| (False, ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % file_disk_template, |
| "--ipolicy-disk-templates=%s" % file_disk_template]), |
| (False, ["gnt-cluster", "modify", |
| "--%s=%s" % (option_name, file_storage_dir)]), |
| (False, ["gnt-cluster", "modify", |
| "--%s=%s" % (option_name, invalid_file_storage_dir)]), |
| # file storage dir is set to an inacceptable path, thus verify |
| # should fail |
| (True, ["gnt-cluster", "verify"]), |
| # unsetting the storage dir while file storage is enabled |
| # should fail |
| (True, ["gnt-cluster", "modify", |
| "--%s=" % option_name]), |
| (False, ["gnt-cluster", "modify", |
| "--%s=%s" % (option_name, file_storage_dir)]), |
| (False, ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % other_disk_template, |
| "--ipolicy-disk-templates=%s" % other_disk_template]), |
| (False, ["gnt-cluster", "modify", |
| "--%s=%s" % (option_name, invalid_file_storage_dir)]), |
| # file storage is set to an inacceptable path, but file storage |
| # is disabled, thus verify should not fail |
| (False, ["gnt-cluster", "verify"]), |
| # unsetting the file storage dir while file storage is not enabled |
| # should be fine |
| (False, ["gnt-cluster", "modify", |
| "--%s=" % option_name]), |
| # resetting everything to sane values |
| (False, ["gnt-cluster", "modify", |
| "--%s=%s" % (option_name, file_storage_dir), |
| "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates), |
| "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)]) |
| ]: |
| AssertCommand(cmd, fail=fail) |
| |
| |
| def TestClusterModifyFileStorageDir(): |
| """gnt-cluster modify --file-storage-dir=...""" |
| TestClusterModifyFileBasedStorageDir( |
| constants.DT_FILE, "default-file-storage-dir", |
| pathutils.DEFAULT_FILE_STORAGE_DIR, |
| "file-storage-dir") |
| |
| |
| def TestClusterModifySharedFileStorageDir(): |
| """gnt-cluster modify --shared-file-storage-dir=...""" |
| TestClusterModifyFileBasedStorageDir( |
| constants.DT_SHARED_FILE, "default-shared-file-storage-dir", |
| pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR, |
| "shared-file-storage-dir") |
| |
| |
| def TestClusterModifyDiskTemplates(): |
| """gnt-cluster modify --enabled-disk-templates=...""" |
| enabled_disk_templates = qa_config.GetEnabledDiskTemplates() |
| default_disk_template = qa_config.GetDefaultDiskTemplate() |
| |
| _TestClusterModifyDiskTemplatesArguments(default_disk_template) |
| _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates) |
| _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates) |
| |
| _RestoreEnabledDiskTemplates() |
| nodes = qa_config.AcquireManyNodes(2) |
| |
| instance_template = enabled_disk_templates[0] |
| instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template) |
| |
| _TestClusterModifyUsedDiskTemplate(instance_template, |
| enabled_disk_templates) |
| |
| qa_instance.TestInstanceRemove(instance) |
| _RestoreEnabledDiskTemplates() |
| |
| |
| def TestClusterModifyInstallImage(): |
| """gnt-cluster modify --install-image=...'""" |
| master = qa_config.GetMasterNode() |
| |
| image = \ |
| GetCommandOutput(master.primary, |
| "mktemp --tmpdir ganeti-install-image.XXXXXX").strip() |
| AssertCommand(["gnt-cluster", "modify", "--install-image=%s" % image]) |
| AssertCommand(["rm", image]) |
| |
| |
| def _RestoreEnabledDiskTemplates(): |
| """Sets the list of enabled disk templates back to the list of enabled disk |
| templates from the QA configuration. This can be used to make sure that |
| the tests that modify the list of disk templates do not interfere with |
| other tests. |
| |
| """ |
| enabled_disk_templates = qa_config.GetEnabledDiskTemplates() |
| cmd = ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates), |
| "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates), |
| ] |
| |
| if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()): |
| vgname = qa_config.get("vg-name", constants.DEFAULT_VG) |
| cmd.append("--vg-name=%s" % vgname) |
| |
| AssertCommand(cmd, fail=False) |
| |
| |
| def _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates): |
| """Tests argument handling of 'gnt-cluster modify' with respect to |
| the parameter '--drbd-usermode-helper'. This test is independent |
| of instances. |
| |
| """ |
| _RestoreEnabledDiskTemplates() |
| |
| if constants.DT_DRBD8 not in enabled_disk_templates: |
| return |
| if constants.DT_PLAIN not in enabled_disk_templates: |
| return |
| |
| drbd_usermode_helper = qa_config.get("drbd-usermode-helper", "/bin/true") |
| bogus_usermode_helper = "/tmp/pinkbunny" |
| for command, fail in [ |
| (["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % constants.DT_DRBD8, |
| "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False), |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper=%s" % drbd_usermode_helper], False), |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper=%s" % bogus_usermode_helper], True), |
| # unsetting helper when DRBD is enabled should not work |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper="], True), |
| (["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % constants.DT_PLAIN, |
| "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False), |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper="], False), |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper=%s" % drbd_usermode_helper], False), |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper=%s" % drbd_usermode_helper, |
| "--enabled-disk-templates=%s" % constants.DT_DRBD8, |
| "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False), |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper=", |
| "--enabled-disk-templates=%s" % constants.DT_PLAIN, |
| "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False), |
| (["gnt-cluster", "modify", |
| "--drbd-usermode-helper=%s" % drbd_usermode_helper, |
| "--enabled-disk-templates=%s" % constants.DT_DRBD8, |
| "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False), |
| ]: |
| AssertCommand(command, fail=fail) |
| _RestoreEnabledDiskTemplates() |
| |
| |
| def _TestClusterModifyDiskTemplatesArguments(default_disk_template): |
| """Tests argument handling of 'gnt-cluster modify' with respect to |
| the parameter '--enabled-disk-templates'. This test is independent |
| of instances. |
| |
| """ |
| _RestoreEnabledDiskTemplates() |
| |
| # bogus templates |
| AssertCommand(["gnt-cluster", "modify", |
| "--enabled-disk-templates=pinkbunny"], |
| fail=True) |
| |
| # duplicate entries do no harm |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s,%s" % |
| (default_disk_template, default_disk_template), |
| "--ipolicy-disk-templates=%s" % default_disk_template], |
| fail=False) |
| |
| |
| def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates): |
| """Tests argument handling of 'gnt-cluster modify' with respect to |
| the parameter '--enabled-disk-templates' and '--vg-name'. This test is |
| independent of instances. |
| |
| """ |
| if not utils.IsLvmEnabled(enabled_disk_templates): |
| # These tests only make sense if lvm is enabled for QA |
| return |
| |
| # determine an LVM and a non-LVM disk template for the tests |
| non_lvm_template = _GetOtherEnabledDiskTemplate(constants.DTS_LVM, |
| enabled_disk_templates) |
| lvm_template = list(set(enabled_disk_templates) & constants.DTS_LVM)[0] |
| |
| vgname = qa_config.get("vg-name", constants.DEFAULT_VG) |
| |
| # Clean start: unset volume group name, disable lvm storage |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % non_lvm_template, |
| "--ipolicy-disk-templates=%s" % non_lvm_template, |
| "--vg-name="], |
| fail=False) |
| |
| # Try to enable lvm, when no volume group is given |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % lvm_template, |
| "--ipolicy-disk-templates=%s" % lvm_template], |
| fail=True) |
| |
| # Set volume group, with lvm still disabled: just a warning |
| AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False) |
| |
| # Try unsetting vg name and enabling lvm at the same time |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % lvm_template, |
| "--ipolicy-disk-templates=%s" % lvm_template, |
| "--vg-name="], |
| fail=True) |
| |
| # Enable lvm with vg name present |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % lvm_template, |
| "--ipolicy-disk-templates=%s" % lvm_template], |
| fail=False) |
| |
| # Try unsetting vg name with lvm still enabled |
| AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True) |
| |
| # Disable lvm with vg name still set |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % non_lvm_template, |
| "--ipolicy-disk-templates=%s" % non_lvm_template, |
| ], |
| fail=False) |
| |
| # Try unsetting vg name with lvm disabled |
| AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False) |
| |
| # Set vg name and enable lvm at the same time |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % lvm_template, |
| "--ipolicy-disk-templates=%s" % lvm_template, |
| "--vg-name=%s" % vgname], |
| fail=False) |
| |
| # Unset vg name and disable lvm at the same time |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % non_lvm_template, |
| "--ipolicy-disk-templates=%s" % non_lvm_template, |
| "--vg-name="], |
| fail=False) |
| |
| _RestoreEnabledDiskTemplates() |
| |
| |
| def _TestClusterModifyUsedDiskTemplate(instance_template, |
| enabled_disk_templates): |
| """Tests that disk templates that are currently in use by instances cannot |
| be disabled on the cluster. |
| |
| """ |
| # If the list of enabled disk templates contains only one template |
| # we need to add some other templates, because the list of enabled disk |
| # templates can only be set to a non-empty list. |
| new_disk_templates = list(set(enabled_disk_templates) |
| - set([instance_template])) |
| if not new_disk_templates: |
| new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK]) |
| - set([instance_template])) |
| AssertCommand( |
| ["gnt-cluster", "modify", |
| "--enabled-disk-templates=%s" % ",".join(new_disk_templates), |
| "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)], |
| fail=True) |
| |
| |
| def TestClusterModifyBe(): |
| """gnt-cluster modify -B""" |
| for fail, cmd in [ |
| # max/min mem |
| (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]), |
| (False, ["gnt-cluster", "modify", "-B", "minmem=256"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]), |
| (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]), |
| (True, ["gnt-cluster", "modify", "-B", "minmem=a"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]), |
| (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]), |
| # vcpus |
| (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]), |
| (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]), |
| (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]), |
| # auto_balance |
| (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]), |
| (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]), |
| (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]), |
| (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]), |
| ]: |
| AssertCommand(cmd, fail=fail) |
| |
| # redo the original-requested BE parameters, if any |
| bep = qa_config.get("backend-parameters", "") |
| if bep: |
| AssertCommand(["gnt-cluster", "modify", "-B", bep]) |
| |
| |
| def _GetClusterIPolicy(): |
| """Return the run-time values of the cluster-level instance policy. |
| |
| @rtype: tuple |
| @return: (policy, specs), where: |
| - policy is a dictionary of the policy values, instance specs excluded |
| - specs is a dictionary containing only the specs, using the internal |
| format (see L{constants.IPOLICY_DEFAULTS} for an example) |
| |
| """ |
| info = qa_utils.GetObjectInfo(["gnt-cluster", "info"]) |
| policy = info["Instance policy - limits for instances"] |
| (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy) |
| |
| # Sanity checks |
| assert "minmax" in ret_specs and "std" in ret_specs |
| assert len(ret_specs["minmax"]) > 0 |
| assert len(ret_policy) > 0 |
| return (ret_policy, ret_specs) |
| |
| |
| def TestClusterModifyIPolicy(): |
| """gnt-cluster modify --ipolicy-*""" |
| basecmd = ["gnt-cluster", "modify"] |
| (old_policy, old_specs) = _GetClusterIPolicy() |
| for par in ["vcpu-ratio", "spindle-ratio"]: |
| curr_val = float(old_policy[par]) |
| test_values = [ |
| (True, 1.0), |
| (True, 1.5), |
| (True, 2), |
| (False, "a"), |
| # Restore the old value |
| (True, curr_val), |
| ] |
| for (good, val) in test_values: |
| cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)] |
| AssertCommand(cmd, fail=not good) |
| if good: |
| curr_val = val |
| # Check the affected parameter |
| (eff_policy, eff_specs) = _GetClusterIPolicy() |
| AssertEqual(float(eff_policy[par]), curr_val) |
| # Check everything else |
| AssertEqual(eff_specs, old_specs) |
| for p in eff_policy.keys(): |
| if p == par: |
| continue |
| AssertEqual(eff_policy[p], old_policy[p]) |
| |
| # Allowing disk templates via ipolicy requires them to be |
| # enabled on the cluster. |
| if not (qa_config.IsTemplateSupported(constants.DT_PLAIN) |
| and qa_config.IsTemplateSupported(constants.DT_DRBD8)): |
| return |
| # Disk templates are treated slightly differently |
| par = "disk-templates" |
| disp_str = "allowed disk templates" |
| curr_val = old_policy[disp_str] |
| test_values = [ |
| (True, constants.DT_PLAIN), |
| (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)), |
| (False, "thisisnotadisktemplate"), |
| (False, ""), |
| # Restore the old value |
| (True, curr_val.replace(" ", "")), |
| ] |
| for (good, val) in test_values: |
| cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)] |
| AssertCommand(cmd, fail=not good) |
| if good: |
| curr_val = val |
| # Check the affected parameter |
| (eff_policy, eff_specs) = _GetClusterIPolicy() |
| AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val) |
| # Check everything else |
| AssertEqual(eff_specs, old_specs) |
| for p in eff_policy.keys(): |
| if p == disp_str: |
| continue |
| AssertEqual(eff_policy[p], old_policy[p]) |
| |
| |
| def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False, |
| old_values=None): |
| """Change instance specs. |
| |
| At most one of new_specs or diff_specs can be specified. |
| |
| @type new_specs: dict |
| @param new_specs: new complete specs, in the same format returned by |
| L{_GetClusterIPolicy} |
| @type diff_specs: dict |
| @param diff_specs: partial specs, it can be an incomplete specifications, but |
| if min/max specs are specified, their number must match the number of the |
| existing specs |
| @type fail: bool |
| @param fail: if the change is expected to fail |
| @type old_values: tuple |
| @param old_values: (old_policy, old_specs), as returned by |
| L{_GetClusterIPolicy} |
| @return: same as L{_GetClusterIPolicy} |
| |
| """ |
| build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts |
| return qa_utils.TestSetISpecs( |
| new_specs=new_specs, diff_specs=diff_specs, |
| get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd, |
| fail=fail, old_values=old_values) |
| |
| |
| def TestClusterModifyISpecs(): |
| """gnt-cluster modify --specs-*""" |
| params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"] |
| (cur_policy, cur_specs) = _GetClusterIPolicy() |
| # This test assumes that there is only one min/max bound |
| assert len(cur_specs[constants.ISPECS_MINMAX]) == 1 |
| for par in params: |
| test_values = [ |
| (True, 0, 4, 12), |
| (True, 4, 4, 12), |
| (True, 4, 12, 12), |
| (True, 4, 4, 4), |
| (False, 4, 0, 12), |
| (False, 4, 16, 12), |
| (False, 4, 4, 0), |
| (False, 12, 4, 4), |
| (False, 12, 4, 0), |
| (False, "a", 4, 12), |
| (False, 0, "a", 12), |
| (False, 0, 4, "a"), |
| # This is to restore the old values |
| (True, |
| cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par], |
| cur_specs[constants.ISPECS_STD][par], |
| cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par]) |
| ] |
| for (good, mn, st, mx) in test_values: |
| new_vals = { |
| constants.ISPECS_MINMAX: [{ |
| constants.ISPECS_MIN: {par: mn}, |
| constants.ISPECS_MAX: {par: mx} |
| }], |
| constants.ISPECS_STD: {par: st} |
| } |
| cur_state = (cur_policy, cur_specs) |
| # We update cur_specs, as we've copied the values to restore already |
| (cur_policy, cur_specs) = TestClusterSetISpecs( |
| diff_specs=new_vals, fail=not good, old_values=cur_state) |
| |
| # Get the ipolicy command |
| mnode = qa_config.GetMasterNode() |
| initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd") |
| modcmd = ["gnt-cluster", "modify"] |
| opts = initcmd.split() |
| assert opts[0:2] == ["gnt-cluster", "init"] |
| for k in range(2, len(opts) - 1): |
| if opts[k].startswith("--ipolicy-"): |
| assert k + 2 <= len(opts) |
| modcmd.extend(opts[k:k + 2]) |
| # Re-apply the ipolicy (this should be a no-op) |
| AssertCommand(modcmd) |
| new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd") |
| AssertEqual(initcmd, new_initcmd) |
| |
| |
| def _TestClusterModifyUserShutdownXen(nodes): |
| """Tests user shutdown cluster wide for the KVM hypervisor. |
| |
| Note that for the Xen hypervisor, the KVM daemon should never run. |
| |
| """ |
| AssertCommand(["gnt-cluster", "modify", "--user-shutdown=true"]) |
| |
| # Give time for kvmd to start and stop on all nodes |
| time.sleep(5) |
| |
| for node in nodes: |
| AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) |
| |
| AssertCommand(["gnt-cluster", "modify", "--user-shutdown=false"]) |
| |
| for node in nodes: |
| AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) |
| |
| |
| def _TestClusterModifyUserShutdownKvm(nodes): |
| """Tests user shutdown cluster wide for the KVM hypervisor. |
| |
| Note that for the KVM hypervisor, the KVM daemon should run |
| according to '--user-shutdown' and whether the node is VM capable. |
| |
| """ |
| # How much time to wait for kvmd to start/stop |
| kvmd_cycle_time = 4 |
| |
| # Start kvmd on all nodes |
| AssertCommand(["gnt-cluster", "modify", "--user-shutdown=true"]) |
| time.sleep(kvmd_cycle_time) |
| for node in nodes: |
| AssertCommand("pgrep ganeti-kvmd", node=node) |
| |
| # Test VM capable node attribute |
| test_node = None |
| |
| for node in nodes: |
| node_info = qa_utils.GetObjectInfo(["gnt-node", "info", node.primary])[0] |
| if "vm_capable" in node_info and node_info["vm_capable"]: |
| test_node = node |
| break |
| |
| if test_node is None: |
| raise qa_error.Error("Failed to find viable node for this test") |
| |
| # Stop kvmd by disabling vm capable |
| AssertCommand(["gnt-node", "modify", "--vm-capable=no", test_node.primary]) |
| time.sleep(kvmd_cycle_time) |
| AssertCommand("pgrep ganeti-kvmd", node=test_node, fail=True) |
| |
| # Start kvmd by enabling vm capable |
| AssertCommand(["gnt-node", "modify", "--vm-capable=yes", test_node.primary]) |
| time.sleep(kvmd_cycle_time) |
| AssertCommand("pgrep ganeti-kvmd", node=test_node) |
| |
| # Stop kvmd on all nodes by removing KVM from the enabled hypervisors |
| enabled_hypervisors = qa_config.GetEnabledHypervisors() |
| |
| AssertCommand(["gnt-cluster", "modify", "--enabled-hypervisors=xen-pvm"]) |
| time.sleep(kvmd_cycle_time) |
| for node in nodes: |
| AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) |
| |
| # Start kvmd on all nodes by restoring KVM to the enabled hypervisors |
| AssertCommand(["gnt-cluster", "modify", |
| "--enabled-hypervisors=%s" % ",".join(enabled_hypervisors)]) |
| time.sleep(kvmd_cycle_time) |
| for node in nodes: |
| AssertCommand("pgrep ganeti-kvmd", node=node) |
| |
| # Stop kvmd on all nodes |
| AssertCommand(["gnt-cluster", "modify", "--user-shutdown=false"]) |
| time.sleep(kvmd_cycle_time) |
| for node in nodes: |
| AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) |
| |
| |
| def TestClusterModifyUserShutdown(): |
| """Tests user shutdown cluster wide. |
| |
| """ |
| enabled_hypervisors = qa_config.GetEnabledHypervisors() |
| nodes = qa_config.get("nodes") |
| |
| for (hv, fn) in [(constants.HT_XEN_PVM, _TestClusterModifyUserShutdownXen), |
| (constants.HT_XEN_HVM, _TestClusterModifyUserShutdownXen), |
| (constants.HT_KVM, _TestClusterModifyUserShutdownKvm)]: |
| if hv in enabled_hypervisors: |
| qa_daemon.TestPauseWatcher() |
| fn(nodes) |
| qa_daemon.TestResumeWatcher() |
| else: |
| print "%s hypervisor is not enabled, skipping test for this hypervisor" \ |
| % hv |
| |
| |
| def TestClusterInfo(): |
| """gnt-cluster info""" |
| AssertCommand(["gnt-cluster", "info"]) |
| |
| |
| def TestClusterRedistConf(): |
| """gnt-cluster redist-conf""" |
| AssertCommand(["gnt-cluster", "redist-conf"]) |
| |
| |
| def TestClusterGetmaster(): |
| """gnt-cluster getmaster""" |
| AssertCommand(["gnt-cluster", "getmaster"]) |
| |
| |
| def TestClusterVersion(): |
| """gnt-cluster version""" |
| AssertCommand(["gnt-cluster", "version"]) |
| |
| |
| def _AssertSsconfCertFiles(): |
| """This asserts that all ssconf_master_candidate_certs have the same content. |
| |
| """ |
| (vcluster_master, _) = qa_config.GetVclusterSettings() |
| if vcluster_master: |
| print "Skipping asserting SsconfCertFiles for Vcluster" |
| return |
| nodes = qa_config.get("nodes") |
| ssconf_file = "/var/lib/ganeti/ssconf_master_candidates_certs" |
| ssconf_content = {} |
| for node in nodes: |
| cmd = ["cat", ssconf_file] |
| print "Ssconf Master Certificates of node '%s'." % node.primary |
| result_output = GetCommandOutput(node.primary, utils.ShellQuoteArgs(cmd)) |
| ssconf_content[node] = result_output |
| |
| # Clean up result to make it comparable: |
| # remove trailing whitespace from each line, remove empty lines, sort lines |
| lines_node = result_output.split('\n') |
| lines_node = [line.strip() for line in lines_node if len(line.strip()) > 0] |
| lines_node.sort() |
| |
| ssconf_content[node] = lines_node |
| |
| first_node = nodes[0] |
| for node in nodes[1:]: |
| if not ssconf_content[node] == ssconf_content[first_node]: |
| raise Exception("Cert list of node '%s' differs from the list of node" |
| " '%s'." % (node, first_node)) |
| |
| |
| def _TestSSHKeyChanges(master_node): |
| """Tests a lot of SSH key type- and size- related functionality. |
| |
| @type master_node: L{qa_config._QaNode} |
| @param master_node: The cluster master. |
| |
| """ |
| # Helper fn to avoid specifying base params too many times |
| def _RenewWithParams(new_params, verify=True, fail=False): |
| AssertCommand(["gnt-cluster", "renew-crypto", "--new-ssh-keys", "-f", |
| "--no-ssh-key-check"] + new_params, fail=fail) |
| if not fail and verify: |
| AssertCommand(["gnt-cluster", "verify"]) |
| |
| # First test the simplest change |
| _RenewWithParams([]) |
| |
| # And stop here if vcluster |
| (vcluster_master, _) = qa_config.GetVclusterSettings() |
| if vcluster_master: |
| print "Skipping further SSH key replacement checks for vcluster" |
| return |
| |
| # And the actual tests |
| with qa_config.AcquireManyNodesCtx(1, exclude=[master_node]) as nodes: |
| node_name = nodes[0].primary |
| |
| # Another helper function for checking whether a specific key can log in |
| def _CheckLoginWithKey(key_path, fail=False): |
| AssertCommand(["ssh", "-oIdentityFile=%s" % key_path, "-oBatchMode=yes", |
| "-oStrictHostKeyChecking=no", "-oIdentitiesOnly=yes", |
| "-F/dev/null", node_name, "true"], |
| fail=fail, forward_agent=False) |
| |
| _RenewWithParams(["--ssh-key-type=dsa"]) |
| _CheckLoginWithKey("/root/.ssh/id_dsa") |
| # Stash the key for now |
| old_key_backup = qa_utils.BackupFile(master_node.primary, |
| "/root/.ssh/id_dsa") |
| |
| try: |
| _RenewWithParams(["--ssh-key-type=rsa"]) |
| _CheckLoginWithKey("/root/.ssh/id_rsa") |
| # And check that we cannot log in with the old key |
| _CheckLoginWithKey(old_key_backup, fail=True) |
| finally: |
| AssertCommand(["rm", "-f", old_key_backup]) |
| |
| _RenewWithParams(["--ssh-key-bits=4096"]) |
| _RenewWithParams(["--ssh-key-bits=521"], fail=True) |
| |
| # Restore the cluster to its pristine state, skipping the verify as we did |
| # way too many already |
| _RenewWithParams(["--ssh-key-type=rsa", "--ssh-key-bits=2048"], |
| verify=False) |
| |
| |
| def TestClusterRenewCrypto(): |
| """gnt-cluster renew-crypto""" |
| master = qa_config.GetMasterNode() |
| |
| # Conflicting options |
| cmd = ["gnt-cluster", "renew-crypto", "--force", |
| "--new-cluster-certificate", "--new-confd-hmac-key"] |
| conflicting = [ |
| ["--new-rapi-certificate", "--rapi-certificate=/dev/null"], |
| ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"], |
| ] |
| for i in conflicting: |
| AssertCommand(cmd + i, fail=True) |
| |
| # Invalid RAPI certificate |
| cmd = ["gnt-cluster", "renew-crypto", "--force", |
| "--rapi-certificate=/dev/null"] |
| AssertCommand(cmd, fail=True) |
| |
| rapi_cert_backup = qa_utils.BackupFile(master.primary, |
| pathutils.RAPI_CERT_FILE) |
| try: |
| # Custom RAPI certificate |
| fh = tempfile.NamedTemporaryFile() |
| |
| # Ensure certificate doesn't cause "gnt-cluster verify" to complain |
| validity = constants.SSL_CERT_EXPIRATION_WARN * 3 |
| |
| utils.GenerateSelfSignedSslCert(fh.name, 1, validity=validity) |
| |
| tmpcert = qa_utils.UploadFile(master.primary, fh.name) |
| try: |
| AssertCommand(["gnt-cluster", "renew-crypto", "--force", |
| "--rapi-certificate=%s" % tmpcert]) |
| finally: |
| AssertCommand(["rm", "-f", tmpcert]) |
| |
| # Custom cluster domain secret |
| cds_fh = tempfile.NamedTemporaryFile() |
| cds_fh.write(utils.GenerateSecret()) |
| cds_fh.write("\n") |
| cds_fh.flush() |
| |
| tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name) |
| try: |
| AssertCommand(["gnt-cluster", "renew-crypto", "--force", |
| "--cluster-domain-secret=%s" % tmpcds]) |
| finally: |
| AssertCommand(["rm", "-f", tmpcds]) |
| |
| # Normal case |
| AssertCommand(["gnt-cluster", "renew-crypto", "--force", |
| "--new-cluster-certificate", "--new-confd-hmac-key", |
| "--new-rapi-certificate", "--new-cluster-domain-secret", |
| "--new-node-certificates", "--new-ssh-keys", |
| "--no-ssh-key-check"]) |
| _AssertSsconfCertFiles() |
| AssertCommand(["gnt-cluster", "verify"]) |
| |
| # Only renew node certificates |
| AssertCommand(["gnt-cluster", "renew-crypto", "--force", |
| "--new-node-certificates"]) |
| _AssertSsconfCertFiles() |
| AssertCommand(["gnt-cluster", "verify"]) |
| |
| # Only renew cluster certificate |
| AssertCommand(["gnt-cluster", "renew-crypto", "--force", |
| "--new-cluster-certificate"]) |
| _AssertSsconfCertFiles() |
| AssertCommand(["gnt-cluster", "verify"]) |
| |
| # Comprehensively test various types of SSH key changes |
| _TestSSHKeyChanges(master) |
| |
| # Restore RAPI certificate |
| AssertCommand(["gnt-cluster", "renew-crypto", "--force", |
| "--rapi-certificate=%s" % rapi_cert_backup]) |
| _AssertSsconfCertFiles() |
| AssertCommand(["gnt-cluster", "verify"]) |
| finally: |
| AssertCommand(["rm", "-f", rapi_cert_backup]) |
| |
| # Since renew-crypto replaced the RAPI cert, reload it. |
| if qa_rapi.Enabled(): |
| qa_rapi.ReloadCertificates() |
| |
| |
| def TestClusterBurnin(): |
| """Burnin""" |
| master = qa_config.GetMasterNode() |
| |
| options = qa_config.get("options", {}) |
| disk_template = options.get("burnin-disk-template", constants.DT_DRBD8) |
| parallel = options.get("burnin-in-parallel", False) |
| check_inst = options.get("burnin-check-instances", False) |
| do_rename = options.get("burnin-rename", "") |
| do_reboot = options.get("burnin-reboot", True) |
| reboot_types = options.get("reboot-types", constants.REBOOT_TYPES) |
| |
| # Get as many instances as we need |
| instances = [] |
| try: |
| try: |
| num = qa_config.get("options", {}).get("burnin-instances", 1) |
| for _ in range(0, num): |
| instances.append(qa_config.AcquireInstance()) |
| except qa_error.OutOfInstancesError: |
| print "Not enough instances, continuing anyway." |
| |
| if len(instances) < 1: |
| raise qa_error.Error("Burnin needs at least one instance") |
| |
| burnin_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), |
| "../tools/burnin") |
| script = qa_utils.UploadFile(master.primary, burnin_file) |
| try: |
| disks = qa_config.GetDiskOptions() |
| # Run burnin |
| cmd = ["env", |
| "PYTHONPATH=%s" % _constants.VERSIONEDSHAREDIR, |
| script, |
| "--os=%s" % qa_config.get("os"), |
| "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM), |
| "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM), |
| "--disk-size=%s" % ",".join([d.get("size") for d in disks]), |
| "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]), |
| "--disk-template=%s" % disk_template] |
| if parallel: |
| cmd.append("--parallel") |
| cmd.append("--early-release") |
| if check_inst: |
| cmd.append("--http-check") |
| if do_rename: |
| cmd.append("--rename=%s" % do_rename) |
| if not do_reboot: |
| cmd.append("--no-reboot") |
| else: |
| cmd.append("--reboot-types=%s" % ",".join(reboot_types)) |
| cmd += [inst.name for inst in instances] |
| AssertCommand(cmd) |
| finally: |
| AssertCommand(["rm", "-f", script]) |
| |
| finally: |
| for inst in instances: |
| inst.Release() |
| |
| |
| def TestClusterMasterFailover(): |
| """gnt-cluster master-failover""" |
| master = qa_config.GetMasterNode() |
| failovermaster = qa_config.AcquireNode(exclude=master) |
| |
| # Flush the configuration to prevent race conditions when loading it |
| # on another node |
| print qa_logging.FormatInfo("Flushing the configuration on the master node") |
| AssertCommand(["gnt-debug", "wconfd", "flushconfig"]) |
| |
| cmd = ["gnt-cluster", "master-failover"] |
| node_list_cmd = ["gnt-node", "list"] |
| try: |
| AssertCommand(cmd, node=failovermaster) |
| AssertCommand(node_list_cmd, node=failovermaster) |
| # Back to original master node |
| AssertCommand(cmd, node=master) |
| AssertCommand(node_list_cmd, node=master) |
| finally: |
| failovermaster.Release() |
| |
| |
| def TestUpgrade(): |
| """Test gnt-cluster upgrade. |
| |
| This tests the 'gnt-cluster upgrade' command by flipping |
| between the current and a different version of Ganeti. |
| To also recover subtle points in the configuration up/down |
| grades, instances are left over both upgrades. |
| |
| """ |
| this_version = qa_config.get("dir-version") |
| other_version = qa_config.get("other-dir-version") |
| if this_version is None or other_version is None: |
| print qa_utils.FormatInfo("Test not run, as versions not specified") |
| return |
| |
| inst_creates = [] |
| upgrade_instances = qa_config.get("upgrade-instances", []) |
| live_instances = [] |
| for (test_name, templ, cf, n) in qa_instance.available_instance_tests: |
| if (qa_config.TestEnabled(test_name) and |
| qa_config.IsTemplateSupported(templ) and |
| templ in upgrade_instances): |
| inst_creates.append((cf, n)) |
| |
| for (cf, n) in inst_creates: |
| nodes = qa_config.AcquireManyNodes(n) |
| live_instances.append(cf(nodes)) |
| |
| AssertRedirectedCommand(["gnt-cluster", "upgrade", "--to", other_version]) |
| AssertRedirectedCommand(["gnt-cluster", "verify"]) |
| |
| for instance in live_instances: |
| qa_instance.TestInstanceRemove(instance) |
| instance.Release() |
| live_instances = [] |
| for (cf, n) in inst_creates: |
| nodes = qa_config.AcquireManyNodes(n) |
| live_instances.append(cf(nodes)) |
| |
| AssertRedirectedCommand(["gnt-cluster", "upgrade", "--to", this_version]) |
| AssertRedirectedCommand(["gnt-cluster", "verify"]) |
| |
| for instance in live_instances: |
| qa_instance.TestInstanceRemove(instance) |
| instance.Release() |
| |
| |
| def _NodeQueueDrainFile(node): |
| """Returns path to queue drain file for a node. |
| |
| """ |
| return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE) |
| |
| |
| def _AssertDrainFile(node, **kwargs): |
| """Checks for the queue drain file. |
| |
| """ |
| AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs) |
| |
| |
| def TestClusterMasterFailoverWithDrainedQueue(): |
| """gnt-cluster master-failover with drained queue""" |
| master = qa_config.GetMasterNode() |
| failovermaster = qa_config.AcquireNode(exclude=master) |
| |
| # Ensure queue is not drained |
| for node in [master, failovermaster]: |
| _AssertDrainFile(node, fail=True) |
| |
| # Drain queue on failover master |
| AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)], |
| node=failovermaster) |
| |
| cmd = ["gnt-cluster", "master-failover"] |
| try: |
| _AssertDrainFile(failovermaster) |
| AssertCommand(cmd, node=failovermaster) |
| _AssertDrainFile(master, fail=True) |
| _AssertDrainFile(failovermaster, fail=True) |
| |
| # Back to original master node |
| AssertCommand(cmd, node=master) |
| finally: |
| failovermaster.Release() |
| |
| # Ensure queue is not drained |
| for node in [master, failovermaster]: |
| _AssertDrainFile(node, fail=True) |
| |
| |
| def TestClusterCopyfile(): |
| """gnt-cluster copyfile""" |
| master = qa_config.GetMasterNode() |
| |
| uniqueid = utils.NewUUID() |
| |
| # Create temporary file |
| f = tempfile.NamedTemporaryFile() |
| f.write(uniqueid) |
| f.flush() |
| f.seek(0) |
| |
| # Upload file to master node |
| testname = qa_utils.UploadFile(master.primary, f.name) |
| try: |
| # Copy file to all nodes |
| AssertCommand(["gnt-cluster", "copyfile", testname]) |
| _CheckFileOnAllNodes(testname, uniqueid) |
| finally: |
| _RemoveFileFromAllNodes(testname) |
| |
| |
| def TestClusterCommand(): |
| """gnt-cluster command""" |
| uniqueid = utils.NewUUID() |
| rfile = "/tmp/gnt%s" % utils.NewUUID() |
| rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid]) |
| cmd = utils.ShellQuoteArgs(["gnt-cluster", "command", |
| "%s >%s" % (rcmd, rfile)]) |
| |
| try: |
| AssertCommand(cmd) |
| _CheckFileOnAllNodes(rfile, uniqueid) |
| finally: |
| _RemoveFileFromAllNodes(rfile) |
| |
| |
| def TestClusterDestroy(): |
| """gnt-cluster destroy""" |
| AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"]) |
| |
| |
| def TestClusterRepairDiskSizes(): |
| """gnt-cluster repair-disk-sizes""" |
| AssertCommand(["gnt-cluster", "repair-disk-sizes"]) |
| |
| |
| def TestSetExclStorCluster(newvalue): |
| """Set the exclusive_storage node parameter at the cluster level. |
| |
| @type newvalue: bool |
| @param newvalue: New value of exclusive_storage |
| @rtype: bool |
| @return: The old value of exclusive_storage |
| |
| """ |
| es_path = ["Default node parameters", "exclusive_storage"] |
| oldvalue = _GetClusterField(es_path) |
| AssertCommand(["gnt-cluster", "modify", "--node-parameters", |
| "exclusive_storage=%s" % newvalue]) |
| effvalue = _GetClusterField(es_path) |
| if effvalue != newvalue: |
| raise qa_error.Error("exclusive_storage has the wrong value: %s instead" |
| " of %s" % (effvalue, newvalue)) |
| qa_config.SetExclusiveStorage(newvalue) |
| return oldvalue |
| |
| |
| def TestExclStorSharedPv(node): |
| """cluster-verify reports LVs that share the same PV with exclusive_storage. |
| |
| """ |
| vgname = qa_config.get("vg-name", constants.DEFAULT_VG) |
| lvname1 = _QA_LV_PREFIX + "vol1" |
| lvname2 = _QA_LV_PREFIX + "vol2" |
| node_name = node.primary |
| AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name) |
| AssertClusterVerify(fail=False, |
| warnings=[constants.CV_ENODEORPHANLV]) |
| AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name) |
| AssertClusterVerify(fail=True, |
| errors=[constants.CV_ENODELVM], |
| warnings=[constants.CV_ENODEORPHANLV]) |
| AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name) |
| AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name) |
| AssertClusterVerify() |
| |
| |
| def TestInstanceCommunication(): |
| """Tests instance communication via 'gnt-cluster modify'""" |
| master = qa_config.GetMasterNode() |
| |
| # Check that the 'default' node group exists |
| cmd = ["gnt-group", "list", "--no-headers", "-o", "name"] |
| result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), "default", msg="Checking 'default' group") |
| |
| # Check that no networks exist |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "name"] |
| result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), "", msg="Checking networks") |
| |
| # Modify cluster parameter 'instance-communication-network' and |
| # check whether the cluster creates the instance communication |
| # network and connects it to the 'default' node group |
| network_name = "mynetwork" |
| |
| cmd = "gnt-cluster modify --instance-communication-network=%s" % network_name |
| result_output = qa_utils.GetCommandOutput(master.primary, cmd) |
| print result_output |
| |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "name", network_name] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), "mynetwork", msg="Checking 'mynetwork'") |
| |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "group_list", |
| network_name] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| # <nodegroup> (<mode>, <link>, <vlan>), for this test, VLAN is nothing |
| AssertEqual(result_output.strip(), "default (routed, communication_rt, )", |
| msg="Checking network connected groups") |
| |
| # Check that the network has the parameters necessary for instance |
| # communication |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "gateway", |
| network_name] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), "-", msg="Checking gateway") |
| |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "gateway6", |
| network_name] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), "-", msg="Checking gateway6") |
| |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "network", |
| network_name] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), constants.INSTANCE_COMMUNICATION_NETWORK4, |
| msg="Checking network") |
| |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "network6", |
| network_name] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), constants.INSTANCE_COMMUNICATION_NETWORK6, |
| msg="Checking network6") |
| |
| # Add a new group and check whether the instance communication |
| # network connects to this new group |
| # |
| # We don't assume any particular group order and allow the output of |
| # 'gnt-network list' to print the 'default' and 'mygroup' groups in |
| # any order. |
| group = "mygroup" |
| |
| cmd = ["gnt-group", "add", group] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| print result_output |
| |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "group_list", |
| network_name] |
| result_output = qa_utils.GetCommandOutput(master.primary, |
| utils.ShellQuoteArgs(cmd)) |
| |
| try: |
| r1 = "mygroup (routed, communication_rt, )," \ |
| " default (routed, communication_rt, )" |
| AssertEqual(result_output.strip(), r1, |
| msg="Checking network connected groups") |
| except qa_error.Error: |
| r2 = "default (routed, communication_rt, )," \ |
| " mygroup (routed, communication_rt, )" |
| AssertEqual(result_output.strip(), r2, |
| msg="Checking network connected groups") |
| |
| # Modify cluster parameter 'instance-communication-network' to the |
| # same value and check that nothing happens. |
| cmd = "gnt-cluster modify --instance-communication-network=%s" % network_name |
| result_output = qa_utils.GetCommandOutput(master.primary, cmd) |
| print result_output |
| |
| # Disable instance communication network, disconnect the instance |
| # communication network and remove it, and remove the group |
| cmd = "gnt-cluster modify --instance-communication-network=" |
| result_output = qa_utils.GetCommandOutput(master.primary, cmd) |
| print result_output |
| |
| cmd = ["gnt-network", "disconnect", network_name] |
| AssertCommand(utils.ShellQuoteArgs(cmd)) |
| |
| cmd = ["gnt-network", "remove", network_name] |
| AssertCommand(utils.ShellQuoteArgs(cmd)) |
| |
| cmd = ["gnt-group", "remove", group] |
| AssertCommand(utils.ShellQuoteArgs(cmd)) |
| |
| # Check that the 'default' node group exists |
| cmd = ["gnt-group", "list", "--no-headers", "-o", "name"] |
| result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), "default", msg="Checking 'default' group") |
| |
| # Check that no networks exist |
| cmd = ["gnt-network", "list", "--no-headers", "-o", "name"] |
| result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) |
| AssertEqual(result_output.strip(), "", msg="Checking networks") |
