test/hs/shelltests/htools-balancing.test - ganeti - Git at Google

 ### std tests

 # test basic parsing
 ./test/hs/hinfo -v -v -p --print-instances $BACKEND_BAL_STD
 >>>= 0
 ./test/hs/hbal -v -v -v -p --print-instances $BACKEND_BAL_STD -G group-01
 >>> !/(Nothing to do, exiting|No solution found)/
 >>>2 !/(Nothing to do, exiting|No solution found)/
 >>>= 0

 # test command output
 ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C -S $T/simu-rebal.standard
 >>> /gnt-instance (failover|migrate|replace-disks)/
 >>>= 0

 # test that correct priorities are accepted
 ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C -S $T/simu-rebal.standard --prio low
 >>> /gnt-instance (failover|migrate|replace-disks)/
 >>>= 0


 # test that hbal won't execute rebalances when using the text backend
 ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -X
 >>>2
 Error: hbal: Execution of commands possible only on LUXI
 >>>= !0

 # test that hbal won't execute any moves if we request an absurdly-high
 # minimum-improvement
 ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C --min-gain 10000 --min-gain-limit 10000
 >>>/No solution found/
 >>>= 0

 # test saving commands
 ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C$T/rebal-cmds.standard
 >>>= 0
 # and now check the file (depends on previous test)
 cat $T/rebal-cmds.standard
 >>> /gnt-instance (failover|migrate|replace-disks)/
 >>>= 0

 # state saved before rebalancing should be identical; depends on the
 # previous test
 diff -u $T/simu-rebal-merged.standard $T/simu-rebal.standard.original
 >>>
 >>>= 0

 # no double rebalance; depends on previous test
 ./test/hs/hbal -t $T/simu-rebal.standard.balanced -G group-01
 >>> /(Nothing to do, exiting|No solution found)/
 >>>= 0

 # hcheck sees no reason to rebalance after rebalancing was already done
 ./test/hs/hcheck -t$T/simu-rebal.standard.balanced --machine-readable
 >>> /HCHECK_INIT_CLUSTER_NEED_REBALANCE=0/
 >>>= 0

 ### now tiered tests

 # test basic parsing
 ./test/hs/hinfo -v -v -p --print-instances $BACKEND_BAL_TIER
 >>>= 0
 ./test/hs/hbal -v -v -v -p --print-instances $BACKEND_BAL_TIER -G group-01
 >>> !/(Nothing to do, exiting|No solution found)/
 >>>2 !/(Nothing to do, exiting|No solution found)/
 >>>= 0

 # test command output
 ./test/hs/hbal $BACKEND_BAL_TIER -G group-01 -C -S $T/simu-rebal.tiered
 >>> /gnt-instance (failover|migrate|replace-disks)/
 >>>= 0

 # test saving commands
 ./test/hs/hbal $BACKEND_BAL_TIER -G group-01 -C$T/rebal-cmds.tiered
 >>>= 0
 # and now check the file (depends on previous test)
 cat $T/rebal-cmds.tiered
 >>> /gnt-instance (failover|migrate|replace-disks)/
 >>>= 0

 # state saved before rebalancing should be identical; depends on the
 # previous test
 diff -u $T/simu-rebal-merged.tiered $T/simu-rebal.tiered.original
 >>>
 >>>= 0

 # no double rebalance; depends on previous test
 ./test/hs/hbal -t $T/simu-rebal.tiered.balanced -G group-01
 >>> /(Nothing to do, exiting|No solution found)/
 >>>= 0

 ### now some other custom tests

 # n+1 bad instances are reported as such
 ./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-01
 >>>/Initial check done: 4 bad nodes, 8 bad instances./
 >>>=0

 # same test again, different message check (shelltest can't test multiple
 # messages via regexp
 ./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-01
 >>>/Cluster is not N\+1 happy, continuing but no guarantee that the cluster will end N\+1 happy./
 >>>2
 >>>=0

 # and hcheck should report this as needs rebalancing
 ./test/hs/hcheck -t$TESTDATA_DIR/n1-failure.data
 >>>/Cluster needs rebalancing./
 >>>= 1

 # ... unless we request no-simulation mode
 ./test/hs/hcheck -t$TESTDATA_DIR/n1-failure.data --no-simulation
 >>>/Running in no-simulation mode./
 >>>= 0

 # and a clean cluster should be reported as such
 ./test/hs/hcheck $BACKEND_BAL_STD
 >>>/No need to rebalance cluster, no problems found./
 >>>= 0

 # ... and even one with non-zero score
 ./test/hs/hcheck -t $TESTDATA_DIR/clean-nonzero-score.data
 >>>/No need to rebalance cluster, no problems found./
 >>>= 0

 # hbal should work on empty groups as well
 ./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-02
 >>>/Group size 0 nodes, 0 instances/
 >>>= 0

 # By default, hbal should assume equal, non-zero utilisation
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data
 >>>/Solution length=1/
 >>>=0

 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data -C
 >>>/gnt-instance migrate -f -n node-01-001 inst-0./
 >>>=0

 # ...but the --ignore-dynu option should be honored
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --ignore-dynu
 >>>/Cluster is already well balanced/
 >>>=0

 # Assuming idle default also gives 0 utilisation
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --idle-default
 >>>/Cluster is already well balanced/
 >>>=0

 # Heavy CPU load can even push instances on the more
 # crowded node
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --idle-default --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json -C
 >>>/gnt-instance migrate -f -n node-01-000 inst-1./
 >>>=0

 # ...but with default assumption about disk/net/mem fully used the move is in
 # the other direction.
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json
 >>>/Solution length=1/
 >>>=0

 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json -C
 >>>/gnt-instance migrate -f -n node-01-001 inst-0./
 >>>=0

 # Still, --ignore-dynu overrides everything
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --idle-default --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json --ignore-dynu
 >>>/Cluster is already well balanced/
 >>>=0

 # On an overcommitted cluster with small amount of memory, taking memory
 # usage into account can make a difference.
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn2.json
 >>>/Solution length=0/
 >>>=0

 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn2.json --mond-kvm-rss -C
 >>>/gnt-instance migrate -f -n node-01-000 inst-1./
 >>>=0

 # Depending on weight, instances can move in either direction
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn3.json --mond-kvm-rss -C --mem-weight=0.5
 >>>/ 1\. inst-0./
 >>>=0

 ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn3.json --mond-kvm-rss -C --mem-weight=3.0
 >>>/ 1\. inst-1./
 >>>=0

 # Test CPU speed is taken into account
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-cpu-speed.data --ignore-dynu
 >>>/inst[12] node-slow:node-fast => node-fast:node-slow/
 >>>=0

 # By default, hbal should not take any action in an overloaded cluster
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data
 >>>/Solution length=0/
 >>>=0

 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data
 >>>/Solution length=0/
 >>>=0

 # ...and suggest to the user to weaken movement restrictions;
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data
 >>>/consider using the --ignore-soft-errors option/
 >>>=0

 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data
 >>>/consider using the --ignore-soft-errors option/
 >>>=0

 # allowing soft errors, an improvement should be found.
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data --ignore-soft-errors
 >>>/Solution length=[1-9]/
 >>>=0

 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data --ignore-soft-errors
 >>>/Solution length=[1-9]/
 >>>=0

 # forthcoming instances can be balanced as well
 ./test/hs/hbal -t$TESTDATA_DIR/hbal-forth.data
 >>>/Solution length=[1-9]/
 >>>=0
	### std tests

	# test basic parsing
	./test/hs/hinfo -v -v -p --print-instances $BACKEND_BAL_STD
	>>>= 0
	./test/hs/hbal -v -v -v -p --print-instances $BACKEND_BAL_STD -G group-01
	>>> !/(Nothing to do, exiting\|No solution found)/
	>>>2 !/(Nothing to do, exiting\|No solution found)/
	>>>= 0

	# test command output
	./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C -S $T/simu-rebal.standard
	>>> /gnt-instance (failover\|migrate\|replace-disks)/
	>>>= 0

	# test that correct priorities are accepted
	./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C -S $T/simu-rebal.standard --prio low
	>>> /gnt-instance (failover\|migrate\|replace-disks)/
	>>>= 0


	# test that hbal won't execute rebalances when using the text backend
	./test/hs/hbal $BACKEND_BAL_STD -G group-01 -X
	>>>2
	Error: hbal: Execution of commands possible only on LUXI
	>>>= !0

	# test that hbal won't execute any moves if we request an absurdly-high
	# minimum-improvement
	./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C --min-gain 10000 --min-gain-limit 10000
	>>>/No solution found/
	>>>= 0

	# test saving commands
	./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C$T/rebal-cmds.standard
	>>>= 0
	# and now check the file (depends on previous test)
	cat $T/rebal-cmds.standard
	>>> /gnt-instance (failover\|migrate\|replace-disks)/
	>>>= 0

	# state saved before rebalancing should be identical; depends on the
	# previous test
	diff -u $T/simu-rebal-merged.standard $T/simu-rebal.standard.original
	>>>
	>>>= 0

	# no double rebalance; depends on previous test
	./test/hs/hbal -t $T/simu-rebal.standard.balanced -G group-01
	>>> /(Nothing to do, exiting\|No solution found)/
	>>>= 0

	# hcheck sees no reason to rebalance after rebalancing was already done
	./test/hs/hcheck -t$T/simu-rebal.standard.balanced --machine-readable
	>>> /HCHECK_INIT_CLUSTER_NEED_REBALANCE=0/
	>>>= 0

	### now tiered tests

	# test basic parsing
	./test/hs/hinfo -v -v -p --print-instances $BACKEND_BAL_TIER
	>>>= 0
	./test/hs/hbal -v -v -v -p --print-instances $BACKEND_BAL_TIER -G group-01
	>>> !/(Nothing to do, exiting\|No solution found)/
	>>>2 !/(Nothing to do, exiting\|No solution found)/
	>>>= 0

	# test command output
	./test/hs/hbal $BACKEND_BAL_TIER -G group-01 -C -S $T/simu-rebal.tiered
	>>> /gnt-instance (failover\|migrate\|replace-disks)/
	>>>= 0

	# test saving commands
	./test/hs/hbal $BACKEND_BAL_TIER -G group-01 -C$T/rebal-cmds.tiered
	>>>= 0
	# and now check the file (depends on previous test)
	cat $T/rebal-cmds.tiered
	>>> /gnt-instance (failover\|migrate\|replace-disks)/
	>>>= 0

	# state saved before rebalancing should be identical; depends on the
	# previous test
	diff -u $T/simu-rebal-merged.tiered $T/simu-rebal.tiered.original
	>>>
	>>>= 0

	# no double rebalance; depends on previous test
	./test/hs/hbal -t $T/simu-rebal.tiered.balanced -G group-01
	>>> /(Nothing to do, exiting\|No solution found)/
	>>>= 0

	### now some other custom tests

	# n+1 bad instances are reported as such
	./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-01
	>>>/Initial check done: 4 bad nodes, 8 bad instances./
	>>>=0

	# same test again, different message check (shelltest can't test multiple
	# messages via regexp
	./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-01
	>>>/Cluster is not N\+1 happy, continuing but no guarantee that the cluster will end N\+1 happy./
	>>>2
	>>>=0

	# and hcheck should report this as needs rebalancing
	./test/hs/hcheck -t$TESTDATA_DIR/n1-failure.data
	>>>/Cluster needs rebalancing./
	>>>= 1

	# ... unless we request no-simulation mode
	./test/hs/hcheck -t$TESTDATA_DIR/n1-failure.data --no-simulation
	>>>/Running in no-simulation mode./
	>>>= 0

	# and a clean cluster should be reported as such
	./test/hs/hcheck $BACKEND_BAL_STD
	>>>/No need to rebalance cluster, no problems found./
	>>>= 0

	# ... and even one with non-zero score
	./test/hs/hcheck -t $TESTDATA_DIR/clean-nonzero-score.data
	>>>/No need to rebalance cluster, no problems found./
	>>>= 0

	# hbal should work on empty groups as well
	./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-02
	>>>/Group size 0 nodes, 0 instances/
	>>>= 0

	# By default, hbal should assume equal, non-zero utilisation
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data
	>>>/Solution length=1/
	>>>=0

	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data -C
	>>>/gnt-instance migrate -f -n node-01-001 inst-0./
	>>>=0

	# ...but the --ignore-dynu option should be honored
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --ignore-dynu
	>>>/Cluster is already well balanced/
	>>>=0

	# Assuming idle default also gives 0 utilisation
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --idle-default
	>>>/Cluster is already well balanced/
	>>>=0

	# Heavy CPU load can even push instances on the more
	# crowded node
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --idle-default --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json -C
	>>>/gnt-instance migrate -f -n node-01-000 inst-1./
	>>>=0

	# ...but with default assumption about disk/net/mem fully used the move is in
	# the other direction.
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json
	>>>/Solution length=1/
	>>>=0

	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json -C
	>>>/gnt-instance migrate -f -n node-01-001 inst-0./
	>>>=0

	# Still, --ignore-dynu overrides everything
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --idle-default --mond --mond-xen --mond-data=$TESTDATA_DIR/dyn1.json --ignore-dynu
	>>>/Cluster is already well balanced/
	>>>=0

	# On an overcommitted cluster with small amount of memory, taking memory
	# usage into account can make a difference.
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn2.json
	>>>/Solution length=0/
	>>>=0

	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn2.json --mond-kvm-rss -C
	>>>/gnt-instance migrate -f -n node-01-000 inst-1./
	>>>=0

	# Depending on weight, instances can move in either direction
	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn3.json --mond-kvm-rss -C --mem-weight=0.5
	>>>/ 1\. inst-0./
	>>>=0

	./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn2.data --idle-default --mond --mond-data=$TESTDATA_DIR/dyn3.json --mond-kvm-rss -C --mem-weight=3.0
	>>>/ 1\. inst-1./
	>>>=0

	# Test CPU speed is taken into account
	./test/hs/hbal -t$TESTDATA_DIR/hbal-cpu-speed.data --ignore-dynu
	>>>/inst[12] node-slow:node-fast => node-fast:node-slow/
	>>>=0

	# By default, hbal should not take any action in an overloaded cluster
	./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data
	>>>/Solution length=0/
	>>>=0

	./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data
	>>>/Solution length=0/
	>>>=0

	# ...and suggest to the user to weaken movement restrictions;
	./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data
	>>>/consider using the --ignore-soft-errors option/
	>>>=0

	./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data
	>>>/consider using the --ignore-soft-errors option/
	>>>=0

	# allowing soft errors, an improvement should be found.
	./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data --ignore-soft-errors
	>>>/Solution length=[1-9]/
	>>>=0

	./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data --ignore-soft-errors
	>>>/Solution length=[1-9]/
	>>>=0

	# forthcoming instances can be balanced as well
	./test/hs/hbal -t$TESTDATA_DIR/hbal-forth.data
	>>>/Solution length=[1-9]/
	>>>=0