Merge branch 'stable-2.15' into stable-2.16
[ganeti-github.git] / qa / qa_utils.py
1 #
2 #
3
4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are
9 # met:
10 #
11 # 1. Redistributions of source code must retain the above copyright notice,
12 # this list of conditions and the following disclaimer.
13 #
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 """Utilities for QA tests.
32
33 """
34
35 import contextlib
36 import copy
37 import datetime
38 import operator
39 import os
40 import random
41 import re
42 import socket
43 import subprocess
44 import sys
45 import tempfile
46 import yaml
47
48 try:
49 import functools
50 except ImportError, err:
51 raise ImportError("Python 2.5 or higher is required: %s" % err)
52
53 from ganeti import utils
54 from ganeti import compat
55 from ganeti import constants
56 from ganeti import ht
57 from ganeti import pathutils
58 from ganeti import vcluster
59
60 import colors
61 import qa_config
62 import qa_error
63
64 from qa_logging import FormatInfo
65
66
67 _MULTIPLEXERS = {}
68
69 #: Unique ID per QA run
70 _RUN_UUID = utils.NewUUID()
71
72 #: Path to the QA query output log file
73 _QA_OUTPUT = pathutils.GetLogFilename("qa-output")
74
75
76 _RETRIES = 3
77
78
79 (INST_DOWN,
80 INST_UP) = range(500, 502)
81
82 (FIRST_ARG,
83 RETURN_VALUE) = range(1000, 1002)
84
85
86 def _RaiseWithInfo(msg, error_desc):
87 """Raises a QA error with the given content, and adds a message if present.
88
89 """
90 if msg:
91 output = "%s: %s" % (msg, error_desc)
92 else:
93 output = error_desc
94 raise qa_error.Error(output)
95
96
97 def AssertIn(item, sequence, msg=None):
98 """Raises an error when item is not in sequence.
99
100 """
101 if item not in sequence:
102 _RaiseWithInfo(msg, "%r not in %r" % (item, sequence))
103
104
105 def AssertNotIn(item, sequence, msg=None):
106 """Raises an error when item is in sequence.
107
108 """
109 if item in sequence:
110 _RaiseWithInfo(msg, "%r in %r" % (item, sequence))
111
112
113 def AssertEqual(first, second, msg=None):
114 """Raises an error when values aren't equal.
115
116 """
117 if not first == second:
118 _RaiseWithInfo(msg, "%r == %r" % (first, second))
119
120
121 def AssertMatch(string, pattern, msg=None):
122 """Raises an error when string doesn't match regexp pattern.
123
124 """
125 if not re.match(pattern, string):
126 _RaiseWithInfo(msg, "%r doesn't match /%r/" % (string, pattern))
127
128
129 def _GetName(entity, fn):
130 """Tries to get name of an entity.
131
132 @type entity: string or dict
133 @param fn: Function retrieving name from entity
134
135 """
136 if isinstance(entity, basestring):
137 result = entity
138 else:
139 result = fn(entity)
140
141 if not ht.TNonEmptyString(result):
142 raise Exception("Invalid name '%s'" % result)
143
144 return result
145
146
147 def _AssertRetCode(rcode, fail, cmdstr, nodename):
148 """Check the return value from a command and possibly raise an exception.
149
150 """
151 if fail and rcode == 0:
152 raise qa_error.Error("Command '%s' on node %s was expected to fail but"
153 " didn't" % (cmdstr, nodename))
154 elif not fail and rcode != 0:
155 raise qa_error.Error("Command '%s' on node %s failed, exit code %s" %
156 (cmdstr, nodename, rcode))
157
158
159 def _PrintCommandOutput(stdout, stderr):
160 """Prints the output of commands, minimizing wasted space.
161
162 @type stdout: string
163 @type stderr: string
164
165 """
166 if stdout:
167 stdout_clean = stdout.rstrip('\n')
168 if stderr:
169 print "Stdout was:\n%s" % stdout_clean
170 else:
171 print stdout_clean
172
173 if stderr:
174 print "Stderr was:"
175 print >> sys.stderr, stderr.rstrip('\n')
176
177
178 def AssertCommand(cmd, fail=False, node=None, log_cmd=True, forward_agent=True,
179 max_seconds=None):
180 """Checks that a remote command succeeds.
181
182 @param cmd: either a string (the command to execute) or a list (to
183 be converted using L{utils.ShellQuoteArgs} into a string)
184 @type fail: boolean or None
185 @param fail: if the command is expected to fail instead of succeeding,
186 or None if we don't care
187 @param node: if passed, it should be the node on which the command
188 should be executed, instead of the master node (can be either a
189 dict or a string)
190 @param log_cmd: if False, the command won't be logged (simply passed to
191 StartSSH)
192 @type forward_agent: boolean
193 @param forward_agent: whether to forward the agent when starting the SSH
194 session or not, sometimes useful for crypto-related
195 operations which can use a key they should not
196 @type max_seconds: double
197 @param max_seconds: fail if the command takes more than C{max_seconds}
198 seconds
199 @return: the return code, stdout and stderr of the command
200 @raise qa_error.Error: if the command fails when it shouldn't or vice versa
201
202 """
203 if node is None:
204 node = qa_config.GetMasterNode()
205
206 nodename = _GetName(node, operator.attrgetter("primary"))
207
208 if isinstance(cmd, basestring):
209 cmdstr = cmd
210 else:
211 cmdstr = utils.ShellQuoteArgs(cmd)
212
213 start = datetime.datetime.now()
214 popen = StartSSH(nodename, cmdstr, log_cmd=log_cmd,
215 forward_agent=forward_agent)
216 # Run the command
217 stdout, stderr = popen.communicate()
218 rcode = popen.returncode
219 duration_seconds = TimedeltaToTotalSeconds(datetime.datetime.now() - start)
220
221 try:
222 if fail is not None:
223 _AssertRetCode(rcode, fail, cmdstr, nodename)
224 finally:
225 if log_cmd:
226 _PrintCommandOutput(stdout, stderr)
227
228 if max_seconds is not None:
229 if duration_seconds > max_seconds:
230 raise qa_error.Error(
231 "Cmd '%s' took %f seconds, maximum of %f was exceeded" %
232 (cmdstr, duration_seconds, max_seconds))
233
234 return rcode, stdout, stderr
235
236
237 def stdout_of(cmd):
238 """Small helper to run a stdout_of.
239 Makes sure the stdout_of returns exit code 0.
240
241 @type cmd: list of strings
242 @param cmd: the stdout_of to run
243
244 @return: Captured, stripped stdout.
245 """
246 _, out, _ = AssertCommand(cmd)
247 return out.strip()
248
249
250 def AssertRedirectedCommand(cmd, fail=False, node=None, log_cmd=True):
251 """Executes a command with redirected output.
252
253 The log will go to the qa-output log file in the ganeti log
254 directory on the node where the command is executed. The fail and
255 node parameters are passed unchanged to AssertCommand.
256
257 @param cmd: the command to be executed, as a list; a string is not
258 supported
259
260 """
261 if not isinstance(cmd, list):
262 raise qa_error.Error("Non-list passed to AssertRedirectedCommand")
263 ofile = utils.ShellQuote(_QA_OUTPUT)
264 cmdstr = utils.ShellQuoteArgs(cmd)
265 AssertCommand("echo ---- $(date) %s ---- >> %s" % (cmdstr, ofile),
266 fail=False, node=node, log_cmd=False)
267 return AssertCommand(cmdstr + " >> %s" % ofile,
268 fail=fail, node=node, log_cmd=log_cmd)
269
270
271 def GetSSHCommand(node, cmd, strict=True, opts=None, tty=False,
272 use_multiplexer=True, forward_agent=True):
273 """Builds SSH command to be executed.
274
275 @type node: string
276 @param node: node the command should run on
277 @type cmd: string
278 @param cmd: command to be executed in the node; if None or empty
279 string, no command will be executed
280 @type strict: boolean
281 @param strict: whether to enable strict host key checking
282 @type opts: list
283 @param opts: list of additional options
284 @type tty: boolean or None
285 @param tty: if we should use tty; if None, will be auto-detected
286 @type use_multiplexer: boolean
287 @param use_multiplexer: if the multiplexer for the node should be used
288 @type forward_agent: boolean
289 @param forward_agent: whether to forward the ssh agent or not
290
291 """
292 args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-lroot"]
293
294 if tty is None:
295 tty = sys.stdout.isatty()
296
297 if tty:
298 args.append("-t")
299
300 # Multiplexers we use right now forward agents, so even if we ought to be
301 # using one, ignore it if agent forwarding is disabled.
302 if not forward_agent:
303 use_multiplexer = False
304
305 args.append("-oStrictHostKeyChecking=%s" % ("yes" if strict else "no", ))
306 args.append("-oClearAllForwardings=yes")
307 args.append("-oForwardAgent=%s" % ("yes" if forward_agent else "no", ))
308 if opts:
309 args.extend(opts)
310 if node in _MULTIPLEXERS and use_multiplexer:
311 spath = _MULTIPLEXERS[node][0]
312 args.append("-oControlPath=%s" % spath)
313 args.append("-oControlMaster=no")
314
315 (vcluster_master, vcluster_basedir) = \
316 qa_config.GetVclusterSettings()
317
318 if vcluster_master:
319 args.append(vcluster_master)
320 args.append("%s/%s/cmd" % (vcluster_basedir, node))
321
322 if cmd:
323 # For virtual clusters the whole command must be wrapped using the "cmd"
324 # script, as that script sets a number of environment variables. If the
325 # command contains shell meta characters the whole command needs to be
326 # quoted.
327 args.append(utils.ShellQuote(cmd))
328 else:
329 args.append(node)
330
331 if cmd:
332 args.append(cmd)
333
334 return args
335
336
337 def StartLocalCommand(cmd, _nolog_opts=False, log_cmd=True, **kwargs):
338 """Starts a local command.
339
340 """
341 if log_cmd:
342 if _nolog_opts:
343 pcmd = [i for i in cmd if not i.startswith("-")]
344 else:
345 pcmd = cmd
346 print "%s %s" % (colors.colorize("Command:", colors.CYAN),
347 utils.ShellQuoteArgs(pcmd))
348 return subprocess.Popen(cmd, shell=False, **kwargs)
349
350
351 def StartSSH(node, cmd, strict=True, log_cmd=True, forward_agent=True):
352 """Starts SSH.
353
354 """
355 ssh_command = GetSSHCommand(node, cmd, strict=strict,
356 forward_agent=forward_agent)
357 return StartLocalCommand(ssh_command, _nolog_opts=True, log_cmd=log_cmd,
358 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
359
360
361 def StartMultiplexer(node):
362 """Starts a multiplexer command.
363
364 @param node: the node for which to open the multiplexer
365
366 """
367 if node in _MULTIPLEXERS:
368 return
369
370 # Note: yes, we only need mktemp, since we'll remove the file anyway
371 sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
372 utils.RemoveFile(sname)
373 opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
374 print "Created socket at %s" % sname
375 child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
376 _MULTIPLEXERS[node] = (sname, child)
377
378
379 def CloseMultiplexers():
380 """Closes all current multiplexers and cleans up.
381
382 """
383 for node in _MULTIPLEXERS.keys():
384 (sname, child) = _MULTIPLEXERS.pop(node)
385 utils.KillProcess(child.pid, timeout=10, waitpid=True)
386 utils.RemoveFile(sname)
387
388
389 def _GetCommandStdout(proc):
390 """Extract the stored standard error, print it and return it.
391
392 """
393 out = proc.stdout.read()
394 sys.stdout.write(out)
395 return out
396
397
398 def _NoTimeout(state):
399 """False iff the command timed out."""
400 rcode, out = state
401
402 return rcode == 0 or not ('TimeoutError' in out or 'timed out' in out)
403
404
405 def GetCommandOutput(node, cmd, tty=False, use_multiplexer=True, log_cmd=True,
406 fail=False):
407 """Returns the output of a command executed on the given node.
408
409 @type node: string
410 @param node: node the command should run on
411 @type cmd: string
412 @param cmd: command to be executed in the node (cannot be empty or None)
413 @type tty: bool or None
414 @param tty: if we should use tty; if None, it will be auto-detected
415 @type use_multiplexer: bool
416 @param use_multiplexer: if the SSH multiplexer provided by the QA should be
417 used or not
418 @type log_cmd: bool
419 @param log_cmd: if the command should be logged
420 @type fail: bool
421 @param fail: whether the command is expected to fail
422 """
423 assert cmd
424
425 def CallCommand():
426 command = GetSSHCommand(node, cmd, tty=tty,
427 use_multiplexer=use_multiplexer)
428 p = StartLocalCommand(command, stdout=subprocess.PIPE, log_cmd=log_cmd)
429 rcode = p.wait()
430 out = _GetCommandStdout(p)
431 return rcode, out
432
433 # TODO: make retries configurable
434 rcode, out = utils.CountRetry(_NoTimeout, CallCommand, _RETRIES)
435 _AssertRetCode(rcode, fail, cmd, node)
436 return out
437
438
439 def GetObjectInfo(infocmd):
440 """Get and parse information about a Ganeti object.
441
442 @type infocmd: list of strings
443 @param infocmd: command to be executed, e.g. ["gnt-cluster", "info"]
444 @return: the information parsed, appropriately stored in dictionaries,
445 lists...
446
447 """
448 master = qa_config.GetMasterNode()
449 cmdline = utils.ShellQuoteArgs(infocmd)
450 info_out = GetCommandOutput(master.primary, cmdline)
451 return yaml.load(info_out)
452
453
454 def UploadFile(node, src):
455 """Uploads a file to a node and returns the filename.
456
457 Caller needs to remove the returned file on the node when it's not needed
458 anymore.
459
460 """
461 # Make sure nobody else has access to it while preserving local permissions
462 mode = os.stat(src).st_mode & 0700
463
464 cmd = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
465 'chmod %o "${tmp}" && '
466 '[[ -f "${tmp}" ]] && '
467 'cat > "${tmp}" && '
468 'echo "${tmp}"') % mode
469
470 f = open(src, "r")
471 try:
472 p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
473 stdout=subprocess.PIPE)
474 AssertEqual(p.wait(), 0)
475
476 # Return temporary filename
477 return _GetCommandStdout(p).strip()
478 finally:
479 f.close()
480
481
482 def UploadData(node, data, mode=0600, filename=None):
483 """Uploads data to a node and returns the filename.
484
485 Caller needs to remove the returned file on the node when it's not needed
486 anymore.
487
488 """
489 if filename:
490 tmp = "tmp=%s" % utils.ShellQuote(filename)
491 else:
492 tmp = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
493 'chmod %o "${tmp}"') % mode
494 cmd = ("%s && "
495 "[[ -f \"${tmp}\" ]] && "
496 "cat > \"${tmp}\" && "
497 "echo \"${tmp}\"") % tmp
498
499 p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False,
500 stdin=subprocess.PIPE, stdout=subprocess.PIPE)
501 p.stdin.write(data)
502 p.stdin.close()
503 AssertEqual(p.wait(), 0)
504
505 # Return temporary filename
506 return _GetCommandStdout(p).strip()
507
508
509 def BackupFile(node, path):
510 """Creates a backup of a file on the node and returns the filename.
511
512 Caller needs to remove the returned file on the node when it's not needed
513 anymore.
514
515 """
516 vpath = MakeNodePath(node, path)
517
518 cmd = ("tmp=$(mktemp .gnt.XXXXXX --tmpdir=$(dirname %s)) && "
519 "[[ -f \"$tmp\" ]] && "
520 "cp %s $tmp && "
521 "echo $tmp") % (utils.ShellQuote(vpath), utils.ShellQuote(vpath))
522
523 # Return temporary filename
524 result = GetCommandOutput(node, cmd).strip()
525
526 print "Backup filename: %s" % result
527
528 return result
529
530
531 @contextlib.contextmanager
532 def CheckFileUnmodified(node, filename):
533 """Checks that the content of a given file remains the same after running a
534 wrapped code.
535
536 @type node: string
537 @param node: node the command should run on
538 @type filename: string
539 @param filename: absolute filename to check
540
541 """
542 cmd = utils.ShellQuoteArgs(["sha1sum", MakeNodePath(node, filename)])
543
544 def Read():
545 return GetCommandOutput(node, cmd).strip()
546
547 # read the configuration
548 before = Read()
549 yield
550 # check that the configuration hasn't changed
551 after = Read()
552 if before != after:
553 raise qa_error.Error("File '%s' has changed unexpectedly on node %s"
554 " during the last operation" % (filename, node))
555
556
557 def ResolveInstanceName(instance):
558 """Gets the full name of an instance.
559
560 @type instance: string
561 @param instance: Instance name
562
563 """
564 info = GetObjectInfo(["gnt-instance", "info", instance])
565 return info[0]["Instance name"]
566
567
568 def ResolveNodeName(node):
569 """Gets the full name of a node.
570
571 """
572 info = GetObjectInfo(["gnt-node", "info", node.primary])
573 return info[0]["Node name"]
574
575
576 def GetNodeInstances(node, secondaries=False):
577 """Gets a list of instances on a node.
578
579 """
580 master = qa_config.GetMasterNode()
581 node_name = ResolveNodeName(node)
582
583 # Get list of all instances
584 cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
585 "--output=name,pnode,snodes"]
586 output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
587
588 instances = []
589 for line in output.splitlines():
590 (name, pnode, snodes) = line.split(":", 2)
591 if ((not secondaries and pnode == node_name) or
592 (secondaries and node_name in snodes.split(","))):
593 instances.append(name)
594
595 return instances
596
597
598 def _SelectQueryFields(rnd, fields):
599 """Generates a list of fields for query tests.
600
601 """
602 # Create copy for shuffling
603 fields = list(fields)
604 rnd.shuffle(fields)
605
606 # Check all fields
607 yield fields
608 yield sorted(fields)
609
610 # Duplicate fields
611 yield fields + fields
612
613 # Check small groups of fields
614 while fields:
615 yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields]
616
617
618 def _List(listcmd, fields, names):
619 """Runs a list command.
620
621 """
622 master = qa_config.GetMasterNode()
623
624 cmd = [listcmd, "list", "--separator=|", "--no-headers",
625 "--output", ",".join(fields)]
626
627 if names:
628 cmd.extend(names)
629
630 return GetCommandOutput(master.primary,
631 utils.ShellQuoteArgs(cmd)).splitlines()
632
633
634 def GenericQueryTest(cmd, fields, namefield="name", test_unknown=True):
635 """Runs a number of tests on query commands.
636
637 @param cmd: Command name
638 @param fields: List of field names
639
640 """
641 rnd = random.Random(hash(cmd))
642
643 fields = list(fields)
644 rnd.shuffle(fields)
645
646 # Test a number of field combinations
647 for testfields in _SelectQueryFields(rnd, fields):
648 AssertRedirectedCommand([cmd, "list", "--output", ",".join(testfields)])
649
650 if namefield is not None:
651 namelist_fn = compat.partial(_List, cmd, [namefield])
652
653 # When no names were requested, the list must be sorted
654 names = namelist_fn(None)
655 AssertEqual(names, utils.NiceSort(names))
656
657 # When requesting specific names, the order must be kept
658 revnames = list(reversed(names))
659 AssertEqual(namelist_fn(revnames), revnames)
660
661 randnames = list(names)
662 rnd.shuffle(randnames)
663 AssertEqual(namelist_fn(randnames), randnames)
664
665 if test_unknown:
666 # Listing unknown items must fail
667 AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"],
668 fail=True)
669
670 # Check exit code for listing unknown field
671 rcode, _, _ = AssertRedirectedCommand([cmd, "list",
672 "--output=field/does/not/exist"],
673 fail=True)
674 AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
675
676
677 def GenericQueryFieldsTest(cmd, fields):
678 master = qa_config.GetMasterNode()
679
680 # Listing fields
681 AssertRedirectedCommand([cmd, "list-fields"])
682 AssertRedirectedCommand([cmd, "list-fields"] + fields)
683
684 # Check listed fields (all, must be sorted)
685 realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
686 output = GetCommandOutput(master.primary,
687 utils.ShellQuoteArgs(realcmd)).splitlines()
688 AssertEqual([line.split("|", 1)[0] for line in output],
689 utils.NiceSort(fields))
690
691 # Check exit code for listing unknown field
692 rcode, _, _ = AssertCommand([cmd, "list-fields", "field/does/not/exist"],
693 fail=True)
694 AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
695
696
697 def AddToEtcHosts(hostnames):
698 """Adds hostnames to /etc/hosts.
699
700 @param hostnames: List of hostnames first used A records, all other CNAMEs
701
702 """
703 master = qa_config.GetMasterNode()
704 tmp_hosts = UploadData(master.primary, "", mode=0644)
705
706 data = []
707 for localhost in ("::1", "127.0.0.1"):
708 data.append("%s %s" % (localhost, " ".join(hostnames)))
709
710 try:
711 AssertCommand("{ cat %s && echo -e '%s'; } > %s && mv %s %s" %
712 (utils.ShellQuote(pathutils.ETC_HOSTS),
713 "\\n".join(data),
714 utils.ShellQuote(tmp_hosts),
715 utils.ShellQuote(tmp_hosts),
716 utils.ShellQuote(pathutils.ETC_HOSTS)))
717 except Exception:
718 AssertCommand(["rm", "-f", tmp_hosts])
719 raise
720
721
722 def RemoveFromEtcHosts(hostnames):
723 """Remove hostnames from /etc/hosts.
724
725 @param hostnames: List of hostnames first used A records, all other CNAMEs
726
727 """
728 master = qa_config.GetMasterNode()
729 tmp_hosts = UploadData(master.primary, "", mode=0644)
730 quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
731
732 sed_data = " ".join(hostnames)
733 try:
734 AssertCommand((r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
735 r" && mv %s %s") %
736 (sed_data, utils.ShellQuote(pathutils.ETC_HOSTS),
737 quoted_tmp_hosts, quoted_tmp_hosts,
738 utils.ShellQuote(pathutils.ETC_HOSTS)))
739 except Exception:
740 AssertCommand(["rm", "-f", tmp_hosts])
741 raise
742
743
744 def RunInstanceCheck(instance, running):
745 """Check if instance is running or not.
746
747 """
748 instance_name = _GetName(instance, operator.attrgetter("name"))
749
750 script = qa_config.GetInstanceCheckScript()
751 if not script:
752 return
753
754 master_node = qa_config.GetMasterNode()
755
756 # Build command to connect to master node
757 master_ssh = GetSSHCommand(master_node.primary, "--")
758
759 if running:
760 running_shellval = "1"
761 running_text = ""
762 else:
763 running_shellval = ""
764 running_text = "not "
765
766 print FormatInfo("Checking if instance '%s' is %srunning" %
767 (instance_name, running_text))
768
769 args = [script, instance_name]
770 env = {
771 "PATH": constants.HOOKS_PATH,
772 "RUN_UUID": _RUN_UUID,
773 "MASTER_SSH": utils.ShellQuoteArgs(master_ssh),
774 "INSTANCE_NAME": instance_name,
775 "INSTANCE_RUNNING": running_shellval,
776 }
777
778 result = os.spawnve(os.P_WAIT, script, args, env)
779 if result != 0:
780 raise qa_error.Error("Instance check failed with result %s" % result)
781
782
783 def _InstanceCheckInner(expected, instarg, args, result):
784 """Helper function used by L{InstanceCheck}.
785
786 """
787 if instarg == FIRST_ARG:
788 instance = args[0]
789 elif instarg == RETURN_VALUE:
790 instance = result
791 else:
792 raise Exception("Invalid value '%s' for instance argument" % instarg)
793
794 if expected in (INST_DOWN, INST_UP):
795 RunInstanceCheck(instance, (expected == INST_UP))
796 elif expected is not None:
797 raise Exception("Invalid value '%s'" % expected)
798
799
800 def InstanceCheck(before, after, instarg):
801 """Decorator to check instance status before and after test.
802
803 @param before: L{INST_DOWN} if instance must be stopped before test,
804 L{INST_UP} if instance must be running before test, L{None} to not check.
805 @param after: L{INST_DOWN} if instance must be stopped after test,
806 L{INST_UP} if instance must be running after test, L{None} to not check.
807 @param instarg: L{FIRST_ARG} to use first argument to test as instance (a
808 dictionary), L{RETURN_VALUE} to use return value (disallows pre-checks)
809
810 """
811 def decorator(fn):
812 @functools.wraps(fn)
813 def wrapper(*args, **kwargs):
814 _InstanceCheckInner(before, instarg, args, NotImplemented)
815
816 result = fn(*args, **kwargs)
817
818 _InstanceCheckInner(after, instarg, args, result)
819
820 return result
821 return wrapper
822 return decorator
823
824
825 def GetNonexistentGroups(count):
826 """Gets group names which shouldn't exist on the cluster.
827
828 @param count: Number of groups to get
829 @rtype: integer
830
831 """
832 return GetNonexistentEntityNames(count, "groups", "group")
833
834
835 def GetNonexistentEntityNames(count, name_config, name_prefix):
836 """Gets entity names which shouldn't exist on the cluster.
837
838 The actualy names can refer to arbitrary entities (for example
839 groups, networks).
840
841 @param count: Number of names to get
842 @rtype: integer
843 @param name_config: name of the leaf in the config containing
844 this entity's configuration, including a 'inexistent-'
845 element
846 @rtype: string
847 @param name_prefix: prefix of the entity's names, used to compose
848 the default values; for example for groups, the prefix is
849 'group' and the generated names are then group1, group2, ...
850 @rtype: string
851
852 """
853 entities = qa_config.get(name_config, {})
854
855 default = [name_prefix + str(i) for i in range(count)]
856 assert count <= len(default)
857
858 name_config_inexistent = "inexistent-" + name_config
859 candidates = entities.get(name_config_inexistent, default)[:count]
860
861 if len(candidates) < count:
862 raise Exception("At least %s non-existent %s are needed" %
863 (count, name_config))
864
865 return candidates
866
867
868 def MakeNodePath(node, path):
869 """Builds an absolute path for a virtual node.
870
871 @type node: string or L{qa_config._QaNode}
872 @param node: Node
873 @type path: string
874 @param path: Path without node-specific prefix
875
876 """
877 (_, basedir) = qa_config.GetVclusterSettings()
878
879 if isinstance(node, basestring):
880 name = node
881 else:
882 name = node.primary
883
884 if basedir:
885 assert path.startswith("/")
886 return "%s%s" % (vcluster.MakeNodeRoot(basedir, name), path)
887 else:
888 return path
889
890
891 def _GetParameterOptions(specs):
892 """Helper to build policy options."""
893 values = ["%s=%s" % (par, val)
894 for (par, val) in specs.items()]
895 return ",".join(values)
896
897
898 def TestSetISpecs(new_specs=None, diff_specs=None, get_policy_fn=None,
899 build_cmd_fn=None, fail=False, old_values=None):
900 """Change instance specs for an object.
901
902 At most one of new_specs or diff_specs can be specified.
903
904 @type new_specs: dict
905 @param new_specs: new complete specs, in the same format returned by
906 L{ParseIPolicy}.
907 @type diff_specs: dict
908 @param diff_specs: partial specs, it can be an incomplete specifications, but
909 if min/max specs are specified, their number must match the number of the
910 existing specs
911 @type get_policy_fn: function
912 @param get_policy_fn: function that returns the current policy as in
913 L{ParseIPolicy}
914 @type build_cmd_fn: function
915 @param build_cmd_fn: function that return the full command line from the
916 options alone
917 @type fail: bool
918 @param fail: if the change is expected to fail
919 @type old_values: tuple
920 @param old_values: (old_policy, old_specs), as returned by
921 L{ParseIPolicy}
922 @return: same as L{ParseIPolicy}
923
924 """
925 assert get_policy_fn is not None
926 assert build_cmd_fn is not None
927 assert new_specs is None or diff_specs is None
928
929 if old_values:
930 (old_policy, old_specs) = old_values
931 else:
932 (old_policy, old_specs) = get_policy_fn()
933
934 if diff_specs:
935 new_specs = copy.deepcopy(old_specs)
936 if constants.ISPECS_MINMAX in diff_specs:
937 AssertEqual(len(new_specs[constants.ISPECS_MINMAX]),
938 len(diff_specs[constants.ISPECS_MINMAX]))
939 for (new_minmax, diff_minmax) in zip(new_specs[constants.ISPECS_MINMAX],
940 diff_specs[constants.ISPECS_MINMAX]):
941 for (key, parvals) in diff_minmax.items():
942 for (par, val) in parvals.items():
943 new_minmax[key][par] = val
944 for (par, val) in diff_specs.get(constants.ISPECS_STD, {}).items():
945 new_specs[constants.ISPECS_STD][par] = val
946
947 if new_specs:
948 cmd = []
949 if (diff_specs is None or constants.ISPECS_MINMAX in diff_specs):
950 minmax_opt_items = []
951 for minmax in new_specs[constants.ISPECS_MINMAX]:
952 minmax_opts = []
953 for key in ["min", "max"]:
954 keyopt = _GetParameterOptions(minmax[key])
955 minmax_opts.append("%s:%s" % (key, keyopt))
956 minmax_opt_items.append("/".join(minmax_opts))
957 cmd.extend([
958 "--ipolicy-bounds-specs",
959 "//".join(minmax_opt_items)
960 ])
961 if diff_specs is None:
962 std_source = new_specs
963 else:
964 std_source = diff_specs
965 std_opt = _GetParameterOptions(std_source.get("std", {}))
966 if std_opt:
967 cmd.extend(["--ipolicy-std-specs", std_opt])
968 AssertCommand(build_cmd_fn(cmd), fail=fail)
969
970 # Check the new state
971 (eff_policy, eff_specs) = get_policy_fn()
972 AssertEqual(eff_policy, old_policy)
973 if fail:
974 AssertEqual(eff_specs, old_specs)
975 else:
976 AssertEqual(eff_specs, new_specs)
977
978 else:
979 (eff_policy, eff_specs) = (old_policy, old_specs)
980
981 return (eff_policy, eff_specs)
982
983
984 def ParseIPolicy(policy):
985 """Parse and split instance an instance policy.
986
987 @type policy: dict
988 @param policy: policy, as returned by L{GetObjectInfo}
989 @rtype: tuple
990 @return: (policy, specs), where:
991 - policy is a dictionary of the policy values, instance specs excluded
992 - specs is a dictionary containing only the specs, using the internal
993 format (see L{constants.IPOLICY_DEFAULTS} for an example)
994
995 """
996 ret_specs = {}
997 ret_policy = {}
998 for (key, val) in policy.items():
999 if key == "bounds specs":
1000 ret_specs[constants.ISPECS_MINMAX] = []
1001 for minmax in val:
1002 ret_minmax = {}
1003 for key in minmax:
1004 keyparts = key.split("/", 1)
1005 assert len(keyparts) > 1
1006 ret_minmax[keyparts[0]] = minmax[key]
1007 ret_specs[constants.ISPECS_MINMAX].append(ret_minmax)
1008 elif key == constants.ISPECS_STD:
1009 ret_specs[key] = val
1010 else:
1011 ret_policy[key] = val
1012 return (ret_policy, ret_specs)
1013
1014
1015 def UsesIPv6Connection(host, port):
1016 """Returns True if the connection to a given host/port could go through IPv6.
1017
1018 """
1019 return any(t[0] == socket.AF_INET6 for t in socket.getaddrinfo(host, port))
1020
1021
1022 def TimedeltaToTotalSeconds(td):
1023 """Returns the total seconds in a C{datetime.timedelta} object.
1024
1025 This performs the same task as the C{datetime.timedelta.total_seconds()}
1026 method which is present in Python 2.7 onwards.
1027
1028 @type td: datetime.timedelta
1029 @param td: timedelta object to convert
1030 @rtype float
1031 @return: total seconds in the timedelta object
1032
1033 """
1034 return ((td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10 ** 6) /
1035 10 ** 6)