Do not add a new Inotify watchers on timer
[ganeti-github.git] / qa / qa_utils.py
1 #
2 #
3
4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are
9 # met:
10 #
11 # 1. Redistributions of source code must retain the above copyright notice,
12 # this list of conditions and the following disclaimer.
13 #
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 """Utilities for QA tests.
32
33 """
34
35 import contextlib
36 import copy
37 import datetime
38 import operator
39 import os
40 import random
41 import re
42 import socket
43 import subprocess
44 import sys
45 import tempfile
46 import yaml
47
48 try:
49 import functools
50 except ImportError, err:
51 raise ImportError("Python 2.5 or higher is required: %s" % err)
52
53 from ganeti import utils
54 from ganeti import compat
55 from ganeti import constants
56 from ganeti import ht
57 from ganeti import pathutils
58 from ganeti import vcluster
59
60 import colors
61 import qa_config
62 import qa_error
63
64 from qa_logging import FormatInfo
65
66
67 _MULTIPLEXERS = {}
68
69 #: Unique ID per QA run
70 _RUN_UUID = utils.NewUUID()
71
72 #: Path to the QA query output log file
73 _QA_OUTPUT = pathutils.GetLogFilename("qa-output")
74
75
76 _RETRIES = 3
77
78
79 (INST_DOWN,
80 INST_UP) = range(500, 502)
81
82 (FIRST_ARG,
83 RETURN_VALUE) = range(1000, 1002)
84
85
86 def _RaiseWithInfo(msg, error_desc):
87 """Raises a QA error with the given content, and adds a message if present.
88
89 """
90 if msg:
91 output = "%s: %s" % (msg, error_desc)
92 else:
93 output = error_desc
94 raise qa_error.Error(output)
95
96
97 def AssertIn(item, sequence, msg=None):
98 """Raises an error when item is not in sequence.
99
100 """
101 if item not in sequence:
102 _RaiseWithInfo(msg, "%r not in %r" % (item, sequence))
103
104
105 def AssertNotIn(item, sequence, msg=None):
106 """Raises an error when item is in sequence.
107
108 """
109 if item in sequence:
110 _RaiseWithInfo(msg, "%r in %r" % (item, sequence))
111
112
113 def AssertEqual(first, second, msg=None):
114 """Raises an error when values aren't equal.
115
116 """
117 if not first == second:
118 _RaiseWithInfo(msg, "%r == %r" % (first, second))
119
120
121 def AssertMatch(string, pattern, msg=None):
122 """Raises an error when string doesn't match regexp pattern.
123
124 """
125 if not re.match(pattern, string):
126 _RaiseWithInfo(msg, "%r doesn't match /%r/" % (string, pattern))
127
128
129 def _GetName(entity, fn):
130 """Tries to get name of an entity.
131
132 @type entity: string or dict
133 @param fn: Function retrieving name from entity
134
135 """
136 if isinstance(entity, basestring):
137 result = entity
138 else:
139 result = fn(entity)
140
141 if not ht.TNonEmptyString(result):
142 raise Exception("Invalid name '%s'" % result)
143
144 return result
145
146
147 def _AssertRetCode(rcode, fail, cmdstr, nodename):
148 """Check the return value from a command and possibly raise an exception.
149
150 """
151 if fail and rcode == 0:
152 raise qa_error.Error("Command '%s' on node %s was expected to fail but"
153 " didn't" % (cmdstr, nodename))
154 elif not fail and rcode != 0:
155 raise qa_error.Error("Command '%s' on node %s failed, exit code %s" %
156 (cmdstr, nodename, rcode))
157
158
159 def _PrintCommandOutput(stdout, stderr):
160 """Prints the output of commands, minimizing wasted space.
161
162 @type stdout: string
163 @type stderr: string
164
165 """
166 if stdout:
167 stdout_clean = stdout.rstrip('\n')
168 if stderr:
169 print "Stdout was:\n%s" % stdout_clean
170 else:
171 print stdout_clean
172
173 if stderr:
174 print "Stderr was:"
175 print >> sys.stderr, stderr.rstrip('\n')
176
177
178 def AssertCommand(cmd, fail=False, node=None, log_cmd=True, max_seconds=None):
179 """Checks that a remote command succeeds.
180
181 @param cmd: either a string (the command to execute) or a list (to
182 be converted using L{utils.ShellQuoteArgs} into a string)
183 @type fail: boolean or None
184 @param fail: if the command is expected to fail instead of succeeding,
185 or None if we don't care
186 @param node: if passed, it should be the node on which the command
187 should be executed, instead of the master node (can be either a
188 dict or a string)
189 @param log_cmd: if False, the command won't be logged (simply passed to
190 StartSSH)
191 @type max_seconds: double
192 @param max_seconds: fail if the command takes more than C{max_seconds}
193 seconds
194 @return: the return code, stdout and stderr of the command
195 @raise qa_error.Error: if the command fails when it shouldn't or vice versa
196
197 """
198 if node is None:
199 node = qa_config.GetMasterNode()
200
201 nodename = _GetName(node, operator.attrgetter("primary"))
202
203 if isinstance(cmd, basestring):
204 cmdstr = cmd
205 else:
206 cmdstr = utils.ShellQuoteArgs(cmd)
207
208 start = datetime.datetime.now()
209 popen = StartSSH(nodename, cmdstr, log_cmd=log_cmd)
210 # Run the command
211 stdout, stderr = popen.communicate()
212 rcode = popen.returncode
213 duration_seconds = TimedeltaToTotalSeconds(datetime.datetime.now() - start)
214
215 try:
216 if fail is not None:
217 _AssertRetCode(rcode, fail, cmdstr, nodename)
218 finally:
219 if log_cmd:
220 _PrintCommandOutput(stdout, stderr)
221
222 if max_seconds is not None:
223 if duration_seconds > max_seconds:
224 raise qa_error.Error(
225 "Cmd '%s' took %f seconds, maximum of %f was exceeded" %
226 (cmdstr, duration_seconds, max_seconds))
227
228 return rcode, stdout, stderr
229
230
231 def stdout_of(cmd):
232 """Small helper to run a stdout_of.
233 Makes sure the stdout_of returns exit code 0.
234
235 @type cmd: list of strings
236 @param cmd: the stdout_of to run
237
238 @return: Captured, stripped stdout.
239 """
240 _, out, _ = AssertCommand(cmd)
241 return out.strip()
242
243
244 def AssertRedirectedCommand(cmd, fail=False, node=None, log_cmd=True):
245 """Executes a command with redirected output.
246
247 The log will go to the qa-output log file in the ganeti log
248 directory on the node where the command is executed. The fail and
249 node parameters are passed unchanged to AssertCommand.
250
251 @param cmd: the command to be executed, as a list; a string is not
252 supported
253
254 """
255 if not isinstance(cmd, list):
256 raise qa_error.Error("Non-list passed to AssertRedirectedCommand")
257 ofile = utils.ShellQuote(_QA_OUTPUT)
258 cmdstr = utils.ShellQuoteArgs(cmd)
259 AssertCommand("echo ---- $(date) %s ---- >> %s" % (cmdstr, ofile),
260 fail=False, node=node, log_cmd=False)
261 return AssertCommand(cmdstr + " >> %s" % ofile,
262 fail=fail, node=node, log_cmd=log_cmd)
263
264
265 def GetSSHCommand(node, cmd, strict=True, opts=None, tty=False,
266 use_multiplexer=True):
267 """Builds SSH command to be executed.
268
269 @type node: string
270 @param node: node the command should run on
271 @type cmd: string
272 @param cmd: command to be executed in the node; if None or empty
273 string, no command will be executed
274 @type strict: boolean
275 @param strict: whether to enable strict host key checking
276 @type opts: list
277 @param opts: list of additional options
278 @type tty: boolean or None
279 @param tty: if we should use tty; if None, will be auto-detected
280 @type use_multiplexer: boolean
281 @param use_multiplexer: if the multiplexer for the node should be used
282
283 """
284 args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-lroot"]
285
286 if tty is None:
287 tty = sys.stdout.isatty()
288
289 if tty:
290 args.append("-t")
291
292 args.append("-oStrictHostKeyChecking=%s" % ("yes" if strict else "no", ))
293 args.append("-oClearAllForwardings=yes")
294 args.append("-oForwardAgent=yes")
295 if opts:
296 args.extend(opts)
297 if node in _MULTIPLEXERS and use_multiplexer:
298 spath = _MULTIPLEXERS[node][0]
299 args.append("-oControlPath=%s" % spath)
300 args.append("-oControlMaster=no")
301
302 (vcluster_master, vcluster_basedir) = \
303 qa_config.GetVclusterSettings()
304
305 if vcluster_master:
306 args.append(vcluster_master)
307 args.append("%s/%s/cmd" % (vcluster_basedir, node))
308
309 if cmd:
310 # For virtual clusters the whole command must be wrapped using the "cmd"
311 # script, as that script sets a number of environment variables. If the
312 # command contains shell meta characters the whole command needs to be
313 # quoted.
314 args.append(utils.ShellQuote(cmd))
315 else:
316 args.append(node)
317
318 if cmd:
319 args.append(cmd)
320
321 return args
322
323
324 def StartLocalCommand(cmd, _nolog_opts=False, log_cmd=True, **kwargs):
325 """Starts a local command.
326
327 """
328 if log_cmd:
329 if _nolog_opts:
330 pcmd = [i for i in cmd if not i.startswith("-")]
331 else:
332 pcmd = cmd
333 print "%s %s" % (colors.colorize("Command:", colors.CYAN),
334 utils.ShellQuoteArgs(pcmd))
335 return subprocess.Popen(cmd, shell=False, **kwargs)
336
337
338 def StartSSH(node, cmd, strict=True, log_cmd=True):
339 """Starts SSH.
340
341 """
342 return StartLocalCommand(GetSSHCommand(node, cmd, strict=strict),
343 _nolog_opts=True, log_cmd=log_cmd,
344 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
345
346
347 def StartMultiplexer(node):
348 """Starts a multiplexer command.
349
350 @param node: the node for which to open the multiplexer
351
352 """
353 if node in _MULTIPLEXERS:
354 return
355
356 # Note: yes, we only need mktemp, since we'll remove the file anyway
357 sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
358 utils.RemoveFile(sname)
359 opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
360 print "Created socket at %s" % sname
361 child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
362 _MULTIPLEXERS[node] = (sname, child)
363
364
365 def CloseMultiplexers():
366 """Closes all current multiplexers and cleans up.
367
368 """
369 for node in _MULTIPLEXERS.keys():
370 (sname, child) = _MULTIPLEXERS.pop(node)
371 utils.KillProcess(child.pid, timeout=10, waitpid=True)
372 utils.RemoveFile(sname)
373
374
375 def _GetCommandStdout(proc):
376 """Extract the stored standard error, print it and return it.
377
378 """
379 out = proc.stdout.read()
380 sys.stdout.write(out)
381 return out
382
383
384 def _NoTimeout(state):
385 """False iff the command timed out."""
386 rcode, out = state
387
388 return rcode == 0 or not ('TimeoutError' in out or 'timed out' in out)
389
390
391 def GetCommandOutput(node, cmd, tty=False, use_multiplexer=True, log_cmd=True,
392 fail=False):
393 """Returns the output of a command executed on the given node.
394
395 @type node: string
396 @param node: node the command should run on
397 @type cmd: string
398 @param cmd: command to be executed in the node (cannot be empty or None)
399 @type tty: bool or None
400 @param tty: if we should use tty; if None, it will be auto-detected
401 @type use_multiplexer: bool
402 @param use_multiplexer: if the SSH multiplexer provided by the QA should be
403 used or not
404 @type log_cmd: bool
405 @param log_cmd: if the command should be logged
406 @type fail: bool
407 @param fail: whether the command is expected to fail
408 """
409 assert cmd
410
411 def CallCommand():
412 command = GetSSHCommand(node, cmd, tty=tty,
413 use_multiplexer=use_multiplexer)
414 p = StartLocalCommand(command, stdout=subprocess.PIPE, log_cmd=log_cmd)
415 rcode = p.wait()
416 out = _GetCommandStdout(p)
417 return rcode, out
418
419 # TODO: make retries configurable
420 rcode, out = utils.CountRetry(_NoTimeout, CallCommand, _RETRIES)
421 _AssertRetCode(rcode, fail, cmd, node)
422 return out
423
424
425 def GetObjectInfo(infocmd):
426 """Get and parse information about a Ganeti object.
427
428 @type infocmd: list of strings
429 @param infocmd: command to be executed, e.g. ["gnt-cluster", "info"]
430 @return: the information parsed, appropriately stored in dictionaries,
431 lists...
432
433 """
434 master = qa_config.GetMasterNode()
435 cmdline = utils.ShellQuoteArgs(infocmd)
436 info_out = GetCommandOutput(master.primary, cmdline)
437 return yaml.load(info_out)
438
439
440 def UploadFile(node, src):
441 """Uploads a file to a node and returns the filename.
442
443 Caller needs to remove the returned file on the node when it's not needed
444 anymore.
445
446 """
447 # Make sure nobody else has access to it while preserving local permissions
448 mode = os.stat(src).st_mode & 0700
449
450 cmd = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
451 'chmod %o "${tmp}" && '
452 '[[ -f "${tmp}" ]] && '
453 'cat > "${tmp}" && '
454 'echo "${tmp}"') % mode
455
456 f = open(src, "r")
457 try:
458 p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
459 stdout=subprocess.PIPE)
460 AssertEqual(p.wait(), 0)
461
462 # Return temporary filename
463 return _GetCommandStdout(p).strip()
464 finally:
465 f.close()
466
467
468 def UploadData(node, data, mode=0600, filename=None):
469 """Uploads data to a node and returns the filename.
470
471 Caller needs to remove the returned file on the node when it's not needed
472 anymore.
473
474 """
475 if filename:
476 tmp = "tmp=%s" % utils.ShellQuote(filename)
477 else:
478 tmp = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && '
479 'chmod %o "${tmp}"') % mode
480 cmd = ("%s && "
481 "[[ -f \"${tmp}\" ]] && "
482 "cat > \"${tmp}\" && "
483 "echo \"${tmp}\"") % tmp
484
485 p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False,
486 stdin=subprocess.PIPE, stdout=subprocess.PIPE)
487 p.stdin.write(data)
488 p.stdin.close()
489 AssertEqual(p.wait(), 0)
490
491 # Return temporary filename
492 return _GetCommandStdout(p).strip()
493
494
495 def BackupFile(node, path):
496 """Creates a backup of a file on the node and returns the filename.
497
498 Caller needs to remove the returned file on the node when it's not needed
499 anymore.
500
501 """
502 vpath = MakeNodePath(node, path)
503
504 cmd = ("tmp=$(mktemp .gnt.XXXXXX --tmpdir=$(dirname %s)) && "
505 "[[ -f \"$tmp\" ]] && "
506 "cp %s $tmp && "
507 "echo $tmp") % (utils.ShellQuote(vpath), utils.ShellQuote(vpath))
508
509 # Return temporary filename
510 result = GetCommandOutput(node, cmd).strip()
511
512 print "Backup filename: %s" % result
513
514 return result
515
516
517 @contextlib.contextmanager
518 def CheckFileUnmodified(node, filename):
519 """Checks that the content of a given file remains the same after running a
520 wrapped code.
521
522 @type node: string
523 @param node: node the command should run on
524 @type filename: string
525 @param filename: absolute filename to check
526
527 """
528 cmd = utils.ShellQuoteArgs(["sha1sum", MakeNodePath(node, filename)])
529
530 def Read():
531 return GetCommandOutput(node, cmd).strip()
532
533 # read the configuration
534 before = Read()
535 yield
536 # check that the configuration hasn't changed
537 after = Read()
538 if before != after:
539 raise qa_error.Error("File '%s' has changed unexpectedly on node %s"
540 " during the last operation" % (filename, node))
541
542
543 def ResolveInstanceName(instance):
544 """Gets the full name of an instance.
545
546 @type instance: string
547 @param instance: Instance name
548
549 """
550 info = GetObjectInfo(["gnt-instance", "info", instance])
551 return info[0]["Instance name"]
552
553
554 def ResolveNodeName(node):
555 """Gets the full name of a node.
556
557 """
558 info = GetObjectInfo(["gnt-node", "info", node.primary])
559 return info[0]["Node name"]
560
561
562 def GetNodeInstances(node, secondaries=False):
563 """Gets a list of instances on a node.
564
565 """
566 master = qa_config.GetMasterNode()
567 node_name = ResolveNodeName(node)
568
569 # Get list of all instances
570 cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
571 "--output=name,pnode,snodes"]
572 output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))
573
574 instances = []
575 for line in output.splitlines():
576 (name, pnode, snodes) = line.split(":", 2)
577 if ((not secondaries and pnode == node_name) or
578 (secondaries and node_name in snodes.split(","))):
579 instances.append(name)
580
581 return instances
582
583
584 def _SelectQueryFields(rnd, fields):
585 """Generates a list of fields for query tests.
586
587 """
588 # Create copy for shuffling
589 fields = list(fields)
590 rnd.shuffle(fields)
591
592 # Check all fields
593 yield fields
594 yield sorted(fields)
595
596 # Duplicate fields
597 yield fields + fields
598
599 # Check small groups of fields
600 while fields:
601 yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields]
602
603
604 def _List(listcmd, fields, names):
605 """Runs a list command.
606
607 """
608 master = qa_config.GetMasterNode()
609
610 cmd = [listcmd, "list", "--separator=|", "--no-headers",
611 "--output", ",".join(fields)]
612
613 if names:
614 cmd.extend(names)
615
616 return GetCommandOutput(master.primary,
617 utils.ShellQuoteArgs(cmd)).splitlines()
618
619
620 def GenericQueryTest(cmd, fields, namefield="name", test_unknown=True):
621 """Runs a number of tests on query commands.
622
623 @param cmd: Command name
624 @param fields: List of field names
625
626 """
627 rnd = random.Random(hash(cmd))
628
629 fields = list(fields)
630 rnd.shuffle(fields)
631
632 # Test a number of field combinations
633 for testfields in _SelectQueryFields(rnd, fields):
634 AssertRedirectedCommand([cmd, "list", "--output", ",".join(testfields)])
635
636 if namefield is not None:
637 namelist_fn = compat.partial(_List, cmd, [namefield])
638
639 # When no names were requested, the list must be sorted
640 names = namelist_fn(None)
641 AssertEqual(names, utils.NiceSort(names))
642
643 # When requesting specific names, the order must be kept
644 revnames = list(reversed(names))
645 AssertEqual(namelist_fn(revnames), revnames)
646
647 randnames = list(names)
648 rnd.shuffle(randnames)
649 AssertEqual(namelist_fn(randnames), randnames)
650
651 if test_unknown:
652 # Listing unknown items must fail
653 AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"],
654 fail=True)
655
656 # Check exit code for listing unknown field
657 rcode, _, _ = AssertRedirectedCommand([cmd, "list",
658 "--output=field/does/not/exist"],
659 fail=True)
660 AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
661
662
663 def GenericQueryFieldsTest(cmd, fields):
664 master = qa_config.GetMasterNode()
665
666 # Listing fields
667 AssertRedirectedCommand([cmd, "list-fields"])
668 AssertRedirectedCommand([cmd, "list-fields"] + fields)
669
670 # Check listed fields (all, must be sorted)
671 realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
672 output = GetCommandOutput(master.primary,
673 utils.ShellQuoteArgs(realcmd)).splitlines()
674 AssertEqual([line.split("|", 1)[0] for line in output],
675 utils.NiceSort(fields))
676
677 # Check exit code for listing unknown field
678 rcode, _, _ = AssertCommand([cmd, "list-fields", "field/does/not/exist"],
679 fail=True)
680 AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD)
681
682
683 def AddToEtcHosts(hostnames):
684 """Adds hostnames to /etc/hosts.
685
686 @param hostnames: List of hostnames first used A records, all other CNAMEs
687
688 """
689 master = qa_config.GetMasterNode()
690 tmp_hosts = UploadData(master.primary, "", mode=0644)
691
692 data = []
693 for localhost in ("::1", "127.0.0.1"):
694 data.append("%s %s" % (localhost, " ".join(hostnames)))
695
696 try:
697 AssertCommand("{ cat %s && echo -e '%s'; } > %s && mv %s %s" %
698 (utils.ShellQuote(pathutils.ETC_HOSTS),
699 "\\n".join(data),
700 utils.ShellQuote(tmp_hosts),
701 utils.ShellQuote(tmp_hosts),
702 utils.ShellQuote(pathutils.ETC_HOSTS)))
703 except Exception:
704 AssertCommand(["rm", "-f", tmp_hosts])
705 raise
706
707
708 def RemoveFromEtcHosts(hostnames):
709 """Remove hostnames from /etc/hosts.
710
711 @param hostnames: List of hostnames first used A records, all other CNAMEs
712
713 """
714 master = qa_config.GetMasterNode()
715 tmp_hosts = UploadData(master.primary, "", mode=0644)
716 quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
717
718 sed_data = " ".join(hostnames)
719 try:
720 AssertCommand((r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
721 r" && mv %s %s") %
722 (sed_data, utils.ShellQuote(pathutils.ETC_HOSTS),
723 quoted_tmp_hosts, quoted_tmp_hosts,
724 utils.ShellQuote(pathutils.ETC_HOSTS)))
725 except Exception:
726 AssertCommand(["rm", "-f", tmp_hosts])
727 raise
728
729
730 def RunInstanceCheck(instance, running):
731 """Check if instance is running or not.
732
733 """
734 instance_name = _GetName(instance, operator.attrgetter("name"))
735
736 script = qa_config.GetInstanceCheckScript()
737 if not script:
738 return
739
740 master_node = qa_config.GetMasterNode()
741
742 # Build command to connect to master node
743 master_ssh = GetSSHCommand(master_node.primary, "--")
744
745 if running:
746 running_shellval = "1"
747 running_text = ""
748 else:
749 running_shellval = ""
750 running_text = "not "
751
752 print FormatInfo("Checking if instance '%s' is %srunning" %
753 (instance_name, running_text))
754
755 args = [script, instance_name]
756 env = {
757 "PATH": constants.HOOKS_PATH,
758 "RUN_UUID": _RUN_UUID,
759 "MASTER_SSH": utils.ShellQuoteArgs(master_ssh),
760 "INSTANCE_NAME": instance_name,
761 "INSTANCE_RUNNING": running_shellval,
762 }
763
764 result = os.spawnve(os.P_WAIT, script, args, env)
765 if result != 0:
766 raise qa_error.Error("Instance check failed with result %s" % result)
767
768
769 def _InstanceCheckInner(expected, instarg, args, result):
770 """Helper function used by L{InstanceCheck}.
771
772 """
773 if instarg == FIRST_ARG:
774 instance = args[0]
775 elif instarg == RETURN_VALUE:
776 instance = result
777 else:
778 raise Exception("Invalid value '%s' for instance argument" % instarg)
779
780 if expected in (INST_DOWN, INST_UP):
781 RunInstanceCheck(instance, (expected == INST_UP))
782 elif expected is not None:
783 raise Exception("Invalid value '%s'" % expected)
784
785
786 def InstanceCheck(before, after, instarg):
787 """Decorator to check instance status before and after test.
788
789 @param before: L{INST_DOWN} if instance must be stopped before test,
790 L{INST_UP} if instance must be running before test, L{None} to not check.
791 @param after: L{INST_DOWN} if instance must be stopped after test,
792 L{INST_UP} if instance must be running after test, L{None} to not check.
793 @param instarg: L{FIRST_ARG} to use first argument to test as instance (a
794 dictionary), L{RETURN_VALUE} to use return value (disallows pre-checks)
795
796 """
797 def decorator(fn):
798 @functools.wraps(fn)
799 def wrapper(*args, **kwargs):
800 _InstanceCheckInner(before, instarg, args, NotImplemented)
801
802 result = fn(*args, **kwargs)
803
804 _InstanceCheckInner(after, instarg, args, result)
805
806 return result
807 return wrapper
808 return decorator
809
810
811 def GetNonexistentGroups(count):
812 """Gets group names which shouldn't exist on the cluster.
813
814 @param count: Number of groups to get
815 @rtype: integer
816
817 """
818 return GetNonexistentEntityNames(count, "groups", "group")
819
820
821 def GetNonexistentEntityNames(count, name_config, name_prefix):
822 """Gets entity names which shouldn't exist on the cluster.
823
824 The actualy names can refer to arbitrary entities (for example
825 groups, networks).
826
827 @param count: Number of names to get
828 @rtype: integer
829 @param name_config: name of the leaf in the config containing
830 this entity's configuration, including a 'inexistent-'
831 element
832 @rtype: string
833 @param name_prefix: prefix of the entity's names, used to compose
834 the default values; for example for groups, the prefix is
835 'group' and the generated names are then group1, group2, ...
836 @rtype: string
837
838 """
839 entities = qa_config.get(name_config, {})
840
841 default = [name_prefix + str(i) for i in range(count)]
842 assert count <= len(default)
843
844 name_config_inexistent = "inexistent-" + name_config
845 candidates = entities.get(name_config_inexistent, default)[:count]
846
847 if len(candidates) < count:
848 raise Exception("At least %s non-existent %s are needed" %
849 (count, name_config))
850
851 return candidates
852
853
854 def MakeNodePath(node, path):
855 """Builds an absolute path for a virtual node.
856
857 @type node: string or L{qa_config._QaNode}
858 @param node: Node
859 @type path: string
860 @param path: Path without node-specific prefix
861
862 """
863 (_, basedir) = qa_config.GetVclusterSettings()
864
865 if isinstance(node, basestring):
866 name = node
867 else:
868 name = node.primary
869
870 if basedir:
871 assert path.startswith("/")
872 return "%s%s" % (vcluster.MakeNodeRoot(basedir, name), path)
873 else:
874 return path
875
876
877 def _GetParameterOptions(specs):
878 """Helper to build policy options."""
879 values = ["%s=%s" % (par, val)
880 for (par, val) in specs.items()]
881 return ",".join(values)
882
883
884 def TestSetISpecs(new_specs=None, diff_specs=None, get_policy_fn=None,
885 build_cmd_fn=None, fail=False, old_values=None):
886 """Change instance specs for an object.
887
888 At most one of new_specs or diff_specs can be specified.
889
890 @type new_specs: dict
891 @param new_specs: new complete specs, in the same format returned by
892 L{ParseIPolicy}.
893 @type diff_specs: dict
894 @param diff_specs: partial specs, it can be an incomplete specifications, but
895 if min/max specs are specified, their number must match the number of the
896 existing specs
897 @type get_policy_fn: function
898 @param get_policy_fn: function that returns the current policy as in
899 L{ParseIPolicy}
900 @type build_cmd_fn: function
901 @param build_cmd_fn: function that return the full command line from the
902 options alone
903 @type fail: bool
904 @param fail: if the change is expected to fail
905 @type old_values: tuple
906 @param old_values: (old_policy, old_specs), as returned by
907 L{ParseIPolicy}
908 @return: same as L{ParseIPolicy}
909
910 """
911 assert get_policy_fn is not None
912 assert build_cmd_fn is not None
913 assert new_specs is None or diff_specs is None
914
915 if old_values:
916 (old_policy, old_specs) = old_values
917 else:
918 (old_policy, old_specs) = get_policy_fn()
919
920 if diff_specs:
921 new_specs = copy.deepcopy(old_specs)
922 if constants.ISPECS_MINMAX in diff_specs:
923 AssertEqual(len(new_specs[constants.ISPECS_MINMAX]),
924 len(diff_specs[constants.ISPECS_MINMAX]))
925 for (new_minmax, diff_minmax) in zip(new_specs[constants.ISPECS_MINMAX],
926 diff_specs[constants.ISPECS_MINMAX]):
927 for (key, parvals) in diff_minmax.items():
928 for (par, val) in parvals.items():
929 new_minmax[key][par] = val
930 for (par, val) in diff_specs.get(constants.ISPECS_STD, {}).items():
931 new_specs[constants.ISPECS_STD][par] = val
932
933 if new_specs:
934 cmd = []
935 if (diff_specs is None or constants.ISPECS_MINMAX in diff_specs):
936 minmax_opt_items = []
937 for minmax in new_specs[constants.ISPECS_MINMAX]:
938 minmax_opts = []
939 for key in ["min", "max"]:
940 keyopt = _GetParameterOptions(minmax[key])
941 minmax_opts.append("%s:%s" % (key, keyopt))
942 minmax_opt_items.append("/".join(minmax_opts))
943 cmd.extend([
944 "--ipolicy-bounds-specs",
945 "//".join(minmax_opt_items)
946 ])
947 if diff_specs is None:
948 std_source = new_specs
949 else:
950 std_source = diff_specs
951 std_opt = _GetParameterOptions(std_source.get("std", {}))
952 if std_opt:
953 cmd.extend(["--ipolicy-std-specs", std_opt])
954 AssertCommand(build_cmd_fn(cmd), fail=fail)
955
956 # Check the new state
957 (eff_policy, eff_specs) = get_policy_fn()
958 AssertEqual(eff_policy, old_policy)
959 if fail:
960 AssertEqual(eff_specs, old_specs)
961 else:
962 AssertEqual(eff_specs, new_specs)
963
964 else:
965 (eff_policy, eff_specs) = (old_policy, old_specs)
966
967 return (eff_policy, eff_specs)
968
969
970 def ParseIPolicy(policy):
971 """Parse and split instance an instance policy.
972
973 @type policy: dict
974 @param policy: policy, as returned by L{GetObjectInfo}
975 @rtype: tuple
976 @return: (policy, specs), where:
977 - policy is a dictionary of the policy values, instance specs excluded
978 - specs is a dictionary containing only the specs, using the internal
979 format (see L{constants.IPOLICY_DEFAULTS} for an example)
980
981 """
982 ret_specs = {}
983 ret_policy = {}
984 for (key, val) in policy.items():
985 if key == "bounds specs":
986 ret_specs[constants.ISPECS_MINMAX] = []
987 for minmax in val:
988 ret_minmax = {}
989 for key in minmax:
990 keyparts = key.split("/", 1)
991 assert len(keyparts) > 1
992 ret_minmax[keyparts[0]] = minmax[key]
993 ret_specs[constants.ISPECS_MINMAX].append(ret_minmax)
994 elif key == constants.ISPECS_STD:
995 ret_specs[key] = val
996 else:
997 ret_policy[key] = val
998 return (ret_policy, ret_specs)
999
1000
1001 def UsesIPv6Connection(host, port):
1002 """Returns True if the connection to a given host/port could go through IPv6.
1003
1004 """
1005 return any(t[0] == socket.AF_INET6 for t in socket.getaddrinfo(host, port))
1006
1007
1008 def TimedeltaToTotalSeconds(td):
1009 """Returns the total seconds in a C{datetime.timedelta} object.
1010
1011 This performs the same task as the C{datetime.timedelta.total_seconds()}
1012 method which is present in Python 2.7 onwards.
1013
1014 @type td: datetime.timedelta
1015 @param td: timedelta object to convert
1016 @rtype float
1017 @return: total seconds in the timedelta object
1018
1019 """
1020 return ((td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10 ** 6) /
1021 10 ** 6)