QA: reload rapi cert after renew crypto
[ganeti-github.git] / qa / ganeti-qa.py
1 #!/usr/bin/python -u
2 #
3
4 # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are
9 # met:
10 #
11 # 1. Redistributions of source code must retain the above copyright notice,
12 # this list of conditions and the following disclaimer.
13 #
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 """Script for doing QA on Ganeti.
32
33 """
34
35 # pylint: disable=C0103
36 # due to invalid name
37
38 import copy
39 import datetime
40 import optparse
41 import sys
42
43 import colors
44 import qa_cluster
45 import qa_config
46 import qa_daemon
47 import qa_env
48 import qa_error
49 import qa_group
50 import qa_instance
51 import qa_iptables
52 import qa_monitoring
53 import qa_network
54 import qa_node
55 import qa_os
56 import qa_performance
57 import qa_job
58 import qa_rapi
59 import qa_tags
60 import qa_utils
61
62 from ganeti import utils
63 from ganeti import rapi # pylint: disable=W0611
64 from ganeti import constants
65 from ganeti import netutils
66
67 import ganeti.rapi.client # pylint: disable=W0611
68 from ganeti.rapi.client import UsesRapiClient
69
70
71 def _FormatHeader(line, end=72, mark="-", color=None):
72 """Fill a line up to the end column.
73
74 """
75 line = (mark * 4) + " " + line + " "
76 line += "-" * (end - len(line))
77 line = line.rstrip()
78 line = colors.colorize(line, color=color)
79 return line
80
81
82 def _DescriptionOf(fn):
83 """Computes the description of an item.
84
85 """
86 if fn.__doc__:
87 desc = fn.__doc__.splitlines()[0].strip()
88 desc = desc.rstrip(".")
89 if fn.__name__:
90 desc = "[" + fn.__name__ + "] " + desc
91 else:
92 desc = "%r" % fn
93
94 return desc
95
96
97 def RunTest(fn, *args, **kwargs):
98 """Runs a test after printing a header.
99
100 """
101
102 tstart = datetime.datetime.now()
103
104 desc = _DescriptionOf(fn)
105
106 print
107 print _FormatHeader("%s start %s" % (tstart, desc),
108 color=colors.YELLOW, mark="<")
109
110 try:
111 retval = fn(*args, **kwargs)
112 print _FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN)
113 return retval
114 except Exception, e:
115 print _FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED)
116 raise
117 finally:
118 tstop = datetime.datetime.now()
119 tdelta = tstop - tstart
120 print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc),
121 color=colors.MAGENTA, mark=">")
122
123
124 def ReportTestSkip(desc, testnames):
125 """Reports that tests have been skipped.
126
127 @type desc: string
128 @param desc: string
129 @type testnames: string or list of string
130 @param testnames: either a single test name in the configuration
131 file, or a list of testnames (which will be AND-ed together)
132
133 """
134 tstart = datetime.datetime.now()
135 # TODO: Formatting test names when non-string names are involved
136 print _FormatHeader("%s skipping %s, test(s) %s disabled" %
137 (tstart, desc, testnames),
138 color=colors.BLUE, mark="*")
139
140
141 def RunTestIf(testnames, fn, *args, **kwargs):
142 """Runs a test conditionally.
143
144 @param testnames: either a single test name in the configuration
145 file, or a list of testnames (which will be AND-ed together)
146
147 """
148 if qa_config.TestEnabled(testnames):
149 RunTest(fn, *args, **kwargs)
150 else:
151 desc = _DescriptionOf(fn)
152 ReportTestSkip(desc, testnames)
153
154
155 def RunTestBlock(fn, *args, **kwargs):
156 """Runs a block of tests after printing a header.
157
158 """
159 tstart = datetime.datetime.now()
160
161 desc = _DescriptionOf(fn)
162
163 print
164 print _FormatHeader("BLOCK %s start %s" % (tstart, desc),
165 color=[colors.YELLOW, colors.BOLD], mark="v")
166
167 try:
168 return fn(*args, **kwargs)
169 except Exception, e:
170 print _FormatHeader("BLOCK FAILED %s: %s" % (desc, e),
171 color=[colors.RED, colors.BOLD])
172 raise
173 finally:
174 tstop = datetime.datetime.now()
175 tdelta = tstop - tstart
176 print _FormatHeader("BLOCK %s time=%s %s" % (tstop, tdelta, desc),
177 color=[colors.MAGENTA, colors.BOLD], mark="^")
178
179
180 def RunEnvTests():
181 """Run several environment tests.
182
183 """
184 RunTestIf("env", qa_env.TestSshConnection)
185 RunTestIf("env", qa_env.TestIcmpPing)
186 RunTestIf("env", qa_env.TestGanetiCommands)
187
188
189 def SetupCluster(rapi_user):
190 """Initializes the cluster.
191
192 @param rapi_user: Login user for RAPI
193 @return: Login secret for RAPI
194
195 """
196 rapi_secret = utils.GenerateSecret()
197 RunTestIf("create-cluster", qa_cluster.TestClusterInit,
198 rapi_user, rapi_secret)
199 if not qa_config.TestEnabled("create-cluster"):
200 # If the cluster is already in place, we assume that exclusive-storage is
201 # already set according to the configuration
202 qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
203 if qa_rapi.Enabled():
204 # To support RAPI on an existing cluster we have to find out the secret
205 rapi_secret = qa_rapi.LookupRapiSecret(rapi_user)
206
207 qa_group.ConfigureGroups()
208
209 # Test on empty cluster
210 RunTestIf("node-list", qa_node.TestNodeList)
211 RunTestIf("instance-list", qa_instance.TestInstanceList)
212 RunTestIf("job-list", qa_job.TestJobList)
213
214 RunTestIf("create-cluster", qa_node.TestNodeAddAll)
215 if not qa_config.TestEnabled("create-cluster"):
216 # consider the nodes are already there
217 qa_node.MarkNodeAddedAll()
218
219 RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
220 RunTestIf("test-jobqueue", qa_job.TestJobCancellation)
221
222 # enable the watcher (unconditionally)
223 RunTest(qa_daemon.TestResumeWatcher)
224
225 RunTestIf("node-list", qa_node.TestNodeList)
226
227 # Test listing fields
228 RunTestIf("node-list", qa_node.TestNodeListFields)
229 RunTestIf("instance-list", qa_instance.TestInstanceListFields)
230 RunTestIf("job-list", qa_job.TestJobListFields)
231 RunTestIf("instance-export", qa_instance.TestBackupListFields)
232
233 RunTestIf("node-info", qa_node.TestNodeInfo)
234
235 return rapi_secret
236
237
238 def RunClusterTests(rapi_user=None, rapi_secret=None):
239 """Runs tests related to gnt-cluster.
240
241 @type rapi_user: string
242 @param rapi_user: name of the rapi user
243 @type rapi_secret: string
244 @param rapi_secret: the rapi secret
245
246 """
247 for test, fn in [
248 ("create-cluster", qa_cluster.TestClusterInitDisk),
249 ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto)
250 ]:
251 RunTestIf(test, fn)
252
253 # Since renew-crypto replaces the RAPI cert, reload it.
254 if qa_rapi.Enabled():
255 if not rapi_user:
256 raise qa_error.Error("No RAPI user given.")
257 if not rapi_secret:
258 raise qa_error.Error("No RAPI secret given.")
259 qa_rapi.Setup(rapi_user, rapi_secret)
260
261 for test, fn in [
262 ("cluster-verify", qa_cluster.TestClusterVerify),
263 ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
264 # TODO: add more cluster modify tests
265 ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
266 ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
267 ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
268 ("cluster-modify", qa_cluster.TestClusterModifyBe),
269 ("cluster-modify", qa_cluster.TestClusterModifyDisk),
270 ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
271 ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir),
272 ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir),
273 ("cluster-modify", qa_cluster.TestClusterModifyInstallImage),
274 ("cluster-modify", qa_cluster.TestClusterModifyUserShutdown),
275 ("cluster-rename", qa_cluster.TestClusterRename),
276 ("cluster-info", qa_cluster.TestClusterVersion),
277 ("cluster-info", qa_cluster.TestClusterInfo),
278 ("cluster-info", qa_cluster.TestClusterGetmaster),
279 ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
280 (["cluster-copyfile", qa_config.NoVirtualCluster],
281 qa_cluster.TestClusterCopyfile),
282 ("cluster-command", qa_cluster.TestClusterCommand),
283 ("cluster-burnin", qa_cluster.TestClusterBurnin),
284 ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
285 ("cluster-master-failover",
286 qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
287 (["cluster-oob", qa_config.NoVirtualCluster],
288 qa_cluster.TestClusterOob),
289 ("cluster-instance-communication", qa_cluster.TestInstanceCommunication),
290 (qa_rapi.Enabled, qa_rapi.TestVersion),
291 (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
292 (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
293 ]:
294 RunTestIf(test, fn)
295
296
297 def RunRepairDiskSizes():
298 """Run the repair disk-sizes test.
299
300 """
301 RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
302
303
304 def RunOsTests():
305 """Runs all tests related to gnt-os.
306
307 """
308 os_enabled = ["os", qa_config.NoVirtualCluster]
309
310 if qa_config.TestEnabled(qa_rapi.Enabled):
311 rapi_getos = qa_rapi.GetOperatingSystems
312 else:
313 rapi_getos = None
314
315 for fn in [
316 qa_os.TestOsList,
317 qa_os.TestOsDiagnose,
318 ]:
319 RunTestIf(os_enabled, fn)
320
321 for fn in [
322 qa_os.TestOsValid,
323 qa_os.TestOsInvalid,
324 qa_os.TestOsPartiallyValid,
325 ]:
326 RunTestIf(os_enabled, fn, rapi_getos)
327
328 for fn in [
329 qa_os.TestOsModifyValid,
330 qa_os.TestOsModifyInvalid,
331 qa_os.TestOsStatesNonExisting,
332 ]:
333 RunTestIf(os_enabled, fn)
334
335
336 def RunCommonInstanceTests(instance, inst_nodes):
337 """Runs a few tests that are common to all disk types.
338
339 """
340 RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
341 RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
342 qa_rapi.TestRapiStoppedInstanceConsole, instance)
343 RunTestIf(["instance-shutdown", "instance-modify"],
344 qa_instance.TestInstanceStoppedModify, instance)
345 RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
346
347 # Test shutdown/start via RAPI
348 RunTestIf(["instance-shutdown", qa_rapi.Enabled],
349 qa_rapi.TestRapiInstanceShutdown, instance)
350 RunTestIf(["instance-shutdown", qa_rapi.Enabled],
351 qa_rapi.TestRapiInstanceStartup, instance)
352
353 RunTestIf("instance-list", qa_instance.TestInstanceList)
354
355 RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
356
357 RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
358 RunTestIf(["instance-modify", qa_rapi.Enabled],
359 qa_rapi.TestRapiInstanceModify, instance)
360
361 RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
362 RunTestIf(["instance-console", qa_rapi.Enabled],
363 qa_rapi.TestRapiInstanceConsole, instance)
364
365 RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
366 instance)
367 DOWN_TESTS = qa_config.Either([
368 "instance-reinstall",
369 "instance-rename",
370 "instance-grow-disk",
371 ])
372
373 # shutdown instance for any 'down' tests
374 RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)
375
376 # now run the 'down' state tests
377 RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
378 RunTestIf(["instance-reinstall", qa_rapi.Enabled],
379 qa_rapi.TestRapiInstanceReinstall, instance)
380
381 if qa_config.TestEnabled("instance-rename"):
382 tgt_instance = qa_config.AcquireInstance()
383 try:
384 rename_source = instance.name
385 rename_target = tgt_instance.name
386 # perform instance rename to the same name
387 RunTest(qa_instance.TestInstanceRenameAndBack,
388 rename_source, rename_source)
389 RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
390 rename_source, rename_source)
391 if rename_target is not None:
392 # perform instance rename to a different name, if we have one configured
393 RunTest(qa_instance.TestInstanceRenameAndBack,
394 rename_source, rename_target)
395 RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
396 rename_source, rename_target)
397 finally:
398 tgt_instance.Release()
399
400 RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)
401
402 # and now start the instance again
403 RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)
404
405 RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
406
407 RunTestIf("tags", qa_tags.TestInstanceTags, instance)
408
409 if instance.disk_template == constants.DT_DRBD8:
410 RunTestIf("cluster-verify",
411 qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes)
412 RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
413
414 RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
415
416 # Lists instances, too
417 RunTestIf("node-list", qa_node.TestNodeList)
418
419 # Some jobs have been run, let's test listing them
420 RunTestIf("job-list", qa_job.TestJobList)
421
422
423 def RunCommonNodeTests():
424 """Run a few common node tests.
425
426 """
427 RunTestIf("node-volumes", qa_node.TestNodeVolumes)
428 RunTestIf("node-storage", qa_node.TestNodeStorage)
429 RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
430
431
432 def RunGroupListTests():
433 """Run tests for listing node groups.
434
435 """
436 RunTestIf("group-list", qa_group.TestGroupList)
437 RunTestIf("group-list", qa_group.TestGroupListFields)
438
439
440 def RunNetworkTests():
441 """Run tests for network management.
442
443 """
444 RunTestIf("network", qa_network.TestNetworkAddRemove)
445 RunTestIf("network", qa_network.TestNetworkConnect)
446 RunTestIf(["network", "tags"], qa_network.TestNetworkTags)
447
448
449 def RunGroupRwTests():
450 """Run tests for adding/removing/renaming groups.
451
452 """
453 RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
454 RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
455 RunTestIf("group-rwops", qa_group.TestGroupModify)
456 RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
457 RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
458 qa_group.GetDefaultGroup())
459
460
461 def RunExportImportTests(instance, inodes):
462 """Tries to export and import the instance.
463
464 @type inodes: list of nodes
465 @param inodes: current nodes of the instance
466
467 """
468 # FIXME: export explicitly bails out on file based storage. other non-lvm
469 # based storage types are untested, though. Also note that import could still
470 # work, but is deeply embedded into the "export" case.
471 if (qa_config.TestEnabled("instance-export") and
472 instance.disk_template not in constants.DTS_FILEBASED):
473 RunTest(qa_instance.TestInstanceExportNoTarget, instance)
474
475 pnode = inodes[0]
476 expnode = qa_config.AcquireNode(exclude=pnode)
477 try:
478 name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
479
480 RunTest(qa_instance.TestBackupList, expnode)
481
482 if qa_config.TestEnabled("instance-import"):
483 newinst = qa_config.AcquireInstance()
484 try:
485 RunTest(qa_instance.TestInstanceImport, newinst, pnode,
486 expnode, name)
487 # Check if starting the instance works
488 RunTest(qa_instance.TestInstanceStartup, newinst)
489 RunTest(qa_instance.TestInstanceRemove, newinst)
490 finally:
491 newinst.Release()
492 finally:
493 expnode.Release()
494
495 # FIXME: inter-cluster-instance-move crashes on file based instances :/
496 # See Issue 414.
497 if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
498 and (instance.disk_template not in constants.DTS_FILEBASED)):
499 newinst = qa_config.AcquireInstance()
500 try:
501 tnode = qa_config.AcquireNode(exclude=inodes)
502 try:
503 RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
504 inodes, tnode)
505 finally:
506 tnode.Release()
507 finally:
508 newinst.Release()
509
510
511 def RunDaemonTests(instance):
512 """Test the ganeti-watcher script.
513
514 """
515 RunTest(qa_daemon.TestPauseWatcher)
516
517 RunTestIf("instance-automatic-restart",
518 qa_daemon.TestInstanceAutomaticRestart, instance)
519 RunTestIf("instance-consecutive-failures",
520 qa_daemon.TestInstanceConsecutiveFailures, instance)
521
522 RunTest(qa_daemon.TestResumeWatcher)
523
524
525 def RunHardwareFailureTests(instance, inodes):
526 """Test cluster internal hardware failure recovery.
527
528 """
529 RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
530 RunTestIf(["instance-failover", qa_rapi.Enabled],
531 qa_rapi.TestRapiInstanceFailover, instance)
532
533 RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
534 RunTestIf(["instance-migrate", qa_rapi.Enabled],
535 qa_rapi.TestRapiInstanceMigrate, instance)
536
537 if qa_config.TestEnabled("instance-replace-disks"):
538 # We just need alternative secondary nodes, hence "- 1"
539 othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
540 try:
541 RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
542 RunTest(qa_instance.TestReplaceDisks,
543 instance, inodes, othernodes)
544 finally:
545 qa_config.ReleaseManyNodes(othernodes)
546 del othernodes
547
548 if qa_config.TestEnabled("instance-recreate-disks"):
549 try:
550 acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
551 othernodes = acquirednodes
552 except qa_error.OutOfNodesError:
553 if len(inodes) > 1:
554 # If the cluster is not big enough, let's reuse some of the nodes, but
555 # with different roles. In this way, we can test a DRBD instance even on
556 # a 3-node cluster.
557 acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
558 othernodes = acquirednodes + inodes[:-1]
559 else:
560 raise
561 try:
562 RunTest(qa_instance.TestRecreateDisks,
563 instance, inodes, othernodes)
564 finally:
565 qa_config.ReleaseManyNodes(acquirednodes)
566
567 if len(inodes) >= 2:
568 RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
569 RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
570 RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1])
571
572
573 def RunExclusiveStorageTests():
574 """Test exclusive storage."""
575 if not qa_config.TestEnabled("cluster-exclusive-storage"):
576 return
577
578 node = qa_config.AcquireNode()
579 try:
580 old_es = qa_cluster.TestSetExclStorCluster(False)
581 qa_node.TestExclStorSingleNode(node)
582
583 qa_cluster.TestSetExclStorCluster(True)
584 qa_cluster.TestExclStorSharedPv(node)
585
586 if qa_config.TestEnabled("instance-add-plain-disk"):
587 # Make sure that the cluster doesn't have any pre-existing problem
588 qa_cluster.AssertClusterVerify()
589
590 # Create and allocate instances
591 instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
592 try:
593 instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
594 try:
595 # cluster-verify checks that disks are allocated correctly
596 qa_cluster.AssertClusterVerify()
597
598 # Remove instances
599 qa_instance.TestInstanceRemove(instance2)
600 qa_instance.TestInstanceRemove(instance1)
601 finally:
602 instance2.Release()
603 finally:
604 instance1.Release()
605
606 if qa_config.TestEnabled("instance-add-drbd-disk"):
607 snode = qa_config.AcquireNode()
608 try:
609 qa_cluster.TestSetExclStorCluster(False)
610 instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
611 try:
612 qa_cluster.TestSetExclStorCluster(True)
613 exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
614 qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
615 qa_instance.TestInstanceRemove(instance)
616 finally:
617 instance.Release()
618 finally:
619 snode.Release()
620 qa_cluster.TestSetExclStorCluster(old_es)
621 finally:
622 node.Release()
623
624
625 def RunCustomSshPortTests():
626 """Test accessing nodes with custom SSH ports.
627
628 This requires removing nodes, adding them to a new group, and then undoing
629 the change.
630 """
631 if not qa_config.TestEnabled("group-custom-ssh-port"):
632 return
633
634 std_port = netutils.GetDaemonPort(constants.SSH)
635 port = 211
636 master = qa_config.GetMasterNode()
637 with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
638 # Checks if the node(s) could be contacted through IPv6.
639 # If yes, better skip the whole test.
640
641 for node in nodes:
642 if qa_utils.UsesIPv6Connection(node.primary, std_port):
643 print ("Node %s is likely to be reached using IPv6,"
644 "skipping the test" % (node.primary, ))
645 return
646
647 for node in nodes:
648 qa_node.NodeRemove(node)
649 with qa_iptables.RulesContext(nodes) as r:
650 with qa_group.NewGroupCtx() as group:
651 qa_group.ModifyGroupSshPort(r, group, nodes, port)
652
653 for node in nodes:
654 qa_node.NodeAdd(node, group=group)
655
656 # Make sure that the cluster doesn't have any pre-existing problem
657 qa_cluster.AssertClusterVerify()
658
659 # Create and allocate instances
660 instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
661 try:
662 instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
663 try:
664 # cluster-verify checks that disks are allocated correctly
665 qa_cluster.AssertClusterVerify()
666
667 # Remove instances
668 qa_instance.TestInstanceRemove(instance2)
669 qa_instance.TestInstanceRemove(instance1)
670 finally:
671 instance2.Release()
672 finally:
673 instance1.Release()
674
675 for node in nodes:
676 qa_node.NodeRemove(node)
677
678 for node in nodes:
679 qa_node.NodeAdd(node)
680
681 qa_cluster.AssertClusterVerify()
682
683
684 def _BuildSpecDict(par, mn, st, mx):
685 return {
686 constants.ISPECS_MINMAX: [{
687 constants.ISPECS_MIN: {par: mn},
688 constants.ISPECS_MAX: {par: mx},
689 }],
690 constants.ISPECS_STD: {par: st},
691 }
692
693
694 def _BuildDoubleSpecDict(index, par, mn, st, mx):
695 new_spec = {
696 constants.ISPECS_MINMAX: [{}, {}],
697 }
698 if st is not None:
699 new_spec[constants.ISPECS_STD] = {par: st}
700 new_spec[constants.ISPECS_MINMAX][index] = {
701 constants.ISPECS_MIN: {par: mn},
702 constants.ISPECS_MAX: {par: mx},
703 }
704 return new_spec
705
706
707 def TestIPolicyPlainInstance():
708 """Test instance policy interaction with instances"""
709 params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
710 if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
711 print "Template %s not supported" % constants.DT_PLAIN
712 return
713
714 # This test assumes that the group policy is empty
715 (_, old_specs) = qa_cluster.TestClusterSetISpecs()
716 # We also assume to have only one min/max bound
717 assert len(old_specs[constants.ISPECS_MINMAX]) == 1
718 node = qa_config.AcquireNode()
719 try:
720 # Log of policy changes, list of tuples:
721 # (full_change, incremental_change, policy_violated)
722 history = []
723 instance = qa_instance.TestInstanceAddWithPlainDisk([node])
724 try:
725 policyerror = [constants.CV_EINSTANCEPOLICY]
726 for par in params:
727 (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
728 # Some specs must be multiple of 4
729 new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
730 history.append((None, new_spec, True))
731 if iminval > 0:
732 # Some specs must be multiple of 4
733 if iminval >= 4:
734 upper = iminval - 4
735 else:
736 upper = iminval - 1
737 new_spec = _BuildSpecDict(par, 0, upper, upper)
738 history.append((None, new_spec, True))
739 history.append((old_specs, None, False))
740
741 # Test with two instance specs
742 double_specs = copy.deepcopy(old_specs)
743 double_specs[constants.ISPECS_MINMAX] = \
744 double_specs[constants.ISPECS_MINMAX] * 2
745 (par1, par2) = params[0:2]
746 (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1)
747 (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2)
748 old_minmax = old_specs[constants.ISPECS_MINMAX][0]
749 history.extend([
750 (double_specs, None, False),
751 # The first min/max limit is being violated
752 (None,
753 _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4,
754 imaxval1 + 4),
755 False),
756 # Both min/max limits are being violated
757 (None,
758 _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4),
759 True),
760 # The second min/max limit is being violated
761 (None,
762 _BuildDoubleSpecDict(0, par1,
763 old_minmax[constants.ISPECS_MIN][par1],
764 old_specs[constants.ISPECS_STD][par1],
765 old_minmax[constants.ISPECS_MAX][par1]),
766 False),
767 (old_specs, None, False),
768 ])
769
770 # Apply the changes, and check policy violations after each change
771 qa_cluster.AssertClusterVerify()
772 for (new_specs, diff_specs, failed) in history:
773 qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
774 diff_specs=diff_specs)
775 if failed:
776 qa_cluster.AssertClusterVerify(warnings=policyerror)
777 else:
778 qa_cluster.AssertClusterVerify()
779
780 qa_instance.TestInstanceRemove(instance)
781 finally:
782 instance.Release()
783
784 # Now we replay the same policy changes, and we expect that the instance
785 # cannot be created for the cases where we had a policy violation above
786 for (new_specs, diff_specs, failed) in history:
787 qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
788 diff_specs=diff_specs)
789 if failed:
790 qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
791 # Instance creation with no policy violation has been tested already
792 finally:
793 node.Release()
794
795
796 def IsExclusiveStorageInstanceTestEnabled():
797 test_name = "exclusive-storage-instance-tests"
798 if qa_config.TestEnabled(test_name):
799 vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
800 vgscmd = utils.ShellQuoteArgs([
801 "vgs", "--noheadings", "-o", "pv_count", vgname,
802 ])
803 nodes = qa_config.GetConfig()["nodes"]
804 for node in nodes:
805 try:
806 pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
807 except Exception, e:
808 msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
809 (node.primary, test_name, e))
810 raise qa_error.Error(msg)
811 if pvnum < 2:
812 raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" %
813 (node.primary, pvnum, test_name))
814 res = True
815 else:
816 res = False
817 return res
818
819
820 def RunInstanceTests():
821 """Create and exercise instances."""
822
823 for (test_name, templ, create_fun, num_nodes) in \
824 qa_instance.available_instance_tests:
825 if (qa_config.TestEnabled(test_name) and
826 qa_config.IsTemplateSupported(templ)):
827 inodes = qa_config.AcquireManyNodes(num_nodes)
828 try:
829 instance = RunTest(create_fun, inodes)
830 try:
831 RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
832 instance)
833 RunTestIf("instance-communication",
834 qa_instance.TestInstanceCommunication,
835 instance,
836 qa_config.GetMasterNode())
837 RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
838 RunDaemonTests(instance)
839 for node in inodes:
840 RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node,
841 templ == constants.DT_DRBD8)
842 if len(inodes) > 1:
843 RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
844 constants.INITIAL_NODE_GROUP_NAME,
845 inodes[0].primary, inodes[1].primary)
846 if qa_config.TestEnabled("instance-convert-disk"):
847 RunTest(qa_instance.TestInstanceShutdown, instance)
848 RunTest(qa_instance.TestInstanceConvertDiskToPlain,
849 instance, inodes)
850 RunTest(qa_instance.TestInstanceStartup, instance)
851 RunTestIf("instance-modify-disks",
852 qa_instance.TestInstanceModifyDisks, instance)
853 RunCommonInstanceTests(instance, inodes)
854 if qa_config.TestEnabled("instance-modify-primary"):
855 othernode = qa_config.AcquireNode()
856 RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
857 instance, inodes[0], othernode)
858 othernode.Release()
859 RunGroupListTests()
860 RunExportImportTests(instance, inodes)
861 RunHardwareFailureTests(instance, inodes)
862 RunRepairDiskSizes()
863 RunTest(qa_instance.TestInstanceRemove, instance)
864 finally:
865 instance.Release()
866 del instance
867 finally:
868 qa_config.ReleaseManyNodes(inodes)
869 qa_cluster.AssertClusterVerify()
870 else:
871 test_desc = "Creating instances of template %s" % templ
872 if not qa_config.TestEnabled(test_name):
873 ReportTestSkip(test_desc, test_name)
874 else:
875 ReportTestSkip(test_desc, "disk template %s" % templ)
876
877
878 def RunMonitoringTests():
879 RunTestIf("mon-collector", qa_monitoring.TestInstStatusCollector)
880
881
882 PARALLEL_TEST_DICT = {
883 "parallel-failover": qa_performance.TestParallelInstanceFailover,
884 "parallel-migration": qa_performance.TestParallelInstanceMigration,
885 "parallel-replace-disks": qa_performance.TestParallelInstanceReplaceDisks,
886 "parallel-reboot": qa_performance.TestParallelInstanceReboot,
887 "parallel-reinstall": qa_performance.TestParallelInstanceReinstall,
888 "parallel-rename": qa_performance.TestParallelInstanceRename,
889 }
890
891
892 def RunPerformanceTests():
893 if not qa_config.TestEnabled("performance"):
894 ReportTestSkip("performance related tests", "performance")
895 return
896
897 if qa_config.TestEnabled("jobqueue-performance"):
898 RunTest(qa_performance.TestParallelMaxInstanceCreationPerformance)
899 RunTest(qa_performance.TestParallelNodeCountInstanceCreationPerformance)
900
901 instances = qa_performance.CreateAllInstances()
902
903 RunTest(qa_performance.TestParallelModify, instances)
904 RunTest(qa_performance.TestParallelInstanceOSOperations, instances)
905 RunTest(qa_performance.TestParallelInstanceQueries, instances)
906
907 qa_performance.RemoveAllInstances(instances)
908
909 RunTest(qa_performance.TestJobQueueSubmissionPerformance)
910
911 if qa_config.TestEnabled("parallel-performance"):
912 if qa_config.IsTemplateSupported(constants.DT_DRBD8):
913 RunTest(qa_performance.TestParallelDRBDInstanceCreationPerformance)
914 if qa_config.IsTemplateSupported(constants.DT_PLAIN):
915 RunTest(qa_performance.TestParallelPlainInstanceCreationPerformance)
916
917 # Preparations need to be made only if some of these tests are enabled
918 if qa_config.IsTemplateSupported(constants.DT_DRBD8) and \
919 qa_config.TestEnabled(qa_config.Either(PARALLEL_TEST_DICT.keys())):
920 inodes = qa_config.AcquireManyNodes(2)
921 try:
922 instance = qa_instance.TestInstanceAddWithDrbdDisk(inodes)
923 try:
924 for (test_name, test_fn) in PARALLEL_TEST_DICT.items():
925 RunTestIf(test_name, test_fn, instance)
926 finally:
927 instance.Release()
928 qa_instance.TestInstanceRemove(instance)
929 finally:
930 qa_config.ReleaseManyNodes(inodes)
931
932
933 def RunQa():
934 """Main QA body.
935
936 """
937 rapi_user = "ganeti-qa"
938
939 RunTestBlock(RunEnvTests)
940 rapi_secret = SetupCluster(rapi_user)
941
942 if qa_rapi.Enabled():
943 # Load RAPI certificate
944 qa_rapi.Setup(rapi_user, rapi_secret)
945
946 RunTestBlock(RunClusterTests, rapi_user=rapi_user, rapi_secret=rapi_secret)
947 RunTestBlock(RunOsTests)
948
949 RunTestIf("tags", qa_tags.TestClusterTags)
950
951 RunTestBlock(RunCommonNodeTests)
952 RunTestBlock(RunGroupListTests)
953 RunTestBlock(RunGroupRwTests)
954 RunTestBlock(RunNetworkTests)
955
956 # The master shouldn't be readded or put offline; "delay" needs a non-master
957 # node to test
958 pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
959 try:
960 RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
961 RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
962 RunTestIf("delay", qa_cluster.TestDelay, pnode)
963 finally:
964 pnode.Release()
965
966 # Make sure the cluster is clean before running instance tests
967 qa_cluster.AssertClusterVerify()
968
969 pnode = qa_config.AcquireNode()
970 try:
971 RunTestIf("tags", qa_tags.TestNodeTags, pnode)
972
973 if qa_rapi.Enabled():
974 RunTest(qa_rapi.TestNode, pnode)
975
976 if (qa_config.TestEnabled("instance-add-plain-disk")
977 and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
978 # Normal instance allocation via RAPI
979 for use_client in [True, False]:
980 rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
981 use_client)
982 try:
983 if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
984 RunCommonInstanceTests(rapi_instance, [pnode])
985 RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
986 finally:
987 rapi_instance.Release()
988 del rapi_instance
989
990 # Multi-instance allocation
991 rapi_instance_one, rapi_instance_two = \
992 RunTest(qa_rapi.TestRapiInstanceMultiAlloc, pnode)
993
994 try:
995 RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_one, True)
996 RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_two, True)
997 finally:
998 rapi_instance_one.Release()
999 rapi_instance_two.Release()
1000 finally:
1001 pnode.Release()
1002
1003 config_list = [
1004 ("default-instance-tests", lambda: None, lambda _: None),
1005 (IsExclusiveStorageInstanceTestEnabled,
1006 lambda: qa_cluster.TestSetExclStorCluster(True),
1007 qa_cluster.TestSetExclStorCluster),
1008 ]
1009 for (conf_name, setup_conf_f, restore_conf_f) in config_list:
1010 if qa_config.TestEnabled(conf_name):
1011 oldconf = setup_conf_f()
1012 RunTestBlock(RunInstanceTests)
1013 restore_conf_f(oldconf)
1014
1015 pnode = qa_config.AcquireNode()
1016 try:
1017 if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
1018 for shutdown in [False, True]:
1019 instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
1020 try:
1021 expnode = qa_config.AcquireNode(exclude=pnode)
1022 try:
1023 if shutdown:
1024 # Stop instance before exporting and removing it
1025 RunTest(qa_instance.TestInstanceShutdown, instance)
1026 RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
1027 RunTest(qa_instance.TestBackupList, expnode)
1028 finally:
1029 expnode.Release()
1030 finally:
1031 instance.Release()
1032 del expnode
1033 del instance
1034 qa_cluster.AssertClusterVerify()
1035
1036 finally:
1037 pnode.Release()
1038
1039 RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade)
1040
1041 RunTestBlock(RunExclusiveStorageTests)
1042 RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
1043 TestIPolicyPlainInstance)
1044
1045 RunTestBlock(RunCustomSshPortTests)
1046
1047 RunTestIf(
1048 "instance-add-restricted-by-disktemplates",
1049 qa_instance.TestInstanceCreationRestrictedByDiskTemplates)
1050
1051 # Test removing instance with offline drbd secondary
1052 if qa_config.TestEnabled(["instance-remove-drbd-offline",
1053 "instance-add-drbd-disk"]):
1054 # Make sure the master is not put offline
1055 snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
1056 try:
1057 pnode = qa_config.AcquireNode(exclude=snode)
1058 try:
1059 instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
1060 set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
1061 set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
1062 RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
1063 set_offline, set_online)
1064 finally:
1065 pnode.Release()
1066 finally:
1067 snode.Release()
1068 qa_cluster.AssertClusterVerify()
1069
1070 RunTestBlock(RunMonitoringTests)
1071
1072 RunPerformanceTests()
1073
1074 RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
1075
1076 RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
1077
1078
1079 @UsesRapiClient
1080 def main():
1081 """Main program.
1082
1083 """
1084 colors.check_for_colors()
1085
1086 parser = optparse.OptionParser(usage="%prog [options] <config-file>")
1087 parser.add_option("--yes-do-it", dest="yes_do_it",
1088 action="store_true",
1089 help="Really execute the tests")
1090 (opts, args) = parser.parse_args()
1091
1092 if len(args) == 1:
1093 (config_file, ) = args
1094 else:
1095 parser.error("Wrong number of arguments.")
1096
1097 if not opts.yes_do_it:
1098 print ("Executing this script irreversibly destroys any Ganeti\n"
1099 "configuration on all nodes involved. If you really want\n"
1100 "to start testing, supply the --yes-do-it option.")
1101 sys.exit(1)
1102
1103 qa_config.Load(config_file)
1104
1105 primary = qa_config.GetMasterNode().primary
1106 qa_utils.StartMultiplexer(primary)
1107 print ("SSH command for primary node: %s" %
1108 utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
1109 print ("SSH command for other nodes: %s" %
1110 utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
1111 try:
1112 RunQa()
1113 finally:
1114 qa_utils.CloseMultiplexers()
1115
1116 if __name__ == "__main__":
1117 main()