After TestNodeModify, fix the pool of master candidates
[ganeti-github.git] / qa / qa_node.py
1 #
2 #
3
4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are
9 # met:
10 #
11 # 1. Redistributions of source code must retain the above copyright notice,
12 # this list of conditions and the following disclaimer.
13 #
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 """Node-related QA tests.
32
33 """
34
35 from ganeti import utils
36 from ganeti import constants
37 from ganeti import query
38 from ganeti import serializer
39
40 import qa_config
41 import qa_error
42 import qa_utils
43
44 from qa_utils import AssertCommand, AssertRedirectedCommand, AssertEqual, \
45 AssertIn, GetCommandOutput
46
47
48 def NodeAdd(node, readd=False, group=None):
49 if not readd and node.added:
50 raise qa_error.Error("Node %s already in cluster" % node.primary)
51 elif readd and not node.added:
52 raise qa_error.Error("Node %s not yet in cluster" % node.primary)
53
54 cmd = ["gnt-node", "add", "--no-ssh-key-check"]
55 if node.secondary:
56 cmd.append("--secondary-ip=%s" % node.secondary)
57 if readd:
58 cmd.append("--readd")
59 if group is not None:
60 cmd.extend(["--node-group", group])
61
62 if not qa_config.GetModifySshSetup():
63 cmd.append("--no-node-setup")
64
65 cmd.append(node.primary)
66
67 AssertCommand(cmd)
68
69 if readd:
70 AssertRedirectedCommand(["gnt-cluster", "verify"])
71
72 if readd:
73 assert node.added
74 else:
75 node.MarkAdded()
76
77
78 def NodeRemove(node):
79 AssertCommand(["gnt-node", "remove", node.primary])
80 node.MarkRemoved()
81
82
83 def MakeNodeOffline(node, value):
84 """gnt-node modify --offline=value"""
85 # value in ["yes", "no"]
86 AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
87
88
89 def TestNodeAddAll():
90 """Adding all nodes to cluster."""
91 master = qa_config.GetMasterNode()
92 for node in qa_config.get("nodes"):
93 if node != master:
94 NodeAdd(node, readd=False)
95
96
97 def MarkNodeAddedAll():
98 """Mark all nodes as added.
99
100 This is useful if we don't create the cluster ourselves (in qa).
101
102 """
103 master = qa_config.GetMasterNode()
104 for node in qa_config.get("nodes"):
105 if node != master:
106 node.MarkAdded()
107
108
109 def TestNodeRemoveAll():
110 """Removing all nodes from cluster."""
111 master = qa_config.GetMasterNode()
112 for node in qa_config.get("nodes"):
113 if node != master:
114 NodeRemove(node)
115
116
117 def TestNodeReadd(node):
118 """gnt-node add --readd"""
119 NodeAdd(node, readd=True)
120
121
122 def TestNodeInfo():
123 """gnt-node info"""
124 AssertCommand(["gnt-node", "info"])
125
126
127 def TestNodeVolumes():
128 """gnt-node volumes"""
129 AssertCommand(["gnt-node", "volumes"])
130
131
132 def TestNodeStorage():
133 """gnt-node storage"""
134 master = qa_config.GetMasterNode()
135
136 # FIXME: test all storage_types in constants.STORAGE_TYPES
137 # as soon as they are implemented.
138 enabled_storage_types = qa_config.GetEnabledStorageTypes()
139 testable_storage_types = list(set(enabled_storage_types).intersection(
140 set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV])))
141
142 for storage_type in testable_storage_types:
143
144 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
145
146 # Test simple list
147 AssertCommand(cmd)
148
149 # Test all storage fields
150 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
151 "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS))]
152 AssertCommand(cmd)
153
154 # Get list of valid storage devices
155 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
156 "--output=node,name,allocatable", "--separator=|",
157 "--no-headers"]
158 output = qa_utils.GetCommandOutput(master.primary,
159 utils.ShellQuoteArgs(cmd))
160
161 # Test with up to two devices
162 testdevcount = 2
163
164 for line in output.splitlines()[:testdevcount]:
165 (node_name, st_name, st_allocatable) = line.split("|")
166
167 # Dummy modification without any changes
168 cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
169 AssertCommand(cmd)
170
171 # Make sure we end up with the same value as before
172 if st_allocatable.lower() == "y":
173 test_allocatable = ["no", "yes"]
174 else:
175 test_allocatable = ["yes", "no"]
176
177 fail = (constants.SF_ALLOCATABLE not in
178 constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
179
180 for i in test_allocatable:
181 AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
182 node_name, storage_type, st_name], fail=fail)
183
184 # Verify list output
185 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
186 "--output=name,allocatable", "--separator=|",
187 "--no-headers", node_name]
188 listout = qa_utils.GetCommandOutput(master.primary,
189 utils.ShellQuoteArgs(cmd))
190 for line in listout.splitlines():
191 (vfy_name, vfy_allocatable) = line.split("|")
192 if vfy_name == st_name and not fail:
193 AssertEqual(vfy_allocatable, i[0].upper())
194 else:
195 AssertEqual(vfy_allocatable, st_allocatable)
196
197 # Test repair functionality
198 fail = (constants.SO_FIX_CONSISTENCY not in
199 constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
200 AssertCommand(["gnt-node", "repair-storage", node_name,
201 storage_type, st_name], fail=fail)
202
203
204 def TestNodeFailover(node, node2):
205 """gnt-node failover"""
206 if qa_utils.GetNodeInstances(node2, secondaries=False):
207 raise qa_error.UnusableNodeError("Secondary node has at least one"
208 " primary instance. This test requires"
209 " it to have no primary instances.")
210
211 # Fail over to secondary node
212 AssertCommand(["gnt-node", "failover", "-f", node.primary])
213
214 # ... and back again.
215 AssertCommand(["gnt-node", "failover", "-f", node2.primary])
216
217
218 def TestNodeMigrate(node, node2):
219 """gnt-node migrate"""
220 if qa_utils.GetNodeInstances(node2, secondaries=False):
221 raise qa_error.UnusableNodeError("Secondary node has at least one"
222 " primary instance. This test requires"
223 " it to have no primary instances.")
224
225 # Migrate to secondary node
226 AssertCommand(["gnt-node", "migrate", "-f", node.primary])
227
228 # ... and back again.
229 AssertCommand(["gnt-node", "migrate", "-f", node2.primary])
230
231
232 def TestNodeEvacuate(node, node2):
233 """gnt-node evacuate"""
234 node3 = qa_config.AcquireNode(exclude=[node, node2])
235 try:
236 if qa_utils.GetNodeInstances(node3, secondaries=True):
237 raise qa_error.UnusableNodeError("Evacuation node has at least one"
238 " secondary instance. This test requires"
239 " it to have no secondary instances.")
240
241 # Evacuate all secondary instances
242 AssertCommand(["gnt-node", "evacuate", "-f",
243 "--new-secondary=%s" % node3.primary, node2.primary])
244
245 # ... and back again.
246 AssertCommand(["gnt-node", "evacuate", "-f",
247 "--new-secondary=%s" % node2.primary, node3.primary])
248 finally:
249 node3.Release()
250
251
252 def TestNodeModify(node):
253 """gnt-node modify"""
254
255 default_pool_size = 10
256 nodes = qa_config.GetAllNodes()
257 test_pool_size = len(nodes) - 1
258
259 # Reduce the number of master candidates, because otherwise all
260 # subsequent 'gnt-cluster verify' commands fail due to not enough
261 # master candidates.
262 AssertCommand(["gnt-cluster", "modify",
263 "--candidate-pool-size=%s" % test_pool_size])
264
265 # make sure enough master candidates will be available by disabling the
266 # master candidate role first with --auto-promote
267 AssertCommand(["gnt-node", "modify", "--master-candidate=no",
268 "--auto-promote", node.primary])
269
270 # now it's save to force-remove the master candidate role
271 for flag in ["master-candidate", "drained", "offline"]:
272 for value in ["yes", "no"]:
273 AssertCommand(["gnt-node", "modify", "--force",
274 "--%s=%s" % (flag, value), node.primary])
275 AssertCommand(["gnt-cluster", "verify"])
276
277 AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary])
278
279 # Test setting secondary IP address
280 AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
281 node.primary])
282
283 AssertRedirectedCommand(["gnt-cluster", "verify"])
284 AssertCommand(["gnt-cluster", "modify",
285 "--candidate-pool-size=%s" % default_pool_size])
286
287 # For test clusters with more nodes than the default pool size,
288 # we now have too many master candidates. To readjust to the original
289 # size, manually demote all nodes and rely on auto-promotion to adjust.
290 if len(nodes) > default_pool_size:
291 master = qa_config.GetMasterNode()
292 for n in nodes:
293 if n.primary != master.primary:
294 AssertCommand(["gnt-node", "modify", "--master-candidate=no",
295 "--auto-promote", n.primary])
296
297
298 def _CreateOobScriptStructure():
299 """Create a simple OOB handling script and its structure."""
300 master = qa_config.GetMasterNode()
301
302 data_path = qa_utils.UploadData(master.primary, "")
303 verify_path = qa_utils.UploadData(master.primary, "")
304 exit_code_path = qa_utils.UploadData(master.primary, "")
305
306 oob_script = (("#!/bin/bash\n"
307 "echo \"$@\" > %s\n"
308 "cat %s\n"
309 "exit $(< %s)\n") %
310 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
311 utils.ShellQuote(exit_code_path)))
312 oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
313
314 return [oob_path, verify_path, data_path, exit_code_path]
315
316
317 def _UpdateOobFile(path, data):
318 """Updates the data file with data."""
319 master = qa_config.GetMasterNode()
320 qa_utils.UploadData(master.primary, data, filename=path)
321
322
323 def _AssertOobCall(verify_path, expected_args):
324 """Assert the OOB call was performed with expetected args."""
325 master = qa_config.GetMasterNode()
326
327 verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
328 output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
329 tty=False)
330
331 AssertEqual(expected_args, output.strip())
332
333
334 def TestOutOfBand():
335 """gnt-node power"""
336 master = qa_config.GetMasterNode()
337
338 node = qa_config.AcquireNode(exclude=master)
339
340 master_name = master.primary
341 node_name = node.primary
342 full_node_name = qa_utils.ResolveNodeName(node)
343
344 (oob_path, verify_path,
345 data_path, exit_code_path) = _CreateOobScriptStructure()
346
347 try:
348 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
349 "oob_program=%s" % oob_path])
350
351 # No data, exit 0
352 _UpdateOobFile(exit_code_path, "0")
353
354 AssertCommand(["gnt-node", "power", "on", node_name])
355 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
356
357 AssertCommand(["gnt-node", "power", "-f", "off", node_name])
358 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
359
360 # Power off on master without options should fail
361 AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
362 # With force master it should still fail
363 AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
364 master_name],
365 fail=True)
366
367 # Verify we can't transform back to online when not yet powered on
368 AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
369 fail=True)
370 # Now reset state
371 AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
372 node_name])
373
374 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
375 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
376
377 # Those commands should fail as they expect output which isn't provided yet
378 # But they should have called the oob helper nevermind
379 AssertCommand(["gnt-node", "power", "status", node_name],
380 fail=True)
381 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
382
383 AssertCommand(["gnt-node", "health", node_name],
384 fail=True)
385 _AssertOobCall(verify_path, "health %s" % full_node_name)
386
387 AssertCommand(["gnt-node", "health"], fail=True)
388
389 # Correct Data, exit 0
390 _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
391
392 AssertCommand(["gnt-node", "power", "status", node_name])
393 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
394
395 _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
396 ["disk0", "CRITICAL"]]))
397
398 AssertCommand(["gnt-node", "health", node_name])
399 _AssertOobCall(verify_path, "health %s" % full_node_name)
400
401 AssertCommand(["gnt-node", "health"])
402
403 # Those commands should fail as they expect no data regardless of exit 0
404 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
405 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
406
407 try:
408 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
409 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
410 finally:
411 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
412
413 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
414 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
415
416 # Data, exit 1 (all should fail)
417 _UpdateOobFile(exit_code_path, "1")
418
419 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
420 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
421
422 try:
423 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
424 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
425 finally:
426 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
427
428 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
429 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
430
431 AssertCommand(["gnt-node", "power", "status", node_name],
432 fail=True)
433 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
434
435 AssertCommand(["gnt-node", "health", node_name],
436 fail=True)
437 _AssertOobCall(verify_path, "health %s" % full_node_name)
438
439 AssertCommand(["gnt-node", "health"], fail=True)
440
441 # No data, exit 1 (all should fail)
442 _UpdateOobFile(data_path, "")
443 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
444 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
445
446 try:
447 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
448 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
449 finally:
450 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
451
452 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
453 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
454
455 AssertCommand(["gnt-node", "power", "status", node_name],
456 fail=True)
457 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
458
459 AssertCommand(["gnt-node", "health", node_name],
460 fail=True)
461 _AssertOobCall(verify_path, "health %s" % full_node_name)
462
463 AssertCommand(["gnt-node", "health"], fail=True)
464
465 # Different OOB script for node
466 verify_path2 = qa_utils.UploadData(master.primary, "")
467 oob_script = ("#!/bin/sh\n"
468 "echo \"$@\" > %s\n") % verify_path2
469 oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
470
471 try:
472 AssertCommand(["gnt-node", "modify", "--node-parameters",
473 "oob_program=%s" % oob_path2, node_name])
474 AssertCommand(["gnt-node", "power", "on", node_name])
475 _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
476 finally:
477 AssertCommand(["gnt-node", "modify", "--node-parameters",
478 "oob_program=default", node_name])
479 AssertCommand(["rm", "-f", oob_path2, verify_path2])
480 finally:
481 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
482 "oob_program="])
483 AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
484 exit_code_path])
485
486
487 def TestNodeList():
488 """gnt-node list"""
489 qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
490
491
492 def TestNodeListFields():
493 """gnt-node list-fields"""
494 qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
495
496
497 def TestNodeListDrbd(node, is_drbd):
498 """gnt-node list-drbd"""
499 master = qa_config.GetMasterNode()
500 result_output = GetCommandOutput(master.primary,
501 "gnt-node list-drbd --no-header %s" %
502 node.primary)
503 # Meaningful to note: there is but one instance, and the node is either the
504 # primary or one of the secondaries
505 if is_drbd:
506 # Invoked for both primary and secondary
507 per_disk_info = result_output.splitlines()
508 for line in per_disk_info:
509 try:
510 drbd_node, _, _, _, _, drbd_peer = line.split()
511 except ValueError:
512 raise qa_error.Error("Could not examine list-drbd output: expected a"
513 " single row of 6 entries, found the following:"
514 " %s" % line)
515
516 AssertIn(node.primary, [drbd_node, drbd_peer],
517 msg="The output %s does not contain the node" % line)
518 else:
519 # Output should be empty, barring newlines
520 AssertEqual(result_output.strip(), "")
521
522
523 def _BuildSetESCmd(action, value, node_name):
524 cmd = ["gnt-node"]
525 if action == "add":
526 cmd.extend(["add", "--readd"])
527 if not qa_config.GetModifySshSetup():
528 cmd.append("--no-node-setup")
529 else:
530 cmd.append("modify")
531 cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
532 return cmd
533
534
535 def TestExclStorSingleNode(node):
536 """gnt-node add/modify cannot change the exclusive_storage flag.
537
538 """
539 for action in ["add", "modify"]:
540 for value in (True, False, "default"):
541 AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)