d99004acdeaca56fee9018773abb021cdacc17fc
[ganeti-github.git] / qa / qa_node.py
1 #
2 #
3
4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5 #
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 # 02110-1301, USA.
20
21
22 """Node-related QA tests.
23
24 """
25
26 from ganeti import utils
27 from ganeti import constants
28 from ganeti import query
29 from ganeti import serializer
30
31 import qa_config
32 import qa_error
33 import qa_utils
34
35 from qa_utils import AssertCommand, AssertEqual
36
37
38 def _NodeAdd(node, readd=False):
39 if not readd and node.added:
40 raise qa_error.Error("Node %s already in cluster" % node.primary)
41 elif readd and not node.added:
42 raise qa_error.Error("Node %s not yet in cluster" % node.primary)
43
44 cmd = ["gnt-node", "add", "--no-ssh-key-check"]
45 if node.secondary:
46 cmd.append("--secondary-ip=%s" % node.secondary)
47 if readd:
48 cmd.append("--readd")
49 cmd.append(node.primary)
50
51 AssertCommand(cmd)
52
53 if readd:
54 assert node.added
55 else:
56 node.MarkAdded()
57
58
59 def _NodeRemove(node):
60 AssertCommand(["gnt-node", "remove", node.primary])
61 node.MarkRemoved()
62
63
64 def MakeNodeOffline(node, value):
65 """gnt-node modify --offline=value"""
66 # value in ["yes", "no"]
67 AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
68
69
70 def TestNodeAddAll():
71 """Adding all nodes to cluster."""
72 master = qa_config.GetMasterNode()
73 for node in qa_config.get("nodes"):
74 if node != master:
75 _NodeAdd(node, readd=False)
76
77
78 def MarkNodeAddedAll():
79 """Mark all nodes as added.
80
81 This is useful if we don't create the cluster ourselves (in qa).
82
83 """
84 master = qa_config.GetMasterNode()
85 for node in qa_config.get("nodes"):
86 if node != master:
87 node.MarkAdded()
88
89
90 def TestNodeRemoveAll():
91 """Removing all nodes from cluster."""
92 master = qa_config.GetMasterNode()
93 for node in qa_config.get("nodes"):
94 if node != master:
95 _NodeRemove(node)
96
97
98 def TestNodeReadd(node):
99 """gnt-node add --readd"""
100 _NodeAdd(node, readd=True)
101
102
103 def TestNodeInfo():
104 """gnt-node info"""
105 AssertCommand(["gnt-node", "info"])
106
107
108 def TestNodeVolumes():
109 """gnt-node volumes"""
110 AssertCommand(["gnt-node", "volumes"])
111
112
113 def TestNodeStorage():
114 """gnt-node storage"""
115 master = qa_config.GetMasterNode()
116
117 for storage_type in constants.VALID_STORAGE_TYPES:
118
119 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
120
121 # Skip file storage if not enabled, otherwise QA will fail; we
122 # just test for basic failure, but otherwise skip the rest of the
123 # tests
124 if storage_type == constants.ST_FILE and not constants.ENABLE_FILE_STORAGE:
125 AssertCommand(cmd, fail=True)
126 continue
127
128 # Test simple list
129 AssertCommand(cmd)
130
131 # Test all storage fields
132 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
133 "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
134 [constants.SF_NODE, constants.SF_TYPE])]
135 AssertCommand(cmd)
136
137 # Get list of valid storage devices
138 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
139 "--output=node,name,allocatable", "--separator=|",
140 "--no-headers"]
141 output = qa_utils.GetCommandOutput(master.primary,
142 utils.ShellQuoteArgs(cmd))
143
144 # Test with up to two devices
145 testdevcount = 2
146
147 for line in output.splitlines()[:testdevcount]:
148 (node_name, st_name, st_allocatable) = line.split("|")
149
150 # Dummy modification without any changes
151 cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
152 AssertCommand(cmd)
153
154 # Make sure we end up with the same value as before
155 if st_allocatable.lower() == "y":
156 test_allocatable = ["no", "yes"]
157 else:
158 test_allocatable = ["yes", "no"]
159
160 fail = (constants.SF_ALLOCATABLE not in
161 constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
162
163 for i in test_allocatable:
164 AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
165 node_name, storage_type, st_name], fail=fail)
166
167 # Verify list output
168 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
169 "--output=name,allocatable", "--separator=|",
170 "--no-headers", node_name]
171 listout = qa_utils.GetCommandOutput(master.primary,
172 utils.ShellQuoteArgs(cmd))
173 for line in listout.splitlines():
174 (vfy_name, vfy_allocatable) = line.split("|")
175 if vfy_name == st_name and not fail:
176 AssertEqual(vfy_allocatable, i[0].upper())
177 else:
178 AssertEqual(vfy_allocatable, st_allocatable)
179
180 # Test repair functionality
181 fail = (constants.SO_FIX_CONSISTENCY not in
182 constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
183 AssertCommand(["gnt-node", "repair-storage", node_name,
184 storage_type, st_name], fail=fail)
185
186
187 def TestNodeFailover(node, node2):
188 """gnt-node failover"""
189 if qa_utils.GetNodeInstances(node2, secondaries=False):
190 raise qa_error.UnusableNodeError("Secondary node has at least one"
191 " primary instance. This test requires"
192 " it to have no primary instances.")
193
194 # Fail over to secondary node
195 AssertCommand(["gnt-node", "failover", "-f", node.primary])
196
197 # ... and back again.
198 AssertCommand(["gnt-node", "failover", "-f", node2.primary])
199
200
201 def TestNodeEvacuate(node, node2):
202 """gnt-node evacuate"""
203 node3 = qa_config.AcquireNode(exclude=[node, node2])
204 try:
205 if qa_utils.GetNodeInstances(node3, secondaries=True):
206 raise qa_error.UnusableNodeError("Evacuation node has at least one"
207 " secondary instance. This test requires"
208 " it to have no secondary instances.")
209
210 # Evacuate all secondary instances
211 AssertCommand(["gnt-node", "evacuate", "-f",
212 "--new-secondary=%s" % node3.primary, node2.primary])
213
214 # ... and back again.
215 AssertCommand(["gnt-node", "evacuate", "-f",
216 "--new-secondary=%s" % node2.primary, node3.primary])
217 finally:
218 node3.Release()
219
220
221 def TestNodeModify(node):
222 """gnt-node modify"""
223 for flag in ["master-candidate", "drained", "offline"]:
224 for value in ["yes", "no"]:
225 AssertCommand(["gnt-node", "modify", "--force",
226 "--%s=%s" % (flag, value), node.primary])
227
228 AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
229 "--auto-promote", node.primary])
230
231 # Test setting secondary IP address
232 AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
233 node.primary])
234
235
236 def _CreateOobScriptStructure():
237 """Create a simple OOB handling script and its structure."""
238 master = qa_config.GetMasterNode()
239
240 data_path = qa_utils.UploadData(master.primary, "")
241 verify_path = qa_utils.UploadData(master.primary, "")
242 exit_code_path = qa_utils.UploadData(master.primary, "")
243
244 oob_script = (("#!/bin/bash\n"
245 "echo \"$@\" > %s\n"
246 "cat %s\n"
247 "exit $(< %s)\n") %
248 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
249 utils.ShellQuote(exit_code_path)))
250 oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
251
252 return [oob_path, verify_path, data_path, exit_code_path]
253
254
255 def _UpdateOobFile(path, data):
256 """Updates the data file with data."""
257 master = qa_config.GetMasterNode()
258 qa_utils.UploadData(master.primary, data, filename=path)
259
260
261 def _AssertOobCall(verify_path, expected_args):
262 """Assert the OOB call was performed with expetected args."""
263 master = qa_config.GetMasterNode()
264
265 verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
266 output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
267 tty=False)
268
269 AssertEqual(expected_args, output.strip())
270
271
272 def TestOutOfBand():
273 """gnt-node power"""
274 master = qa_config.GetMasterNode()
275
276 node = qa_config.AcquireNode(exclude=master)
277
278 master_name = master.primary
279 node_name = node.primary
280 full_node_name = qa_utils.ResolveNodeName(node)
281
282 (oob_path, verify_path,
283 data_path, exit_code_path) = _CreateOobScriptStructure()
284
285 try:
286 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
287 "oob_program=%s" % oob_path])
288
289 # No data, exit 0
290 _UpdateOobFile(exit_code_path, "0")
291
292 AssertCommand(["gnt-node", "power", "on", node_name])
293 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
294
295 AssertCommand(["gnt-node", "power", "-f", "off", node_name])
296 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
297
298 # Power off on master without options should fail
299 AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
300 # With force master it should still fail
301 AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
302 master_name],
303 fail=True)
304
305 # Verify we can't transform back to online when not yet powered on
306 AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
307 fail=True)
308 # Now reset state
309 AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
310 node_name])
311
312 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
313 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
314
315 # Those commands should fail as they expect output which isn't provided yet
316 # But they should have called the oob helper nevermind
317 AssertCommand(["gnt-node", "power", "status", node_name],
318 fail=True)
319 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
320
321 AssertCommand(["gnt-node", "health", node_name],
322 fail=True)
323 _AssertOobCall(verify_path, "health %s" % full_node_name)
324
325 AssertCommand(["gnt-node", "health"], fail=True)
326
327 # Correct Data, exit 0
328 _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
329
330 AssertCommand(["gnt-node", "power", "status", node_name])
331 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
332
333 _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
334 ["disk0", "CRITICAL"]]))
335
336 AssertCommand(["gnt-node", "health", node_name])
337 _AssertOobCall(verify_path, "health %s" % full_node_name)
338
339 AssertCommand(["gnt-node", "health"])
340
341 # Those commands should fail as they expect no data regardless of exit 0
342 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
343 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
344
345 try:
346 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
347 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
348 finally:
349 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
350
351 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
352 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
353
354 # Data, exit 1 (all should fail)
355 _UpdateOobFile(exit_code_path, "1")
356
357 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
358 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
359
360 try:
361 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
362 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
363 finally:
364 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
365
366 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
367 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
368
369 AssertCommand(["gnt-node", "power", "status", node_name],
370 fail=True)
371 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
372
373 AssertCommand(["gnt-node", "health", node_name],
374 fail=True)
375 _AssertOobCall(verify_path, "health %s" % full_node_name)
376
377 AssertCommand(["gnt-node", "health"], fail=True)
378
379 # No data, exit 1 (all should fail)
380 _UpdateOobFile(data_path, "")
381 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
382 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
383
384 try:
385 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
386 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
387 finally:
388 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
389
390 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
391 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
392
393 AssertCommand(["gnt-node", "power", "status", node_name],
394 fail=True)
395 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
396
397 AssertCommand(["gnt-node", "health", node_name],
398 fail=True)
399 _AssertOobCall(verify_path, "health %s" % full_node_name)
400
401 AssertCommand(["gnt-node", "health"], fail=True)
402
403 # Different OOB script for node
404 verify_path2 = qa_utils.UploadData(master.primary, "")
405 oob_script = ("#!/bin/sh\n"
406 "echo \"$@\" > %s\n") % verify_path2
407 oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
408
409 try:
410 AssertCommand(["gnt-node", "modify", "--node-parameters",
411 "oob_program=%s" % oob_path2, node_name])
412 AssertCommand(["gnt-node", "power", "on", node_name])
413 _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
414 finally:
415 AssertCommand(["gnt-node", "modify", "--node-parameters",
416 "oob_program=default", node_name])
417 AssertCommand(["rm", "-f", oob_path2, verify_path2])
418 finally:
419 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
420 "oob_program="])
421 AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
422 exit_code_path])
423
424
425 def TestNodeList():
426 """gnt-node list"""
427 qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
428
429
430 def TestNodeListFields():
431 """gnt-node list-fields"""
432 qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
433
434
435 def TestNodeListDrbd(node):
436 """gnt-node list-drbd"""
437 AssertCommand(["gnt-node", "list-drbd", node.primary])
438
439
440 def _BuildSetESCmd(action, value, node_name):
441 cmd = ["gnt-node"]
442 if action == "add":
443 cmd.extend(["add", "--readd"])
444 else:
445 cmd.append("modify")
446 cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
447 return cmd
448
449
450 def TestExclStorSingleNode(node):
451 """gnt-node add/modify cannot change the exclusive_storage flag.
452
453 """
454 for action in ["add", "modify"]:
455 for value in (True, False, "default"):
456 AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)