Add tags in network objects
[ganeti-github.git] / lib / query.py
1 #
2 #
3
4 # Copyright (C) 2010, 2011, 2012 Google Inc.
5 #
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 # 02110-1301, USA.
20
21
22 """Module for query operations
23
24 How it works:
25
26 - Add field definitions
27 - See how L{NODE_FIELDS} is built
28 - Each field gets:
29 - Query field definition (L{objects.QueryFieldDefinition}, use
30 L{_MakeField} for creating), containing:
31 - Name, must be lowercase and match L{FIELD_NAME_RE}
32 - Title for tables, must not contain whitespace and match
33 L{TITLE_RE}
34 - Value data type, e.g. L{constants.QFT_NUMBER}
35 - Human-readable description, must not end with punctuation or
36 contain newlines
37 - Data request type, see e.g. C{NQ_*}
38 - OR-ed flags, see C{QFF_*}
39 - A retrieval function, see L{Query.__init__} for description
40 - Pass list of fields through L{_PrepareFieldList} for preparation and
41 checks
42 - Instantiate L{Query} with prepared field list definition and selected fields
43 - Call L{Query.RequestedData} to determine what data to collect/compute
44 - Call L{Query.Query} or L{Query.OldStyleQuery} with collected data and use
45 result
46 - Data container must support iteration using C{__iter__}
47 - Items are passed to retrieval functions and can have any format
48 - Call L{Query.GetFields} to get list of definitions for selected fields
49
50 @attention: Retrieval functions must be idempotent. They can be called multiple
51 times, in any order and any number of times.
52
53 """
54
55 import logging
56 import operator
57 import re
58
59 from ganeti import constants
60 from ganeti import errors
61 from ganeti import utils
62 from ganeti import compat
63 from ganeti import objects
64 from ganeti import ht
65 from ganeti import runtime
66 from ganeti import qlang
67 from ganeti import jstore
68
69 from ganeti.constants import (QFT_UNKNOWN, QFT_TEXT, QFT_BOOL, QFT_NUMBER,
70 QFT_UNIT, QFT_TIMESTAMP, QFT_OTHER,
71 RS_NORMAL, RS_UNKNOWN, RS_NODATA,
72 RS_UNAVAIL, RS_OFFLINE)
73
74 (NETQ_CONFIG,
75 NETQ_GROUP,
76 NETQ_STATS,
77 NETQ_INST) = range(300, 304)
78
79 # Constants for requesting data from the caller/data provider. Each property
80 # collected/computed separately by the data provider should have its own to
81 # only collect the requested data and not more.
82
83 (NQ_CONFIG,
84 NQ_INST,
85 NQ_LIVE,
86 NQ_GROUP,
87 NQ_OOB) = range(1, 6)
88
89 (IQ_CONFIG,
90 IQ_LIVE,
91 IQ_DISKUSAGE,
92 IQ_CONSOLE,
93 IQ_NODES) = range(100, 105)
94
95 (LQ_MODE,
96 LQ_OWNER,
97 LQ_PENDING) = range(10, 13)
98
99 (GQ_CONFIG,
100 GQ_NODE,
101 GQ_INST,
102 GQ_DISKPARAMS) = range(200, 204)
103
104 (CQ_CONFIG,
105 CQ_QUEUE_DRAINED,
106 CQ_WATCHER_PAUSE) = range(300, 303)
107
108 (JQ_ARCHIVED, ) = range(400, 401)
109
110 # Query field flags
111 QFF_HOSTNAME = 0x01
112 QFF_IP_ADDRESS = 0x02
113 QFF_JOB_ID = 0x04
114 QFF_SPLIT_TIMESTAMP = 0x08
115 # Next values: 0x10, 0x20, 0x40, 0x80, 0x100, 0x200
116 QFF_ALL = (QFF_HOSTNAME | QFF_IP_ADDRESS | QFF_JOB_ID | QFF_SPLIT_TIMESTAMP)
117
118 FIELD_NAME_RE = re.compile(r"^[a-z0-9/._]+$")
119 TITLE_RE = re.compile(r"^[^\s]+$")
120 DOC_RE = re.compile(r"^[A-Z].*[^.,?!]$")
121
122 #: Verification function for each field type
123 _VERIFY_FN = {
124 QFT_UNKNOWN: ht.TNone,
125 QFT_TEXT: ht.TString,
126 QFT_BOOL: ht.TBool,
127 QFT_NUMBER: ht.TInt,
128 QFT_UNIT: ht.TInt,
129 QFT_TIMESTAMP: ht.TNumber,
130 QFT_OTHER: lambda _: True,
131 }
132
133 # Unique objects for special field statuses
134 _FS_UNKNOWN = object()
135 _FS_NODATA = object()
136 _FS_UNAVAIL = object()
137 _FS_OFFLINE = object()
138
139 #: List of all special status
140 _FS_ALL = frozenset([_FS_UNKNOWN, _FS_NODATA, _FS_UNAVAIL, _FS_OFFLINE])
141
142 #: VType to QFT mapping
143 _VTToQFT = {
144 # TODO: fix validation of empty strings
145 constants.VTYPE_STRING: QFT_OTHER, # since VTYPE_STRINGs can be empty
146 constants.VTYPE_MAYBE_STRING: QFT_OTHER,
147 constants.VTYPE_BOOL: QFT_BOOL,
148 constants.VTYPE_SIZE: QFT_UNIT,
149 constants.VTYPE_INT: QFT_NUMBER,
150 }
151
152 _SERIAL_NO_DOC = "%s object serial number, incremented on each modification"
153
154
155 def _GetUnknownField(ctx, item): # pylint: disable=W0613
156 """Gets the contents of an unknown field.
157
158 """
159 return _FS_UNKNOWN
160
161
162 def _GetQueryFields(fielddefs, selected):
163 """Calculates the internal list of selected fields.
164
165 Unknown fields are returned as L{constants.QFT_UNKNOWN}.
166
167 @type fielddefs: dict
168 @param fielddefs: Field definitions
169 @type selected: list of strings
170 @param selected: List of selected fields
171
172 """
173 result = []
174
175 for name in selected:
176 try:
177 fdef = fielddefs[name]
178 except KeyError:
179 fdef = (_MakeField(name, name, QFT_UNKNOWN, "Unknown field '%s'" % name),
180 None, 0, _GetUnknownField)
181
182 assert len(fdef) == 4
183
184 result.append(fdef)
185
186 return result
187
188
189 def GetAllFields(fielddefs):
190 """Extract L{objects.QueryFieldDefinition} from field definitions.
191
192 @rtype: list of L{objects.QueryFieldDefinition}
193
194 """
195 return [fdef for (fdef, _, _, _) in fielddefs]
196
197
198 class _FilterHints:
199 """Class for filter analytics.
200
201 When filters are used, the user of the L{Query} class usually doesn't know
202 exactly which items will be necessary for building the result. It therefore
203 has to prepare and compute the input data for potentially returning
204 everything.
205
206 There are two ways to optimize this. The first, and simpler, is to assign
207 each field a group of data, so that the caller can determine which
208 computations are necessary depending on the data groups requested. The list
209 of referenced groups must also be computed for fields referenced in the
210 filter.
211
212 The second is restricting the items based on a primary key. The primary key
213 is usually a unique name (e.g. a node name). This class extracts all
214 referenced names from a filter. If it encounters any filter condition which
215 disallows such a list to be determined (e.g. a non-equality filter), all
216 names will be requested.
217
218 The end-effect is that any operation other than L{qlang.OP_OR} and
219 L{qlang.OP_EQUAL} will make the query more expensive.
220
221 """
222 def __init__(self, namefield):
223 """Initializes this class.
224
225 @type namefield: string
226 @param namefield: Field caller is interested in
227
228 """
229 self._namefield = namefield
230
231 #: Whether all names need to be requested (e.g. if a non-equality operator
232 #: has been used)
233 self._allnames = False
234
235 #: Which names to request
236 self._names = None
237
238 #: Data kinds referenced by the filter (used by L{Query.RequestedData})
239 self._datakinds = set()
240
241 def RequestedNames(self):
242 """Returns all requested values.
243
244 Returns C{None} if list of values can't be determined (e.g. encountered
245 non-equality operators).
246
247 @rtype: list
248
249 """
250 if self._allnames or self._names is None:
251 return None
252
253 return utils.UniqueSequence(self._names)
254
255 def ReferencedData(self):
256 """Returns all kinds of data referenced by the filter.
257
258 """
259 return frozenset(self._datakinds)
260
261 def _NeedAllNames(self):
262 """Changes internal state to request all names.
263
264 """
265 self._allnames = True
266 self._names = None
267
268 def NoteLogicOp(self, op):
269 """Called when handling a logic operation.
270
271 @type op: string
272 @param op: Operator
273
274 """
275 if op != qlang.OP_OR:
276 self._NeedAllNames()
277
278 def NoteUnaryOp(self, op, datakind): # pylint: disable=W0613
279 """Called when handling an unary operation.
280
281 @type op: string
282 @param op: Operator
283
284 """
285 if datakind is not None:
286 self._datakinds.add(datakind)
287
288 self._NeedAllNames()
289
290 def NoteBinaryOp(self, op, datakind, name, value):
291 """Called when handling a binary operation.
292
293 @type op: string
294 @param op: Operator
295 @type name: string
296 @param name: Left-hand side of operator (field name)
297 @param value: Right-hand side of operator
298
299 """
300 if datakind is not None:
301 self._datakinds.add(datakind)
302
303 if self._allnames:
304 return
305
306 # If any operator other than equality was used, all names need to be
307 # retrieved
308 if op == qlang.OP_EQUAL and name == self._namefield:
309 if self._names is None:
310 self._names = []
311 self._names.append(value)
312 else:
313 self._NeedAllNames()
314
315
316 def _WrapLogicOp(op_fn, sentences, ctx, item):
317 """Wrapper for logic operator functions.
318
319 """
320 return op_fn(fn(ctx, item) for fn in sentences)
321
322
323 def _WrapUnaryOp(op_fn, inner, ctx, item):
324 """Wrapper for unary operator functions.
325
326 """
327 return op_fn(inner(ctx, item))
328
329
330 def _WrapBinaryOp(op_fn, retrieval_fn, value, ctx, item):
331 """Wrapper for binary operator functions.
332
333 """
334 return op_fn(retrieval_fn(ctx, item), value)
335
336
337 def _WrapNot(fn, lhs, rhs):
338 """Negates the result of a wrapped function.
339
340 """
341 return not fn(lhs, rhs)
342
343
344 def _PrepareRegex(pattern):
345 """Compiles a regular expression.
346
347 """
348 try:
349 return re.compile(pattern)
350 except re.error, err:
351 raise errors.ParameterError("Invalid regex pattern (%s)" % err)
352
353
354 def _PrepareSplitTimestamp(value):
355 """Prepares a value for comparison by L{_MakeSplitTimestampComparison}.
356
357 """
358 if ht.TNumber(value):
359 return value
360 else:
361 return utils.MergeTime(value)
362
363
364 def _MakeSplitTimestampComparison(fn):
365 """Compares split timestamp values after converting to float.
366
367 """
368 return lambda lhs, rhs: fn(utils.MergeTime(lhs), rhs)
369
370
371 def _MakeComparisonChecks(fn):
372 """Prepares flag-specific comparisons using a comparison function.
373
374 """
375 return [
376 (QFF_SPLIT_TIMESTAMP, _MakeSplitTimestampComparison(fn),
377 _PrepareSplitTimestamp),
378 (QFF_JOB_ID, lambda lhs, rhs: fn(jstore.ParseJobId(lhs), rhs),
379 jstore.ParseJobId),
380 (None, fn, None),
381 ]
382
383
384 class _FilterCompilerHelper:
385 """Converts a query filter to a callable usable for filtering.
386
387 """
388 # String statement has no effect, pylint: disable=W0105
389
390 #: How deep filters can be nested
391 _LEVELS_MAX = 10
392
393 # Unique identifiers for operator groups
394 (_OPTYPE_LOGIC,
395 _OPTYPE_UNARY,
396 _OPTYPE_BINARY) = range(1, 4)
397
398 """Functions for equality checks depending on field flags.
399
400 List of tuples containing flags and a callable receiving the left- and
401 right-hand side of the operator. The flags are an OR-ed value of C{QFF_*}
402 (e.g. L{QFF_HOSTNAME} or L{QFF_SPLIT_TIMESTAMP}).
403
404 Order matters. The first item with flags will be used. Flags are checked
405 using binary AND.
406
407 """
408 _EQUALITY_CHECKS = [
409 (QFF_HOSTNAME,
410 lambda lhs, rhs: utils.MatchNameComponent(rhs, [lhs],
411 case_sensitive=False),
412 None),
413 (QFF_SPLIT_TIMESTAMP, _MakeSplitTimestampComparison(operator.eq),
414 _PrepareSplitTimestamp),
415 (None, operator.eq, None),
416 ]
417
418 """Known operators
419
420 Operator as key (C{qlang.OP_*}), value a tuple of operator group
421 (C{_OPTYPE_*}) and a group-specific value:
422
423 - C{_OPTYPE_LOGIC}: Callable taking any number of arguments; used by
424 L{_HandleLogicOp}
425 - C{_OPTYPE_UNARY}: Always C{None}; details handled by L{_HandleUnaryOp}
426 - C{_OPTYPE_BINARY}: Callable taking exactly two parameters, the left- and
427 right-hand side of the operator, used by L{_HandleBinaryOp}
428
429 """
430 _OPS = {
431 # Logic operators
432 qlang.OP_OR: (_OPTYPE_LOGIC, compat.any),
433 qlang.OP_AND: (_OPTYPE_LOGIC, compat.all),
434
435 # Unary operators
436 qlang.OP_NOT: (_OPTYPE_UNARY, None),
437 qlang.OP_TRUE: (_OPTYPE_UNARY, None),
438
439 # Binary operators
440 qlang.OP_EQUAL: (_OPTYPE_BINARY, _EQUALITY_CHECKS),
441 qlang.OP_NOT_EQUAL:
442 (_OPTYPE_BINARY, [(flags, compat.partial(_WrapNot, fn), valprepfn)
443 for (flags, fn, valprepfn) in _EQUALITY_CHECKS]),
444 qlang.OP_LT: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.lt)),
445 qlang.OP_LE: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.le)),
446 qlang.OP_GT: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.gt)),
447 qlang.OP_GE: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.ge)),
448 qlang.OP_REGEXP: (_OPTYPE_BINARY, [
449 (None, lambda lhs, rhs: rhs.search(lhs), _PrepareRegex),
450 ]),
451 qlang.OP_CONTAINS: (_OPTYPE_BINARY, [
452 (None, operator.contains, None),
453 ]),
454 }
455
456 def __init__(self, fields):
457 """Initializes this class.
458
459 @param fields: Field definitions (return value of L{_PrepareFieldList})
460
461 """
462 self._fields = fields
463 self._hints = None
464 self._op_handler = None
465
466 def __call__(self, hints, qfilter):
467 """Converts a query filter into a callable function.
468
469 @type hints: L{_FilterHints} or None
470 @param hints: Callbacks doing analysis on filter
471 @type qfilter: list
472 @param qfilter: Filter structure
473 @rtype: callable
474 @return: Function receiving context and item as parameters, returning
475 boolean as to whether item matches filter
476
477 """
478 self._op_handler = {
479 self._OPTYPE_LOGIC:
480 (self._HandleLogicOp, getattr(hints, "NoteLogicOp", None)),
481 self._OPTYPE_UNARY:
482 (self._HandleUnaryOp, getattr(hints, "NoteUnaryOp", None)),
483 self._OPTYPE_BINARY:
484 (self._HandleBinaryOp, getattr(hints, "NoteBinaryOp", None)),
485 }
486
487 try:
488 filter_fn = self._Compile(qfilter, 0)
489 finally:
490 self._op_handler = None
491
492 return filter_fn
493
494 def _Compile(self, qfilter, level):
495 """Inner function for converting filters.
496
497 Calls the correct handler functions for the top-level operator. This
498 function is called recursively (e.g. for logic operators).
499
500 """
501 if not (isinstance(qfilter, (list, tuple)) and qfilter):
502 raise errors.ParameterError("Invalid filter on level %s" % level)
503
504 # Limit recursion
505 if level >= self._LEVELS_MAX:
506 raise errors.ParameterError("Only up to %s levels are allowed (filter"
507 " nested too deep)" % self._LEVELS_MAX)
508
509 # Create copy to be modified
510 operands = qfilter[:]
511 op = operands.pop(0)
512
513 try:
514 (kind, op_data) = self._OPS[op]
515 except KeyError:
516 raise errors.ParameterError("Unknown operator '%s'" % op)
517
518 (handler, hints_cb) = self._op_handler[kind]
519
520 return handler(hints_cb, level, op, op_data, operands)
521
522 def _LookupField(self, name):
523 """Returns a field definition by name.
524
525 """
526 try:
527 return self._fields[name]
528 except KeyError:
529 raise errors.ParameterError("Unknown field '%s'" % name)
530
531 def _HandleLogicOp(self, hints_fn, level, op, op_fn, operands):
532 """Handles logic operators.
533
534 @type hints_fn: callable
535 @param hints_fn: Callback doing some analysis on the filter
536 @type level: integer
537 @param level: Current depth
538 @type op: string
539 @param op: Operator
540 @type op_fn: callable
541 @param op_fn: Function implementing operator
542 @type operands: list
543 @param operands: List of operands
544
545 """
546 if hints_fn:
547 hints_fn(op)
548
549 return compat.partial(_WrapLogicOp, op_fn,
550 [self._Compile(op, level + 1) for op in operands])
551
552 def _HandleUnaryOp(self, hints_fn, level, op, op_fn, operands):
553 """Handles unary operators.
554
555 @type hints_fn: callable
556 @param hints_fn: Callback doing some analysis on the filter
557 @type level: integer
558 @param level: Current depth
559 @type op: string
560 @param op: Operator
561 @type op_fn: callable
562 @param op_fn: Function implementing operator
563 @type operands: list
564 @param operands: List of operands
565
566 """
567 assert op_fn is None
568
569 if len(operands) != 1:
570 raise errors.ParameterError("Unary operator '%s' expects exactly one"
571 " operand" % op)
572
573 if op == qlang.OP_TRUE:
574 (_, datakind, _, retrieval_fn) = self._LookupField(operands[0])
575
576 if hints_fn:
577 hints_fn(op, datakind)
578
579 op_fn = operator.truth
580 arg = retrieval_fn
581 elif op == qlang.OP_NOT:
582 if hints_fn:
583 hints_fn(op, None)
584
585 op_fn = operator.not_
586 arg = self._Compile(operands[0], level + 1)
587 else:
588 raise errors.ProgrammerError("Can't handle operator '%s'" % op)
589
590 return compat.partial(_WrapUnaryOp, op_fn, arg)
591
592 def _HandleBinaryOp(self, hints_fn, level, op, op_data, operands):
593 """Handles binary operators.
594
595 @type hints_fn: callable
596 @param hints_fn: Callback doing some analysis on the filter
597 @type level: integer
598 @param level: Current depth
599 @type op: string
600 @param op: Operator
601 @param op_data: Functions implementing operators
602 @type operands: list
603 @param operands: List of operands
604
605 """
606 # Unused arguments, pylint: disable=W0613
607 try:
608 (name, value) = operands
609 except (ValueError, TypeError):
610 raise errors.ParameterError("Invalid binary operator, expected exactly"
611 " two operands")
612
613 (fdef, datakind, field_flags, retrieval_fn) = self._LookupField(name)
614
615 assert fdef.kind != QFT_UNKNOWN
616
617 # TODO: Type conversions?
618
619 verify_fn = _VERIFY_FN[fdef.kind]
620 if not verify_fn(value):
621 raise errors.ParameterError("Unable to compare field '%s' (type '%s')"
622 " with '%s', expected %s" %
623 (name, fdef.kind, value.__class__.__name__,
624 verify_fn))
625
626 if hints_fn:
627 hints_fn(op, datakind, name, value)
628
629 for (fn_flags, fn, valprepfn) in op_data:
630 if fn_flags is None or fn_flags & field_flags:
631 # Prepare value if necessary (e.g. compile regular expression)
632 if valprepfn:
633 value = valprepfn(value)
634
635 return compat.partial(_WrapBinaryOp, fn, retrieval_fn, value)
636
637 raise errors.ProgrammerError("Unable to find operator implementation"
638 " (op '%s', flags %s)" % (op, field_flags))
639
640
641 def _CompileFilter(fields, hints, qfilter):
642 """Converts a query filter into a callable function.
643
644 See L{_FilterCompilerHelper} for details.
645
646 @rtype: callable
647
648 """
649 return _FilterCompilerHelper(fields)(hints, qfilter)
650
651
652 class Query:
653 def __init__(self, fieldlist, selected, qfilter=None, namefield=None):
654 """Initializes this class.
655
656 The field definition is a dictionary with the field's name as a key and a
657 tuple containing, in order, the field definition object
658 (L{objects.QueryFieldDefinition}, the data kind to help calling code
659 collect data and a retrieval function. The retrieval function is called
660 with two parameters, in order, the data container and the item in container
661 (see L{Query.Query}).
662
663 Users of this class can call L{RequestedData} before preparing the data
664 container to determine what data is needed.
665
666 @type fieldlist: dictionary
667 @param fieldlist: Field definitions
668 @type selected: list of strings
669 @param selected: List of selected fields
670
671 """
672 assert namefield is None or namefield in fieldlist
673
674 self._fields = _GetQueryFields(fieldlist, selected)
675
676 self._filter_fn = None
677 self._requested_names = None
678 self._filter_datakinds = frozenset()
679
680 if qfilter is not None:
681 # Collect requested names if wanted
682 if namefield:
683 hints = _FilterHints(namefield)
684 else:
685 hints = None
686
687 # Build filter function
688 self._filter_fn = _CompileFilter(fieldlist, hints, qfilter)
689 if hints:
690 self._requested_names = hints.RequestedNames()
691 self._filter_datakinds = hints.ReferencedData()
692
693 if namefield is None:
694 self._name_fn = None
695 else:
696 (_, _, _, self._name_fn) = fieldlist[namefield]
697
698 def RequestedNames(self):
699 """Returns all names referenced in the filter.
700
701 If there is no filter or operators are preventing determining the exact
702 names, C{None} is returned.
703
704 """
705 return self._requested_names
706
707 def RequestedData(self):
708 """Gets requested kinds of data.
709
710 @rtype: frozenset
711
712 """
713 return (self._filter_datakinds |
714 frozenset(datakind for (_, datakind, _, _) in self._fields
715 if datakind is not None))
716
717 def GetFields(self):
718 """Returns the list of fields for this query.
719
720 Includes unknown fields.
721
722 @rtype: List of L{objects.QueryFieldDefinition}
723
724 """
725 return GetAllFields(self._fields)
726
727 def Query(self, ctx, sort_by_name=True):
728 """Execute a query.
729
730 @param ctx: Data container passed to field retrieval functions, must
731 support iteration using C{__iter__}
732 @type sort_by_name: boolean
733 @param sort_by_name: Whether to sort by name or keep the input data's
734 ordering
735
736 """
737 sort = (self._name_fn and sort_by_name)
738
739 result = []
740
741 for idx, item in enumerate(ctx):
742 if not (self._filter_fn is None or self._filter_fn(ctx, item)):
743 continue
744
745 row = [_ProcessResult(fn(ctx, item)) for (_, _, _, fn) in self._fields]
746
747 # Verify result
748 if __debug__:
749 _VerifyResultRow(self._fields, row)
750
751 if sort:
752 (status, name) = _ProcessResult(self._name_fn(ctx, item))
753 assert status == constants.RS_NORMAL
754 # TODO: Are there cases where we wouldn't want to use NiceSort?
755 # Answer: if the name field is non-string...
756 result.append((utils.NiceSortKey(name), idx, row))
757 else:
758 result.append(row)
759
760 if not sort:
761 return result
762
763 # TODO: Would "heapq" be more efficient than sorting?
764
765 # Sorting in-place instead of using "sorted()"
766 result.sort()
767
768 assert not result or (len(result[0]) == 3 and len(result[-1]) == 3)
769
770 return map(operator.itemgetter(2), result)
771
772 def OldStyleQuery(self, ctx, sort_by_name=True):
773 """Query with "old" query result format.
774
775 See L{Query.Query} for arguments.
776
777 """
778 unknown = set(fdef.name for (fdef, _, _, _) in self._fields
779 if fdef.kind == QFT_UNKNOWN)
780 if unknown:
781 raise errors.OpPrereqError("Unknown output fields selected: %s" %
782 (utils.CommaJoin(unknown), ),
783 errors.ECODE_INVAL)
784
785 return [[value for (_, value) in row]
786 for row in self.Query(ctx, sort_by_name=sort_by_name)]
787
788
789 def _ProcessResult(value):
790 """Converts result values into externally-visible ones.
791
792 """
793 if value is _FS_UNKNOWN:
794 return (RS_UNKNOWN, None)
795 elif value is _FS_NODATA:
796 return (RS_NODATA, None)
797 elif value is _FS_UNAVAIL:
798 return (RS_UNAVAIL, None)
799 elif value is _FS_OFFLINE:
800 return (RS_OFFLINE, None)
801 else:
802 return (RS_NORMAL, value)
803
804
805 def _VerifyResultRow(fields, row):
806 """Verifies the contents of a query result row.
807
808 @type fields: list
809 @param fields: Field definitions for result
810 @type row: list of tuples
811 @param row: Row data
812
813 """
814 assert len(row) == len(fields)
815 errs = []
816 for ((status, value), (fdef, _, _, _)) in zip(row, fields):
817 if status == RS_NORMAL:
818 if not _VERIFY_FN[fdef.kind](value):
819 errs.append("normal field %s fails validation (value is %s)" %
820 (fdef.name, value))
821 elif value is not None:
822 errs.append("abnormal field %s has a non-None value" % fdef.name)
823 assert not errs, ("Failed validation: %s in row %s" %
824 (utils.CommaJoin(errs), row))
825
826
827 def _FieldDictKey((fdef, _, flags, fn)):
828 """Generates key for field dictionary.
829
830 """
831 assert fdef.name and fdef.title, "Name and title are required"
832 assert FIELD_NAME_RE.match(fdef.name)
833 assert TITLE_RE.match(fdef.title)
834 assert (DOC_RE.match(fdef.doc) and len(fdef.doc.splitlines()) == 1 and
835 fdef.doc.strip() == fdef.doc), \
836 "Invalid description for field '%s'" % fdef.name
837 assert callable(fn)
838 assert (flags & ~QFF_ALL) == 0, "Unknown flags for field '%s'" % fdef.name
839
840 return fdef.name
841
842
843 def _PrepareFieldList(fields, aliases):
844 """Prepares field list for use by L{Query}.
845
846 Converts the list to a dictionary and does some verification.
847
848 @type fields: list of tuples; (L{objects.QueryFieldDefinition}, data
849 kind, retrieval function)
850 @param fields: List of fields, see L{Query.__init__} for a better
851 description
852 @type aliases: list of tuples; (alias, target)
853 @param aliases: list of tuples containing aliases; for each
854 alias/target pair, a duplicate will be created in the field list
855 @rtype: dict
856 @return: Field dictionary for L{Query}
857
858 """
859 if __debug__:
860 duplicates = utils.FindDuplicates(fdef.title.lower()
861 for (fdef, _, _, _) in fields)
862 assert not duplicates, "Duplicate title(s) found: %r" % duplicates
863
864 result = utils.SequenceToDict(fields, key=_FieldDictKey)
865
866 for alias, target in aliases:
867 assert alias not in result, "Alias %s overrides an existing field" % alias
868 assert target in result, "Missing target %s for alias %s" % (target, alias)
869 (fdef, k, flags, fn) = result[target]
870 fdef = fdef.Copy()
871 fdef.name = alias
872 result[alias] = (fdef, k, flags, fn)
873
874 assert len(result) == len(fields) + len(aliases)
875 assert compat.all(name == fdef.name
876 for (name, (fdef, _, _, _)) in result.items())
877
878 return result
879
880
881 def GetQueryResponse(query, ctx, sort_by_name=True):
882 """Prepares the response for a query.
883
884 @type query: L{Query}
885 @param ctx: Data container, see L{Query.Query}
886 @type sort_by_name: boolean
887 @param sort_by_name: Whether to sort by name or keep the input data's
888 ordering
889
890 """
891 return objects.QueryResponse(data=query.Query(ctx, sort_by_name=sort_by_name),
892 fields=query.GetFields()).ToDict()
893
894
895 def QueryFields(fielddefs, selected):
896 """Returns list of available fields.
897
898 @type fielddefs: dict
899 @param fielddefs: Field definitions
900 @type selected: list of strings
901 @param selected: List of selected fields
902 @return: List of L{objects.QueryFieldDefinition}
903
904 """
905 if selected is None:
906 # Client requests all fields, sort by name
907 fdefs = utils.NiceSort(GetAllFields(fielddefs.values()),
908 key=operator.attrgetter("name"))
909 else:
910 # Keep order as requested by client
911 fdefs = Query(fielddefs, selected).GetFields()
912
913 return objects.QueryFieldsResponse(fields=fdefs).ToDict()
914
915
916 def _MakeField(name, title, kind, doc):
917 """Wrapper for creating L{objects.QueryFieldDefinition} instances.
918
919 @param name: Field name as a regular expression
920 @param title: Human-readable title
921 @param kind: Field type
922 @param doc: Human-readable description
923
924 """
925 return objects.QueryFieldDefinition(name=name, title=title, kind=kind,
926 doc=doc)
927
928
929 def _StaticValueInner(value, ctx, _): # pylint: disable=W0613
930 """Returns a static value.
931
932 """
933 return value
934
935
936 def _StaticValue(value):
937 """Prepares a function to return a static value.
938
939 """
940 return compat.partial(_StaticValueInner, value)
941
942
943 def _GetNodeRole(node, master_name):
944 """Determine node role.
945
946 @type node: L{objects.Node}
947 @param node: Node object
948 @type master_name: string
949 @param master_name: Master node name
950
951 """
952 if node.name == master_name:
953 return constants.NR_MASTER
954 elif node.master_candidate:
955 return constants.NR_MCANDIDATE
956 elif node.drained:
957 return constants.NR_DRAINED
958 elif node.offline:
959 return constants.NR_OFFLINE
960 else:
961 return constants.NR_REGULAR
962
963
964 def _GetItemAttr(attr):
965 """Returns a field function to return an attribute of the item.
966
967 @param attr: Attribute name
968
969 """
970 getter = operator.attrgetter(attr)
971 return lambda _, item: getter(item)
972
973
974 def _GetNDParam(name):
975 """Return a field function to return an ND parameter out of the context.
976
977 """
978 def _helper(ctx, _):
979 if ctx.ndparams is None:
980 return _FS_UNAVAIL
981 else:
982 return ctx.ndparams.get(name, None)
983 return _helper
984
985
986 def _BuildNDFields(is_group):
987 """Builds all the ndparam fields.
988
989 @param is_group: whether this is called at group or node level
990
991 """
992 if is_group:
993 field_kind = GQ_CONFIG
994 else:
995 field_kind = NQ_GROUP
996 return [(_MakeField("ndp/%s" % name,
997 constants.NDS_PARAMETER_TITLES.get(name,
998 "ndp/%s" % name),
999 _VTToQFT[kind], "The \"%s\" node parameter" % name),
1000 field_kind, 0, _GetNDParam(name))
1001 for name, kind in constants.NDS_PARAMETER_TYPES.items()]
1002
1003
1004 def _ConvWrapInner(convert, fn, ctx, item):
1005 """Wrapper for converting values.
1006
1007 @param convert: Conversion function receiving value as single parameter
1008 @param fn: Retrieval function
1009
1010 """
1011 value = fn(ctx, item)
1012
1013 # Is the value an abnormal status?
1014 if compat.any(value is fs for fs in _FS_ALL):
1015 # Return right away
1016 return value
1017
1018 # TODO: Should conversion function also receive context, item or both?
1019 return convert(value)
1020
1021
1022 def _ConvWrap(convert, fn):
1023 """Convenience wrapper for L{_ConvWrapInner}.
1024
1025 @param convert: Conversion function receiving value as single parameter
1026 @param fn: Retrieval function
1027
1028 """
1029 return compat.partial(_ConvWrapInner, convert, fn)
1030
1031
1032 def _GetItemTimestamp(getter):
1033 """Returns function for getting timestamp of item.
1034
1035 @type getter: callable
1036 @param getter: Function to retrieve timestamp attribute
1037
1038 """
1039 def fn(_, item):
1040 """Returns a timestamp of item.
1041
1042 """
1043 timestamp = getter(item)
1044 if timestamp is None:
1045 # Old configs might not have all timestamps
1046 return _FS_UNAVAIL
1047 else:
1048 return timestamp
1049
1050 return fn
1051
1052
1053 def _GetItemTimestampFields(datatype):
1054 """Returns common timestamp fields.
1055
1056 @param datatype: Field data type for use by L{Query.RequestedData}
1057
1058 """
1059 return [
1060 (_MakeField("ctime", "CTime", QFT_TIMESTAMP, "Creation timestamp"),
1061 datatype, 0, _GetItemTimestamp(operator.attrgetter("ctime"))),
1062 (_MakeField("mtime", "MTime", QFT_TIMESTAMP, "Modification timestamp"),
1063 datatype, 0, _GetItemTimestamp(operator.attrgetter("mtime"))),
1064 ]
1065
1066
1067 class NodeQueryData:
1068 """Data container for node data queries.
1069
1070 """
1071 def __init__(self, nodes, live_data, master_name, node_to_primary,
1072 node_to_secondary, groups, oob_support, cluster):
1073 """Initializes this class.
1074
1075 """
1076 self.nodes = nodes
1077 self.live_data = live_data
1078 self.master_name = master_name
1079 self.node_to_primary = node_to_primary
1080 self.node_to_secondary = node_to_secondary
1081 self.groups = groups
1082 self.oob_support = oob_support
1083 self.cluster = cluster
1084
1085 # Used for individual rows
1086 self.curlive_data = None
1087 self.ndparams = None
1088
1089 def __iter__(self):
1090 """Iterate over all nodes.
1091
1092 This function has side-effects and only one instance of the resulting
1093 generator should be used at a time.
1094
1095 """
1096 for node in self.nodes:
1097 group = self.groups.get(node.group, None)
1098 if group is None:
1099 self.ndparams = None
1100 else:
1101 self.ndparams = self.cluster.FillND(node, group)
1102 if self.live_data:
1103 self.curlive_data = self.live_data.get(node.name, None)
1104 else:
1105 self.curlive_data = None
1106 yield node
1107
1108
1109 #: Fields that are direct attributes of an L{objects.Node} object
1110 _NODE_SIMPLE_FIELDS = {
1111 "drained": ("Drained", QFT_BOOL, 0, "Whether node is drained"),
1112 "master_candidate": ("MasterC", QFT_BOOL, 0,
1113 "Whether node is a master candidate"),
1114 "master_capable": ("MasterCapable", QFT_BOOL, 0,
1115 "Whether node can become a master candidate"),
1116 "name": ("Node", QFT_TEXT, QFF_HOSTNAME, "Node name"),
1117 "offline": ("Offline", QFT_BOOL, 0, "Whether node is marked offline"),
1118 "serial_no": ("SerialNo", QFT_NUMBER, 0, _SERIAL_NO_DOC % "Node"),
1119 "uuid": ("UUID", QFT_TEXT, 0, "Node UUID"),
1120 "vm_capable": ("VMCapable", QFT_BOOL, 0, "Whether node can host instances"),
1121 }
1122
1123
1124 #: Fields requiring talking to the node
1125 # Note that none of these are available for non-vm_capable nodes
1126 _NODE_LIVE_FIELDS = {
1127 "bootid": ("BootID", QFT_TEXT, "bootid",
1128 "Random UUID renewed for each system reboot, can be used"
1129 " for detecting reboots by tracking changes"),
1130 "cnodes": ("CNodes", QFT_NUMBER, "cpu_nodes",
1131 "Number of NUMA domains on node (if exported by hypervisor)"),
1132 "csockets": ("CSockets", QFT_NUMBER, "cpu_sockets",
1133 "Number of physical CPU sockets (if exported by hypervisor)"),
1134 "ctotal": ("CTotal", QFT_NUMBER, "cpu_total", "Number of logical processors"),
1135 "dfree": ("DFree", QFT_UNIT, "vg_free",
1136 "Available disk space in volume group"),
1137 "dtotal": ("DTotal", QFT_UNIT, "vg_size",
1138 "Total disk space in volume group used for instance disk"
1139 " allocation"),
1140 "mfree": ("MFree", QFT_UNIT, "memory_free",
1141 "Memory available for instance allocations"),
1142 "mnode": ("MNode", QFT_UNIT, "memory_dom0",
1143 "Amount of memory used by node (dom0 for Xen)"),
1144 "mtotal": ("MTotal", QFT_UNIT, "memory_total",
1145 "Total amount of memory of physical machine"),
1146 }
1147
1148
1149 def _GetGroup(cb):
1150 """Build function for calling another function with an node group.
1151
1152 @param cb: The callback to be called with the nodegroup
1153
1154 """
1155 def fn(ctx, node):
1156 """Get group data for a node.
1157
1158 @type ctx: L{NodeQueryData}
1159 @type inst: L{objects.Node}
1160 @param inst: Node object
1161
1162 """
1163 ng = ctx.groups.get(node.group, None)
1164 if ng is None:
1165 # Nodes always have a group, or the configuration is corrupt
1166 return _FS_UNAVAIL
1167
1168 return cb(ctx, node, ng)
1169
1170 return fn
1171
1172
1173 def _GetNodeGroup(ctx, node, ng): # pylint: disable=W0613
1174 """Returns the name of a node's group.
1175
1176 @type ctx: L{NodeQueryData}
1177 @type node: L{objects.Node}
1178 @param node: Node object
1179 @type ng: L{objects.NodeGroup}
1180 @param ng: The node group this node belongs to
1181
1182 """
1183 return ng.name
1184
1185
1186 def _GetNodePower(ctx, node):
1187 """Returns the node powered state
1188
1189 @type ctx: L{NodeQueryData}
1190 @type node: L{objects.Node}
1191 @param node: Node object
1192
1193 """
1194 if ctx.oob_support[node.name]:
1195 return node.powered
1196
1197 return _FS_UNAVAIL
1198
1199
1200 def _GetNdParams(ctx, node, ng):
1201 """Returns the ndparams for this node.
1202
1203 @type ctx: L{NodeQueryData}
1204 @type node: L{objects.Node}
1205 @param node: Node object
1206 @type ng: L{objects.NodeGroup}
1207 @param ng: The node group this node belongs to
1208
1209 """
1210 return ctx.cluster.SimpleFillND(ng.FillND(node))
1211
1212
1213 def _GetLiveNodeField(field, kind, ctx, node):
1214 """Gets the value of a "live" field from L{NodeQueryData}.
1215
1216 @param field: Live field name
1217 @param kind: Data kind, one of L{constants.QFT_ALL}
1218 @type ctx: L{NodeQueryData}
1219 @type node: L{objects.Node}
1220 @param node: Node object
1221
1222 """
1223 if node.offline:
1224 return _FS_OFFLINE
1225
1226 if not node.vm_capable:
1227 return _FS_UNAVAIL
1228
1229 if not ctx.curlive_data:
1230 return _FS_NODATA
1231
1232 try:
1233 value = ctx.curlive_data[field]
1234 except KeyError:
1235 return _FS_UNAVAIL
1236
1237 if kind == QFT_TEXT:
1238 return value
1239
1240 assert kind in (QFT_NUMBER, QFT_UNIT)
1241
1242 # Try to convert into number
1243 try:
1244 return int(value)
1245 except (ValueError, TypeError):
1246 logging.exception("Failed to convert node field '%s' (value %r) to int",
1247 value, field)
1248 return _FS_UNAVAIL
1249
1250
1251 def _GetNodeHvState(_, node):
1252 """Converts node's hypervisor state for query result.
1253
1254 """
1255 hv_state = node.hv_state
1256
1257 if hv_state is None:
1258 return _FS_UNAVAIL
1259
1260 return dict((name, value.ToDict()) for (name, value) in hv_state.items())
1261
1262
1263 def _GetNodeDiskState(_, node):
1264 """Converts node's disk state for query result.
1265
1266 """
1267 disk_state = node.disk_state
1268
1269 if disk_state is None:
1270 return _FS_UNAVAIL
1271
1272 return dict((disk_kind, dict((name, value.ToDict())
1273 for (name, value) in kind_state.items()))
1274 for (disk_kind, kind_state) in disk_state.items())
1275
1276
1277 def _BuildNodeFields():
1278 """Builds list of fields for node queries.
1279
1280 """
1281 fields = [
1282 (_MakeField("pip", "PrimaryIP", QFT_TEXT, "Primary IP address"),
1283 NQ_CONFIG, 0, _GetItemAttr("primary_ip")),
1284 (_MakeField("sip", "SecondaryIP", QFT_TEXT, "Secondary IP address"),
1285 NQ_CONFIG, 0, _GetItemAttr("secondary_ip")),
1286 (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), NQ_CONFIG, 0,
1287 lambda ctx, node: list(node.GetTags())),
1288 (_MakeField("master", "IsMaster", QFT_BOOL, "Whether node is master"),
1289 NQ_CONFIG, 0, lambda ctx, node: node.name == ctx.master_name),
1290 (_MakeField("group", "Group", QFT_TEXT, "Node group"), NQ_GROUP, 0,
1291 _GetGroup(_GetNodeGroup)),
1292 (_MakeField("group.uuid", "GroupUUID", QFT_TEXT, "UUID of node group"),
1293 NQ_CONFIG, 0, _GetItemAttr("group")),
1294 (_MakeField("powered", "Powered", QFT_BOOL,
1295 "Whether node is thought to be powered on"),
1296 NQ_OOB, 0, _GetNodePower),
1297 (_MakeField("ndparams", "NodeParameters", QFT_OTHER,
1298 "Merged node parameters"),
1299 NQ_GROUP, 0, _GetGroup(_GetNdParams)),
1300 (_MakeField("custom_ndparams", "CustomNodeParameters", QFT_OTHER,
1301 "Custom node parameters"),
1302 NQ_GROUP, 0, _GetItemAttr("ndparams")),
1303 (_MakeField("hv_state", "HypervisorState", QFT_OTHER, "Hypervisor state"),
1304 NQ_CONFIG, 0, _GetNodeHvState),
1305 (_MakeField("disk_state", "DiskState", QFT_OTHER, "Disk state"),
1306 NQ_CONFIG, 0, _GetNodeDiskState),
1307 ]
1308
1309 fields.extend(_BuildNDFields(False))
1310
1311 # Node role
1312 role_values = (constants.NR_MASTER, constants.NR_MCANDIDATE,
1313 constants.NR_REGULAR, constants.NR_DRAINED,
1314 constants.NR_OFFLINE)
1315 role_doc = ("Node role; \"%s\" for master, \"%s\" for master candidate,"
1316 " \"%s\" for regular, \"%s\" for drained, \"%s\" for offline" %
1317 role_values)
1318 fields.append((_MakeField("role", "Role", QFT_TEXT, role_doc), NQ_CONFIG, 0,
1319 lambda ctx, node: _GetNodeRole(node, ctx.master_name)))
1320 assert set(role_values) == constants.NR_ALL
1321
1322 def _GetLength(getter):
1323 return lambda ctx, node: len(getter(ctx)[node.name])
1324
1325 def _GetList(getter):
1326 return lambda ctx, node: list(getter(ctx)[node.name])
1327
1328 # Add fields operating on instance lists
1329 for prefix, titleprefix, docword, getter in \
1330 [("p", "Pri", "primary", operator.attrgetter("node_to_primary")),
1331 ("s", "Sec", "secondary", operator.attrgetter("node_to_secondary"))]:
1332 # TODO: Allow filterting by hostname in list
1333 fields.extend([
1334 (_MakeField("%sinst_cnt" % prefix, "%sinst" % prefix.upper(), QFT_NUMBER,
1335 "Number of instances with this node as %s" % docword),
1336 NQ_INST, 0, _GetLength(getter)),
1337 (_MakeField("%sinst_list" % prefix, "%sInstances" % titleprefix,
1338 QFT_OTHER,
1339 "List of instances with this node as %s" % docword),
1340 NQ_INST, 0, _GetList(getter)),
1341 ])
1342
1343 # Add simple fields
1344 fields.extend([
1345 (_MakeField(name, title, kind, doc), NQ_CONFIG, flags, _GetItemAttr(name))
1346 for (name, (title, kind, flags, doc)) in _NODE_SIMPLE_FIELDS.items()
1347 ])
1348
1349 # Add fields requiring live data
1350 fields.extend([
1351 (_MakeField(name, title, kind, doc), NQ_LIVE, 0,
1352 compat.partial(_GetLiveNodeField, nfield, kind))
1353 for (name, (title, kind, nfield, doc)) in _NODE_LIVE_FIELDS.items()
1354 ])
1355
1356 # Add timestamps
1357 fields.extend(_GetItemTimestampFields(NQ_CONFIG))
1358
1359 return _PrepareFieldList(fields, [])
1360
1361
1362 class InstanceQueryData:
1363 """Data container for instance data queries.
1364
1365 """
1366 def __init__(self, instances, cluster, disk_usage, offline_nodes, bad_nodes,
1367 live_data, wrongnode_inst, console, nodes, groups):
1368 """Initializes this class.
1369
1370 @param instances: List of instance objects
1371 @param cluster: Cluster object
1372 @type disk_usage: dict; instance name as key
1373 @param disk_usage: Per-instance disk usage
1374 @type offline_nodes: list of strings
1375 @param offline_nodes: List of offline nodes
1376 @type bad_nodes: list of strings
1377 @param bad_nodes: List of faulty nodes
1378 @type live_data: dict; instance name as key
1379 @param live_data: Per-instance live data
1380 @type wrongnode_inst: set
1381 @param wrongnode_inst: Set of instances running on wrong node(s)
1382 @type console: dict; instance name as key
1383 @param console: Per-instance console information
1384 @type nodes: dict; node name as key
1385 @param nodes: Node objects
1386
1387 """
1388 assert len(set(bad_nodes) & set(offline_nodes)) == len(offline_nodes), \
1389 "Offline nodes not included in bad nodes"
1390 assert not (set(live_data.keys()) & set(bad_nodes)), \
1391 "Found live data for bad or offline nodes"
1392
1393 self.instances = instances
1394 self.cluster = cluster
1395 self.disk_usage = disk_usage
1396 self.offline_nodes = offline_nodes
1397 self.bad_nodes = bad_nodes
1398 self.live_data = live_data
1399 self.wrongnode_inst = wrongnode_inst
1400 self.console = console
1401 self.nodes = nodes
1402 self.groups = groups
1403
1404 # Used for individual rows
1405 self.inst_hvparams = None
1406 self.inst_beparams = None
1407 self.inst_osparams = None
1408 self.inst_nicparams = None
1409
1410 def __iter__(self):
1411 """Iterate over all instances.
1412
1413 This function has side-effects and only one instance of the resulting
1414 generator should be used at a time.
1415
1416 """
1417 for inst in self.instances:
1418 self.inst_hvparams = self.cluster.FillHV(inst, skip_globals=True)
1419 self.inst_beparams = self.cluster.FillBE(inst)
1420 self.inst_osparams = self.cluster.SimpleFillOS(inst.os, inst.osparams)
1421 self.inst_nicparams = [self.cluster.SimpleFillNIC(nic.nicparams)
1422 for nic in inst.nics]
1423
1424 yield inst
1425
1426
1427 def _GetInstOperState(ctx, inst):
1428 """Get instance's operational status.
1429
1430 @type ctx: L{InstanceQueryData}
1431 @type inst: L{objects.Instance}
1432 @param inst: Instance object
1433
1434 """
1435 # Can't use RS_OFFLINE here as it would describe the instance to
1436 # be offline when we actually don't know due to missing data
1437 if inst.primary_node in ctx.bad_nodes:
1438 return _FS_NODATA
1439 else:
1440 return bool(ctx.live_data.get(inst.name))
1441
1442
1443 def _GetInstLiveData(name):
1444 """Build function for retrieving live data.
1445
1446 @type name: string
1447 @param name: Live data field name
1448
1449 """
1450 def fn(ctx, inst):
1451 """Get live data for an instance.
1452
1453 @type ctx: L{InstanceQueryData}
1454 @type inst: L{objects.Instance}
1455 @param inst: Instance object
1456
1457 """
1458 if (inst.primary_node in ctx.bad_nodes or
1459 inst.primary_node in ctx.offline_nodes):
1460 # Can't use RS_OFFLINE here as it would describe the instance to be
1461 # offline when we actually don't know due to missing data
1462 return _FS_NODATA
1463
1464 if inst.name in ctx.live_data:
1465 data = ctx.live_data[inst.name]
1466 if name in data:
1467 return data[name]
1468
1469 return _FS_UNAVAIL
1470
1471 return fn
1472
1473
1474 def _GetInstStatus(ctx, inst):
1475 """Get instance status.
1476
1477 @type ctx: L{InstanceQueryData}
1478 @type inst: L{objects.Instance}
1479 @param inst: Instance object
1480
1481 """
1482 if inst.primary_node in ctx.offline_nodes:
1483 return constants.INSTST_NODEOFFLINE
1484
1485 if inst.primary_node in ctx.bad_nodes:
1486 return constants.INSTST_NODEDOWN
1487
1488 if bool(ctx.live_data.get(inst.name)):
1489 if inst.name in ctx.wrongnode_inst:
1490 return constants.INSTST_WRONGNODE
1491 elif inst.admin_state == constants.ADMINST_UP:
1492 return constants.INSTST_RUNNING
1493 else:
1494 return constants.INSTST_ERRORUP
1495
1496 if inst.admin_state == constants.ADMINST_UP:
1497 return constants.INSTST_ERRORDOWN
1498 elif inst.admin_state == constants.ADMINST_DOWN:
1499 return constants.INSTST_ADMINDOWN
1500
1501 return constants.INSTST_ADMINOFFLINE
1502
1503
1504 def _GetInstDiskSize(index):
1505 """Build function for retrieving disk size.
1506
1507 @type index: int
1508 @param index: Disk index
1509
1510 """
1511 def fn(_, inst):
1512 """Get size of a disk.
1513
1514 @type inst: L{objects.Instance}
1515 @param inst: Instance object
1516
1517 """
1518 try:
1519 return inst.disks[index].size
1520 except IndexError:
1521 return _FS_UNAVAIL
1522
1523 return fn
1524
1525
1526 def _GetInstNic(index, cb):
1527 """Build function for calling another function with an instance NIC.
1528
1529 @type index: int
1530 @param index: NIC index
1531 @type cb: callable
1532 @param cb: Callback
1533
1534 """
1535 def fn(ctx, inst):
1536 """Call helper function with instance NIC.
1537
1538 @type ctx: L{InstanceQueryData}
1539 @type inst: L{objects.Instance}
1540 @param inst: Instance object
1541
1542 """
1543 try:
1544 nic = inst.nics[index]
1545 except IndexError:
1546 return _FS_UNAVAIL
1547
1548 return cb(ctx, index, nic)
1549
1550 return fn
1551
1552
1553 def _GetInstNicNetwork(ctx, _, nic): # pylint: disable=W0613
1554 """Get a NIC's Network.
1555
1556 @type ctx: L{InstanceQueryData}
1557 @type nic: L{objects.NIC}
1558 @param nic: NIC object
1559
1560 """
1561 if nic.network is None:
1562 return _FS_UNAVAIL
1563 else:
1564 return nic.network
1565
1566
1567 def _GetInstNicIp(ctx, _, nic): # pylint: disable=W0613
1568 """Get a NIC's IP address.
1569
1570 @type ctx: L{InstanceQueryData}
1571 @type nic: L{objects.NIC}
1572 @param nic: NIC object
1573
1574 """
1575 if nic.ip is None:
1576 return _FS_UNAVAIL
1577 else:
1578 return nic.ip
1579
1580
1581 def _GetInstNicBridge(ctx, index, _):
1582 """Get a NIC's bridge.
1583
1584 @type ctx: L{InstanceQueryData}
1585 @type index: int
1586 @param index: NIC index
1587
1588 """
1589 assert len(ctx.inst_nicparams) >= index
1590
1591 nicparams = ctx.inst_nicparams[index]
1592
1593 if nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
1594 return nicparams[constants.NIC_LINK]
1595 else:
1596 return _FS_UNAVAIL
1597
1598
1599 def _GetInstAllNicBridges(ctx, inst):
1600 """Get all network bridges for an instance.
1601
1602 @type ctx: L{InstanceQueryData}
1603 @type inst: L{objects.Instance}
1604 @param inst: Instance object
1605
1606 """
1607 assert len(ctx.inst_nicparams) == len(inst.nics)
1608
1609 result = []
1610
1611 for nicp in ctx.inst_nicparams:
1612 if nicp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED:
1613 result.append(nicp[constants.NIC_LINK])
1614 else:
1615 result.append(None)
1616
1617 assert len(result) == len(inst.nics)
1618
1619 return result
1620
1621
1622 def _GetInstNicParam(name):
1623 """Build function for retrieving a NIC parameter.
1624
1625 @type name: string
1626 @param name: Parameter name
1627
1628 """
1629 def fn(ctx, index, _):
1630 """Get a NIC's bridge.
1631
1632 @type ctx: L{InstanceQueryData}
1633 @type inst: L{objects.Instance}
1634 @param inst: Instance object
1635 @type nic: L{objects.NIC}
1636 @param nic: NIC object
1637
1638 """
1639 assert len(ctx.inst_nicparams) >= index
1640 return ctx.inst_nicparams[index][name]
1641
1642 return fn
1643
1644
1645 def _GetInstanceNetworkFields():
1646 """Get instance fields involving network interfaces.
1647
1648 @return: Tuple containing list of field definitions used as input for
1649 L{_PrepareFieldList} and a list of aliases
1650
1651 """
1652 nic_mac_fn = lambda ctx, _, nic: nic.mac
1653 nic_mode_fn = _GetInstNicParam(constants.NIC_MODE)
1654 nic_link_fn = _GetInstNicParam(constants.NIC_LINK)
1655
1656 fields = [
1657 # All NICs
1658 (_MakeField("nic.count", "NICs", QFT_NUMBER,
1659 "Number of network interfaces"),
1660 IQ_CONFIG, 0, lambda ctx, inst: len(inst.nics)),
1661 (_MakeField("nic.macs", "NIC_MACs", QFT_OTHER,
1662 "List containing each network interface's MAC address"),
1663 IQ_CONFIG, 0, lambda ctx, inst: [nic.mac for nic in inst.nics]),
1664 (_MakeField("nic.ips", "NIC_IPs", QFT_OTHER,
1665 "List containing each network interface's IP address"),
1666 IQ_CONFIG, 0, lambda ctx, inst: [nic.ip for nic in inst.nics]),
1667 (_MakeField("nic.modes", "NIC_modes", QFT_OTHER,
1668 "List containing each network interface's mode"), IQ_CONFIG, 0,
1669 lambda ctx, inst: [nicp[constants.NIC_MODE]
1670 for nicp in ctx.inst_nicparams]),
1671 (_MakeField("nic.links", "NIC_links", QFT_OTHER,
1672 "List containing each network interface's link"), IQ_CONFIG, 0,
1673 lambda ctx, inst: [nicp[constants.NIC_LINK]
1674 for nicp in ctx.inst_nicparams]),
1675 (_MakeField("nic.bridges", "NIC_bridges", QFT_OTHER,
1676 "List containing each network interface's bridge"),
1677 IQ_CONFIG, 0, _GetInstAllNicBridges),
1678 (_MakeField("nic.networks", "NIC_networks", QFT_OTHER,
1679 "List containing each interface's network"), IQ_CONFIG, 0,
1680 lambda ctx, inst: [nic.network for nic in inst.nics]),
1681 ]
1682
1683 # NICs by number
1684 for i in range(constants.MAX_NICS):
1685 numtext = utils.FormatOrdinal(i + 1)
1686 fields.extend([
1687 (_MakeField("nic.ip/%s" % i, "NicIP/%s" % i, QFT_TEXT,
1688 "IP address of %s network interface" % numtext),
1689 IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicIp)),
1690 (_MakeField("nic.mac/%s" % i, "NicMAC/%s" % i, QFT_TEXT,
1691 "MAC address of %s network interface" % numtext),
1692 IQ_CONFIG, 0, _GetInstNic(i, nic_mac_fn)),
1693 (_MakeField("nic.mode/%s" % i, "NicMode/%s" % i, QFT_TEXT,
1694 "Mode of %s network interface" % numtext),
1695 IQ_CONFIG, 0, _GetInstNic(i, nic_mode_fn)),
1696 (_MakeField("nic.link/%s" % i, "NicLink/%s" % i, QFT_TEXT,
1697 "Link of %s network interface" % numtext),
1698 IQ_CONFIG, 0, _GetInstNic(i, nic_link_fn)),
1699 (_MakeField("nic.bridge/%s" % i, "NicBridge/%s" % i, QFT_TEXT,
1700 "Bridge of %s network interface" % numtext),
1701 IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicBridge)),
1702 (_MakeField("nic.network/%s" % i, "NicNetwork/%s" % i, QFT_TEXT,
1703 "Network of %s network interface" % numtext),
1704 IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicNetwork)),
1705 ])
1706
1707 aliases = [
1708 # Legacy fields for first NIC
1709 ("ip", "nic.ip/0"),
1710 ("mac", "nic.mac/0"),
1711 ("bridge", "nic.bridge/0"),
1712 ("nic_mode", "nic.mode/0"),
1713 ("nic_link", "nic.link/0"),
1714 ("nic_network", "nic.network/0"),
1715 ]
1716
1717 return (fields, aliases)
1718
1719
1720 def _GetInstDiskUsage(ctx, inst):
1721 """Get disk usage for an instance.
1722
1723 @type ctx: L{InstanceQueryData}
1724 @type inst: L{objects.Instance}
1725 @param inst: Instance object
1726
1727 """
1728 usage = ctx.disk_usage[inst.name]
1729
1730 if usage is None:
1731 usage = 0
1732
1733 return usage
1734
1735
1736 def _GetInstanceConsole(ctx, inst):
1737 """Get console information for instance.
1738
1739 @type ctx: L{InstanceQueryData}
1740 @type inst: L{objects.Instance}
1741 @param inst: Instance object
1742
1743 """
1744 consinfo = ctx.console[inst.name]
1745
1746 if consinfo is None:
1747 return _FS_UNAVAIL
1748
1749 return consinfo
1750
1751
1752 def _GetInstanceDiskFields():
1753 """Get instance fields involving disks.
1754
1755 @return: List of field definitions used as input for L{_PrepareFieldList}
1756
1757 """
1758 fields = [
1759 (_MakeField("disk_usage", "DiskUsage", QFT_UNIT,
1760 "Total disk space used by instance on each of its nodes;"
1761 " this is not the disk size visible to the instance, but"
1762 " the usage on the node"),
1763 IQ_DISKUSAGE, 0, _GetInstDiskUsage),
1764 (_MakeField("disk.count", "Disks", QFT_NUMBER, "Number of disks"),
1765 IQ_CONFIG, 0, lambda ctx, inst: len(inst.disks)),
1766 (_MakeField("disk.sizes", "Disk_sizes", QFT_OTHER, "List of disk sizes"),
1767 IQ_CONFIG, 0, lambda ctx, inst: [disk.size for disk in inst.disks]),
1768 ]
1769
1770 # Disks by number
1771 fields.extend([
1772 (_MakeField("disk.size/%s" % i, "Disk/%s" % i, QFT_UNIT,
1773 "Disk size of %s disk" % utils.FormatOrdinal(i + 1)),
1774 IQ_CONFIG, 0, _GetInstDiskSize(i))
1775 for i in range(constants.MAX_DISKS)
1776 ])
1777
1778 return fields
1779
1780
1781 def _GetInstanceParameterFields():
1782 """Get instance fields involving parameters.
1783
1784 @return: List of field definitions used as input for L{_PrepareFieldList}
1785
1786 """
1787 fields = [
1788 # Filled parameters
1789 (_MakeField("hvparams", "HypervisorParameters", QFT_OTHER,
1790 "Hypervisor parameters (merged)"),
1791 IQ_CONFIG, 0, lambda ctx, _: ctx.inst_hvparams),
1792 (_MakeField("beparams", "BackendParameters", QFT_OTHER,
1793 "Backend parameters (merged)"),
1794 IQ_CONFIG, 0, lambda ctx, _: ctx.inst_beparams),
1795 (_MakeField("osparams", "OpSysParameters", QFT_OTHER,
1796 "Operating system parameters (merged)"),
1797 IQ_CONFIG, 0, lambda ctx, _: ctx.inst_osparams),
1798
1799 # Unfilled parameters
1800 (_MakeField("custom_hvparams", "CustomHypervisorParameters", QFT_OTHER,
1801 "Custom hypervisor parameters"),
1802 IQ_CONFIG, 0, _GetItemAttr("hvparams")),
1803 (_MakeField("custom_beparams", "CustomBackendParameters", QFT_OTHER,
1804 "Custom backend parameters",),
1805 IQ_CONFIG, 0, _GetItemAttr("beparams")),
1806 (_MakeField("custom_osparams", "CustomOpSysParameters", QFT_OTHER,
1807 "Custom operating system parameters",),
1808 IQ_CONFIG, 0, _GetItemAttr("osparams")),
1809 (_MakeField("custom_nicparams", "CustomNicParameters", QFT_OTHER,
1810 "Custom network interface parameters"),
1811 IQ_CONFIG, 0, lambda ctx, inst: [nic.nicparams for nic in inst.nics]),
1812 ]
1813
1814 # HV params
1815 def _GetInstHvParam(name):
1816 return lambda ctx, _: ctx.inst_hvparams.get(name, _FS_UNAVAIL)
1817
1818 fields.extend([
1819 (_MakeField("hv/%s" % name,
1820 constants.HVS_PARAMETER_TITLES.get(name, "hv/%s" % name),
1821 _VTToQFT[kind], "The \"%s\" hypervisor parameter" % name),
1822 IQ_CONFIG, 0, _GetInstHvParam(name))
1823 for name, kind in constants.HVS_PARAMETER_TYPES.items()
1824 if name not in constants.HVC_GLOBALS
1825 ])
1826
1827 # BE params
1828 def _GetInstBeParam(name):
1829 return lambda ctx, _: ctx.inst_beparams.get(name, None)
1830
1831 fields.extend([
1832 (_MakeField("be/%s" % name,
1833 constants.BES_PARAMETER_TITLES.get(name, "be/%s" % name),
1834 _VTToQFT[kind], "The \"%s\" backend parameter" % name),
1835 IQ_CONFIG, 0, _GetInstBeParam(name))
1836 for name, kind in constants.BES_PARAMETER_TYPES.items()
1837 ])
1838
1839 return fields
1840
1841
1842 _INST_SIMPLE_FIELDS = {
1843 "disk_template": ("Disk_template", QFT_TEXT, 0, "Instance disk template"),
1844 "hypervisor": ("Hypervisor", QFT_TEXT, 0, "Hypervisor name"),
1845 "name": ("Instance", QFT_TEXT, QFF_HOSTNAME, "Instance name"),
1846 # Depending on the hypervisor, the port can be None
1847 "network_port": ("Network_port", QFT_OTHER, 0,
1848 "Instance network port if available (e.g. for VNC console)"),
1849 "os": ("OS", QFT_TEXT, 0, "Operating system"),
1850 "serial_no": ("SerialNo", QFT_NUMBER, 0, _SERIAL_NO_DOC % "Instance"),
1851 "uuid": ("UUID", QFT_TEXT, 0, "Instance UUID"),
1852 }
1853
1854
1855 def _GetInstNodeGroup(ctx, default, node_name):
1856 """Gets group UUID of an instance node.
1857
1858 @type ctx: L{InstanceQueryData}
1859 @param default: Default value
1860 @type node_name: string
1861 @param node_name: Node name
1862
1863 """
1864 try:
1865 node = ctx.nodes[node_name]
1866 except KeyError:
1867 return default
1868 else:
1869 return node.group
1870
1871
1872 def _GetInstNodeGroupName(ctx, default, node_name):
1873 """Gets group name of an instance node.
1874
1875 @type ctx: L{InstanceQueryData}
1876 @param default: Default value
1877 @type node_name: string
1878 @param node_name: Node name
1879
1880 """
1881 try:
1882 node = ctx.nodes[node_name]
1883 except KeyError:
1884 return default
1885
1886 try:
1887 group = ctx.groups[node.group]
1888 except KeyError:
1889 return default
1890
1891 return group.name
1892
1893
1894 def _BuildInstanceFields():
1895 """Builds list of fields for instance queries.
1896
1897 """
1898 fields = [
1899 (_MakeField("pnode", "Primary_node", QFT_TEXT, "Primary node"),
1900 IQ_CONFIG, QFF_HOSTNAME, _GetItemAttr("primary_node")),
1901 (_MakeField("pnode.group", "PrimaryNodeGroup", QFT_TEXT,
1902 "Primary node's group"),
1903 IQ_NODES, 0,
1904 lambda ctx, inst: _GetInstNodeGroupName(ctx, _FS_UNAVAIL,
1905 inst.primary_node)),
1906 (_MakeField("pnode.group.uuid", "PrimaryNodeGroupUUID", QFT_TEXT,
1907 "Primary node's group UUID"),
1908 IQ_NODES, 0,
1909 lambda ctx, inst: _GetInstNodeGroup(ctx, _FS_UNAVAIL, inst.primary_node)),
1910 # TODO: Allow filtering by secondary node as hostname
1911 (_MakeField("snodes", "Secondary_Nodes", QFT_OTHER,
1912 "Secondary nodes; usually this will just be one node"),
1913 IQ_CONFIG, 0, lambda ctx, inst: list(inst.secondary_nodes)),
1914 (_MakeField("snodes.group", "SecondaryNodesGroups", QFT_OTHER,
1915 "Node groups of secondary nodes"),
1916 IQ_NODES, 0,
1917 lambda ctx, inst: map(compat.partial(_GetInstNodeGroupName, ctx, None),
1918 inst.secondary_nodes)),
1919 (_MakeField("snodes.group.uuid", "SecondaryNodesGroupsUUID", QFT_OTHER,
1920 "Node group UUIDs of secondary nodes"),
1921 IQ_NODES, 0,
1922 lambda ctx, inst: map(compat.partial(_GetInstNodeGroup, ctx, None),
1923 inst.secondary_nodes)),
1924 (_MakeField("admin_state", "InstanceState", QFT_TEXT,
1925 "Desired state of instance"),
1926 IQ_CONFIG, 0, _GetItemAttr("admin_state")),
1927 (_MakeField("admin_up", "Autostart", QFT_BOOL,
1928 "Desired state of instance"),
1929 IQ_CONFIG, 0, lambda ctx, inst: inst.admin_state == constants.ADMINST_UP),
1930 (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), IQ_CONFIG, 0,
1931 lambda ctx, inst: list(inst.GetTags())),
1932 (_MakeField("console", "Console", QFT_OTHER,
1933 "Instance console information"), IQ_CONSOLE, 0,
1934 _GetInstanceConsole),
1935 ]
1936
1937 # Add simple fields
1938 fields.extend([
1939 (_MakeField(name, title, kind, doc), IQ_CONFIG, flags, _GetItemAttr(name))
1940 for (name, (title, kind, flags, doc)) in _INST_SIMPLE_FIELDS.items()
1941 ])
1942
1943 # Fields requiring talking to the node
1944 fields.extend([
1945 (_MakeField("oper_state", "Running", QFT_BOOL, "Actual state of instance"),
1946 IQ_LIVE, 0, _GetInstOperState),
1947 (_MakeField("oper_ram", "Memory", QFT_UNIT,
1948 "Actual memory usage as seen by hypervisor"),
1949 IQ_LIVE, 0, _GetInstLiveData("memory")),
1950 (_MakeField("oper_vcpus", "VCPUs", QFT_NUMBER,
1951 "Actual number of VCPUs as seen by hypervisor"),
1952 IQ_LIVE, 0, _GetInstLiveData("vcpus")),
1953 ])
1954
1955 # Status field
1956 status_values = (constants.INSTST_RUNNING, constants.INSTST_ADMINDOWN,
1957 constants.INSTST_WRONGNODE, constants.INSTST_ERRORUP,
1958 constants.INSTST_ERRORDOWN, constants.INSTST_NODEDOWN,
1959 constants.INSTST_NODEOFFLINE, constants.INSTST_ADMINOFFLINE)
1960 status_doc = ("Instance status; \"%s\" if instance is set to be running"
1961 " and actually is, \"%s\" if instance is stopped and"
1962 " is not running, \"%s\" if instance running, but not on its"
1963 " designated primary node, \"%s\" if instance should be"
1964 " stopped, but is actually running, \"%s\" if instance should"
1965 " run, but doesn't, \"%s\" if instance's primary node is down,"
1966 " \"%s\" if instance's primary node is marked offline,"
1967 " \"%s\" if instance is offline and does not use dynamic"
1968 " resources" % status_values)
1969 fields.append((_MakeField("status", "Status", QFT_TEXT, status_doc),
1970 IQ_LIVE, 0, _GetInstStatus))
1971 assert set(status_values) == constants.INSTST_ALL, \
1972 "Status documentation mismatch"
1973
1974 (network_fields, network_aliases) = _GetInstanceNetworkFields()
1975
1976 fields.extend(network_fields)
1977 fields.extend(_GetInstanceParameterFields())
1978 fields.extend(_GetInstanceDiskFields())
1979 fields.extend(_GetItemTimestampFields(IQ_CONFIG))
1980
1981 aliases = [
1982 ("vcpus", "be/vcpus"),
1983 ("be/memory", "be/maxmem"),
1984 ("sda_size", "disk.size/0"),
1985 ("sdb_size", "disk.size/1"),
1986 ] + network_aliases
1987
1988 return _PrepareFieldList(fields, aliases)
1989
1990
1991 class LockQueryData:
1992 """Data container for lock data queries.
1993
1994 """
1995 def __init__(self, lockdata):
1996 """Initializes this class.
1997
1998 """
1999 self.lockdata = lockdata
2000
2001 def __iter__(self):
2002 """Iterate over all locks.
2003
2004 """
2005 return iter(self.lockdata)
2006
2007
2008 def _GetLockOwners(_, data):
2009 """Returns a sorted list of a lock's current owners.
2010
2011 """
2012 (_, _, owners, _) = data
2013
2014 if owners:
2015 owners = utils.NiceSort(owners)
2016
2017 return owners
2018
2019
2020 def _GetLockPending(_, data):
2021 """Returns a sorted list of a lock's pending acquires.
2022
2023 """
2024 (_, _, _, pending) = data
2025
2026 if pending:
2027 pending = [(mode, utils.NiceSort(names))
2028 for (mode, names) in pending]
2029
2030 return pending
2031
2032
2033 def _BuildLockFields():
2034 """Builds list of fields for lock queries.
2035
2036 """
2037 return _PrepareFieldList([
2038 # TODO: Lock names are not always hostnames. Should QFF_HOSTNAME be used?
2039 (_MakeField("name", "Name", QFT_TEXT, "Lock name"), None, 0,
2040 lambda ctx, (name, mode, owners, pending): name),
2041 (_MakeField("mode", "Mode", QFT_OTHER,
2042 "Mode in which the lock is currently acquired"
2043 " (exclusive or shared)"),
2044 LQ_MODE, 0, lambda ctx, (name, mode, owners, pending): mode),
2045 (_MakeField("owner", "Owner", QFT_OTHER, "Current lock owner(s)"),
2046 LQ_OWNER, 0, _GetLockOwners),
2047 (_MakeField("pending", "Pending", QFT_OTHER,
2048 "Threads waiting for the lock"),
2049 LQ_PENDING, 0, _GetLockPending),
2050 ], [])
2051
2052
2053 class GroupQueryData:
2054 """Data container for node group data queries.
2055
2056 """
2057 def __init__(self, cluster, groups, group_to_nodes, group_to_instances,
2058 want_diskparams):
2059 """Initializes this class.
2060
2061 @param cluster: Cluster object
2062 @param groups: List of node group objects
2063 @type group_to_nodes: dict; group UUID as key
2064 @param group_to_nodes: Per-group list of nodes
2065 @type group_to_instances: dict; group UUID as key
2066 @param group_to_instances: Per-group list of (primary) instances
2067 @type want_diskparams: bool
2068 @param want_diskparams: Whether diskparamters should be calculated
2069
2070 """
2071 self.groups = groups
2072 self.group_to_nodes = group_to_nodes
2073 self.group_to_instances = group_to_instances
2074 self.cluster = cluster
2075 self.want_diskparams = want_diskparams
2076
2077 # Used for individual rows
2078 self.group_ipolicy = None
2079 self.ndparams = None
2080 self.group_dp = None
2081
2082 def __iter__(self):
2083 """Iterate over all node groups.
2084
2085 This function has side-effects and only one instance of the resulting
2086 generator should be used at a time.
2087
2088 """
2089 for group in self.groups:
2090 self.group_ipolicy = self.cluster.SimpleFillIPolicy(group.ipolicy)
2091 self.ndparams = self.cluster.SimpleFillND(group.ndparams)
2092 if self.want_diskparams:
2093 self.group_dp = self.cluster.SimpleFillDP(group.diskparams)
2094 else:
2095 self.group_dp = None
2096 yield group
2097
2098
2099 _GROUP_SIMPLE_FIELDS = {
2100 "alloc_policy": ("AllocPolicy", QFT_TEXT, "Allocation policy for group"),
2101 "name": ("Group", QFT_TEXT, "Group name"),
2102 "serial_no": ("SerialNo", QFT_NUMBER, _SERIAL_NO_DOC % "Group"),
2103 "uuid": ("UUID", QFT_TEXT, "Group UUID"),
2104 }
2105
2106
2107 def _BuildGroupFields():
2108 """Builds list of fields for node group queries.
2109
2110 """
2111 # Add simple fields
2112 fields = [(_MakeField(name, title, kind, doc), GQ_CONFIG, 0,
2113 _GetItemAttr(name))
2114 for (name, (title, kind, doc)) in _GROUP_SIMPLE_FIELDS.items()]
2115
2116 def _GetLength(getter):
2117 return lambda ctx, group: len(getter(ctx)[group.uuid])
2118
2119 def _GetSortedList(getter):
2120 return lambda ctx, group: utils.NiceSort(getter(ctx)[group.uuid])
2121
2122 group_to_nodes = operator.attrgetter("group_to_nodes")
2123 group_to_instances = operator.attrgetter("group_to_instances")
2124
2125 # Add fields for nodes
2126 fields.extend([
2127 (_MakeField("node_cnt", "Nodes", QFT_NUMBER, "Number of nodes"),
2128 GQ_NODE, 0, _GetLength(group_to_nodes)),
2129 (_MakeField("node_list", "NodeList", QFT_OTHER, "List of nodes"),
2130 GQ_NODE, 0, _GetSortedList(group_to_nodes)),
2131 ])
2132
2133 # Add fields for instances
2134 fields.extend([
2135 (_MakeField("pinst_cnt", "Instances", QFT_NUMBER,
2136 "Number of primary instances"),
2137 GQ_INST, 0, _GetLength(group_to_instances)),
2138 (_MakeField("pinst_list", "InstanceList", QFT_OTHER,
2139 "List of primary instances"),
2140 GQ_INST, 0, _GetSortedList(group_to_instances)),
2141 ])
2142
2143 # Other fields
2144 fields.extend([
2145 (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), GQ_CONFIG, 0,
2146 lambda ctx, group: list(group.GetTags())),
2147 (_MakeField("ipolicy", "InstancePolicy", QFT_OTHER,
2148 "Instance policy limitations (merged)"),
2149 GQ_CONFIG, 0, lambda ctx, _: ctx.group_ipolicy),
2150 (_MakeField("custom_ipolicy", "CustomInstancePolicy", QFT_OTHER,
2151 "Custom instance policy limitations"),
2152 GQ_CONFIG, 0, _GetItemAttr("ipolicy")),
2153 (_MakeField("custom_ndparams", "CustomNDParams", QFT_OTHER,
2154 "Custom node parameters"),
2155 GQ_CONFIG, 0, _GetItemAttr("ndparams")),
2156 (_MakeField("ndparams", "NDParams", QFT_OTHER,
2157 "Node parameters"),
2158 GQ_CONFIG, 0, lambda ctx, _: ctx.ndparams),
2159 (_MakeField("diskparams", "DiskParameters", QFT_OTHER,
2160 "Disk parameters (merged)"),
2161 GQ_DISKPARAMS, 0, lambda ctx, _: ctx.group_dp),
2162 (_MakeField("custom_diskparams", "CustomDiskParameters", QFT_OTHER,
2163 "Custom disk parameters"),
2164 GQ_CONFIG, 0, _GetItemAttr("diskparams")),
2165 ])
2166
2167 # ND parameters
2168 fields.extend(_BuildNDFields(True))
2169
2170 fields.extend(_GetItemTimestampFields(GQ_CONFIG))
2171
2172 return _PrepareFieldList(fields, [])
2173
2174
2175 class OsInfo(objects.ConfigObject):
2176 __slots__ = [
2177 "name",
2178 "valid",
2179 "hidden",
2180 "blacklisted",
2181 "variants",
2182 "api_versions",
2183 "parameters",
2184 "node_status",
2185 ]
2186
2187
2188 def _BuildOsFields():
2189 """Builds list of fields for operating system queries.
2190
2191 """
2192 fields = [
2193 (_MakeField("name", "Name", QFT_TEXT, "Operating system name"),
2194 None, 0, _GetItemAttr("name")),
2195 (_MakeField("valid", "Valid", QFT_BOOL,
2196 "Whether operating system definition is valid"),
2197 None, 0, _GetItemAttr("valid")),
2198 (_MakeField("hidden", "Hidden", QFT_BOOL,
2199 "Whether operating system is hidden"),
2200 None, 0, _GetItemAttr("hidden")),
2201 (_MakeField("blacklisted", "Blacklisted", QFT_BOOL,
2202 "Whether operating system is blacklisted"),
2203 None, 0, _GetItemAttr("blacklisted")),
2204 (_MakeField("variants", "Variants", QFT_OTHER,
2205 "Operating system variants"),
2206 None, 0, _ConvWrap(utils.NiceSort, _GetItemAttr("variants"))),
2207 (_MakeField("api_versions", "ApiVersions", QFT_OTHER,
2208 "Operating system API versions"),
2209 None, 0, _ConvWrap(sorted, _GetItemAttr("api_versions"))),
2210 (_MakeField("parameters", "Parameters", QFT_OTHER,
2211 "Operating system parameters"),
2212 None, 0, _ConvWrap(compat.partial(utils.NiceSort, key=compat.fst),
2213 _GetItemAttr("parameters"))),
2214 (_MakeField("node_status", "NodeStatus", QFT_OTHER,
2215 "Status from node"),
2216 None, 0, _GetItemAttr("node_status")),
2217 ]
2218
2219 return _PrepareFieldList(fields, [])
2220
2221
2222 def _JobUnavailInner(fn, ctx, (job_id, job)): # pylint: disable=W0613
2223 """Return L{_FS_UNAVAIL} if job is None.
2224
2225 When listing specifc jobs (e.g. "gnt-job list 1 2 3"), a job may not be
2226 found, in which case this function converts it to L{_FS_UNAVAIL}.
2227
2228 """
2229 if job is None:
2230 return _FS_UNAVAIL
2231 else:
2232 return fn(job)
2233
2234
2235 def _JobUnavail(inner):
2236 """Wrapper for L{_JobUnavailInner}.
2237
2238 """
2239 return compat.partial(_JobUnavailInner, inner)
2240
2241
2242 def _PerJobOpInner(fn, job):
2243 """Executes a function per opcode in a job.
2244
2245 """
2246 return map(fn, job.ops)
2247
2248
2249 def _PerJobOp(fn):
2250 """Wrapper for L{_PerJobOpInner}.
2251
2252 """
2253 return _JobUnavail(compat.partial(_PerJobOpInner, fn))
2254
2255
2256 def _JobTimestampInner(fn, job):
2257 """Converts unavailable timestamp to L{_FS_UNAVAIL}.
2258
2259 """
2260 timestamp = fn(job)
2261
2262 if timestamp is None:
2263 return _FS_UNAVAIL
2264 else:
2265 return timestamp
2266
2267
2268 def _JobTimestamp(fn):
2269 """Wrapper for L{_JobTimestampInner}.
2270
2271 """
2272 return _JobUnavail(compat.partial(_JobTimestampInner, fn))
2273
2274
2275 def _BuildJobFields():
2276 """Builds list of fields for job queries.
2277
2278 """
2279 fields = [
2280 (_MakeField("id", "ID", QFT_NUMBER, "Job ID"),
2281 None, QFF_JOB_ID, lambda _, (job_id, job): job_id),
2282 (_MakeField("status", "Status", QFT_TEXT, "Job status"),
2283 None, 0, _JobUnavail(lambda job: job.CalcStatus())),
2284 (_MakeField("priority", "Priority", QFT_NUMBER,
2285 ("Current job priority (%s to %s)" %
2286 (constants.OP_PRIO_LOWEST, constants.OP_PRIO_HIGHEST))),
2287 None, 0, _JobUnavail(lambda job: job.CalcPriority())),
2288 (_MakeField("archived", "Archived", QFT_BOOL, "Whether job is archived"),
2289 JQ_ARCHIVED, 0, lambda _, (job_id, job): job.archived),
2290 (_MakeField("ops", "OpCodes", QFT_OTHER, "List of all opcodes"),
2291 None, 0, _PerJobOp(lambda op: op.input.__getstate__())),
2292 (_MakeField("opresult", "OpCode_result", QFT_OTHER,
2293 "List of opcodes results"),
2294 None, 0, _PerJobOp(operator.attrgetter("result"))),
2295 (_MakeField("opstatus", "OpCode_status", QFT_OTHER,
2296 "List of opcodes status"),
2297 None, 0, _PerJobOp(operator.attrgetter("status"))),
2298 (_MakeField("oplog", "OpCode_log", QFT_OTHER,
2299 "List of opcode output logs"),
2300 None, 0, _PerJobOp(operator.attrgetter("log"))),
2301 (_MakeField("opstart", "OpCode_start", QFT_OTHER,
2302 "List of opcode start timestamps (before acquiring locks)"),
2303 None, 0, _PerJobOp(operator.attrgetter("start_timestamp"))),
2304 (_MakeField("opexec", "OpCode_exec", QFT_OTHER,
2305 "List of opcode execution start timestamps (after acquiring"
2306 " locks)"),
2307 None, 0, _PerJobOp(operator.attrgetter("exec_timestamp"))),
2308 (_MakeField("opend", "OpCode_end", QFT_OTHER,
2309 "List of opcode execution end timestamps"),
2310 None, 0, _PerJobOp(operator.attrgetter("end_timestamp"))),
2311 (_MakeField("oppriority", "OpCode_prio", QFT_OTHER,
2312 "List of opcode priorities"),
2313 None, 0, _PerJobOp(operator.attrgetter("priority"))),
2314 (_MakeField("summary", "Summary", QFT_OTHER,
2315 "List of per-opcode summaries"),
2316 None, 0, _PerJobOp(lambda op: op.input.Summary())),
2317 ]
2318
2319 # Timestamp fields
2320 for (name, attr, title, desc) in [
2321 ("received_ts", "received_timestamp", "Received",
2322 "Timestamp of when job was received"),
2323 ("start_ts", "start_timestamp", "Start", "Timestamp of job start"),
2324 ("end_ts", "end_timestamp", "End", "Timestamp of job end"),
2325 ]:
2326 getter = operator.attrgetter(attr)
2327 fields.extend([
2328 (_MakeField(name, title, QFT_OTHER,
2329 "%s (tuple containing seconds and microseconds)" % desc),
2330 None, QFF_SPLIT_TIMESTAMP, _JobTimestamp(getter)),
2331 ])
2332
2333 return _PrepareFieldList(fields, [])
2334
2335
2336 def _GetExportName(_, (node_name, expname)): # pylint: disable=W0613
2337 """Returns an export name if available.
2338
2339 """
2340 if expname is None:
2341 return _FS_UNAVAIL
2342 else:
2343 return expname
2344
2345
2346 def _BuildExportFields():
2347 """Builds list of fields for exports.
2348
2349 """
2350 fields = [
2351 (_MakeField("node", "Node", QFT_TEXT, "Node name"),
2352 None, QFF_HOSTNAME, lambda _, (node_name, expname): node_name),
2353 (_MakeField("export", "Export", QFT_TEXT, "Export name"),
2354 None, 0, _GetExportName),
2355 ]
2356
2357 return _PrepareFieldList(fields, [])
2358
2359
2360 _CLUSTER_VERSION_FIELDS = {
2361 "software_version": ("SoftwareVersion", QFT_TEXT, constants.RELEASE_VERSION,
2362 "Software version"),
2363 "protocol_version": ("ProtocolVersion", QFT_NUMBER,
2364 constants.PROTOCOL_VERSION,
2365 "RPC protocol version"),
2366 "config_version": ("ConfigVersion", QFT_NUMBER, constants.CONFIG_VERSION,
2367 "Configuration format version"),
2368 "os_api_version": ("OsApiVersion", QFT_NUMBER, max(constants.OS_API_VERSIONS),
2369 "API version for OS template scripts"),
2370 "export_version": ("ExportVersion", QFT_NUMBER, constants.EXPORT_VERSION,
2371 "Import/export file format version"),
2372 }
2373
2374
2375 _CLUSTER_SIMPLE_FIELDS = {
2376 "cluster_name": ("Name", QFT_TEXT, QFF_HOSTNAME, "Cluster name"),
2377 "master_node": ("Master", QFT_TEXT, QFF_HOSTNAME, "Master node name"),
2378 "volume_group_name": ("VgName", QFT_TEXT, 0, "LVM volume group name"),
2379 }
2380
2381
2382 class ClusterQueryData:
2383 def __init__(self, cluster, drain_flag, watcher_pause):
2384 """Initializes this class.
2385
2386 @type cluster: L{objects.Cluster}
2387 @param cluster: Instance of cluster object
2388 @type drain_flag: bool
2389 @param drain_flag: Whether job queue is drained
2390 @type watcher_pause: number
2391 @param watcher_pause: Until when watcher is paused (Unix timestamp)
2392
2393 """
2394 self._cluster = cluster
2395 self.drain_flag = drain_flag
2396 self.watcher_pause = watcher_pause
2397
2398 def __iter__(self):
2399 return iter([self._cluster])
2400
2401
2402 def _ClusterWatcherPause(ctx, _):
2403 """Returns until when watcher is paused (if available).
2404
2405 """
2406 if ctx.watcher_pause is None:
2407 return _FS_UNAVAIL
2408 else:
2409 return ctx.watcher_pause
2410
2411
2412 def _BuildClusterFields():
2413 """Builds list of fields for cluster information.
2414
2415 """
2416 fields = [
2417 (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), CQ_CONFIG, 0,
2418 lambda ctx, cluster: list(cluster.GetTags())),
2419 (_MakeField("architecture", "ArchInfo", QFT_OTHER,
2420 "Architecture information"), None, 0,
2421 lambda ctx, _: runtime.GetArchInfo()),
2422 (_MakeField("drain_flag", "QueueDrained", QFT_BOOL,
2423 "Flag whether job queue is drained"), CQ_QUEUE_DRAINED, 0,
2424 lambda ctx, _: ctx.drain_flag),
2425 (_MakeField("watcher_pause", "WatcherPause", QFT_TIMESTAMP,
2426 "Until when watcher is paused"), CQ_WATCHER_PAUSE, 0,
2427 _ClusterWatcherPause),
2428 ]
2429
2430 # Simple fields
2431 fields.extend([
2432 (_MakeField(name, title, kind, doc), CQ_CONFIG, flags, _GetItemAttr(name))
2433 for (name, (title, kind, flags, doc)) in _CLUSTER_SIMPLE_FIELDS.items()
2434 ])
2435
2436 # Version fields
2437 fields.extend([
2438 (_MakeField(name, title, kind, doc), None, 0, _StaticValue(value))
2439 for (name, (title, kind, value, doc)) in _CLUSTER_VERSION_FIELDS.items()
2440 ])
2441
2442 # Add timestamps
2443 fields.extend(_GetItemTimestampFields(CQ_CONFIG))
2444
2445 return _PrepareFieldList(fields, [
2446 ("name", "cluster_name"),
2447 ])
2448
2449
2450 class NetworkQueryData:
2451 """Data container for network data queries.
2452
2453 """
2454 def __init__(self, networks, network_to_groups,
2455 network_to_instances, stats):
2456 """Initializes this class.
2457
2458 @param networks: List of network objects
2459 @type network_to_groups: dict; network UUID as key
2460 @param network_to_groups: Per-network list of groups
2461 @type network_to_instances: dict; network UUID as key
2462 @param network_to_instances: Per-network list of instances
2463 @type stats: dict; network UUID as key
2464 @param stats: Per-network usage statistics
2465
2466 """
2467 self.networks = networks
2468 self.network_to_groups = network_to_groups
2469 self.network_to_instances = network_to_instances
2470 self.stats = stats
2471
2472 def __iter__(self):
2473 """Iterate over all networks.
2474
2475 """
2476 for net in self.networks:
2477 if self.stats:
2478 self.curstats = self.stats.get(net.uuid, None)
2479 else:
2480 self.curstats = None
2481 yield net
2482
2483
2484 _NETWORK_SIMPLE_FIELDS = {
2485 "name": ("Network", QFT_TEXT, 0, "The network"),
2486 "network": ("Subnet", QFT_TEXT, 0, "The subnet"),
2487 "gateway": ("Gateway", QFT_OTHER, 0, "The gateway"),
2488 "network6": ("IPv6Subnet", QFT_OTHER, 0, "The ipv6 subnet"),
2489 "gateway6": ("IPv6Gateway", QFT_OTHER, 0, "The ipv6 gateway"),
2490 "mac_prefix": ("MacPrefix", QFT_OTHER, 0, "The mac prefix"),
2491 "network_type": ("NetworkType", QFT_OTHER, 0, "The network type"),
2492 }
2493
2494
2495 _NETWORK_STATS_FIELDS = {
2496 "free_count": ("FreeCount", QFT_NUMBER, 0, "How many addresses are free"),
2497 "reserved_count": ("ReservedCount", QFT_NUMBER, 0, "How many addresses are reserved"),
2498 "map": ("Map", QFT_TEXT, 0, "The actual mapping"),
2499 "external_reservations": ("ExternalReservations", QFT_TEXT, 0, "The external reservations"),
2500 }
2501
2502
2503 def _GetNetworkStatsField(field, kind, ctx, net):
2504 """Gets the value of a "stats" field from L{NetworkQueryData}.
2505
2506 @param field: Field name
2507 @param kind: Data kind, one of L{constants.QFT_ALL}
2508 @type ctx: L{NetworkQueryData}
2509
2510 """
2511
2512 try:
2513 value = ctx.curstats[field]
2514 except KeyError:
2515 return _FS_UNAVAIL
2516
2517 if kind == QFT_TEXT:
2518 return value
2519
2520 assert kind in (QFT_NUMBER, QFT_UNIT)
2521
2522 # Try to convert into number
2523 try:
2524 return int(value)
2525 except (ValueError, TypeError):
2526 logging.exception("Failed to convert network field '%s' (value %r) to int",
2527 value, field)
2528 return _FS_UNAVAIL
2529
2530
2531 def _BuildNetworkFields():
2532 """Builds list of fields for network queries.
2533
2534 """
2535 fields = [
2536 (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), IQ_CONFIG, 0,
2537 lambda ctx, inst: list(inst.GetTags())),
2538 ]
2539
2540 # Add simple fields
2541 fields.extend([
2542 (_MakeField(name, title, kind, doc),
2543 NETQ_CONFIG, 0, _GetItemAttr(name))
2544 for (name, (title, kind, flags, doc)) in _NETWORK_SIMPLE_FIELDS.items()
2545 ])
2546
2547 def _GetLength(getter):
2548 return lambda ctx, network: len(getter(ctx)[network.uuid])
2549
2550 def _GetSortedList(getter):
2551 return lambda ctx, network: utils.NiceSort(getter(ctx)[network.uuid])
2552
2553 network_to_groups = operator.attrgetter("network_to_groups")
2554 network_to_instances = operator.attrgetter("network_to_instances")
2555
2556 # Add fields for node groups
2557 fields.extend([
2558 (_MakeField("group_cnt", "NodeGroups", QFT_NUMBER, "Number of nodegroups"),
2559 NETQ_GROUP, 0, _GetLength(network_to_groups)),
2560 (_MakeField("group_list", "GroupList", QFT_OTHER, "List of nodegroups"),
2561 NETQ_GROUP, 0, _GetSortedList(network_to_groups)),
2562 ])
2563
2564 # Add fields for instances
2565 fields.extend([
2566 (_MakeField("inst_cnt", "Instances", QFT_NUMBER, "Number of instances"),
2567 NETQ_INST, 0, _GetLength(network_to_instances)),
2568 (_MakeField("inst_list", "InstanceList", QFT_OTHER, "List of instances"),
2569 NETQ_INST, 0, _GetSortedList(network_to_instances)),
2570 ])
2571
2572 # Add fields for usage statistics
2573 fields.extend([
2574 (_MakeField(name, title, kind, doc), NETQ_STATS, 0,
2575 compat.partial(_GetNetworkStatsField, name, kind))
2576 for (name, (title, kind, flags, doc)) in _NETWORK_STATS_FIELDS.items()
2577 ])
2578
2579 return _PrepareFieldList(fields, [])
2580
2581 #: Fields for cluster information
2582 CLUSTER_FIELDS = _BuildClusterFields()
2583
2584 #: Fields available for node queries
2585 NODE_FIELDS = _BuildNodeFields()
2586
2587 #: Fields available for instance queries
2588 INSTANCE_FIELDS = _BuildInstanceFields()
2589
2590 #: Fields available for lock queries
2591 LOCK_FIELDS = _BuildLockFields()
2592
2593 #: Fields available for node group queries
2594 GROUP_FIELDS = _BuildGroupFields()
2595
2596 #: Fields available for operating system queries
2597 OS_FIELDS = _BuildOsFields()
2598
2599 #: Fields available for job queries
2600 JOB_FIELDS = _BuildJobFields()
2601
2602 #: Fields available for exports
2603 EXPORT_FIELDS = _BuildExportFields()
2604
2605 #: Fields available for network queries
2606 NETWORK_FIELDS = _BuildNetworkFields()
2607
2608 #: All available resources
2609 ALL_FIELDS = {
2610 constants.QR_CLUSTER: CLUSTER_FIELDS,
2611 constants.QR_INSTANCE: INSTANCE_FIELDS,
2612 constants.QR_NODE: NODE_FIELDS,
2613 constants.QR_LOCK: LOCK_FIELDS,
2614 constants.QR_GROUP: GROUP_FIELDS,
2615 constants.QR_OS: OS_FIELDS,
2616 constants.QR_JOB: JOB_FIELDS,
2617 constants.QR_EXPORT: EXPORT_FIELDS,
2618 constants.QR_NETWORK: NETWORK_FIELDS,
2619 }
2620
2621 #: All available field lists
2622 ALL_FIELD_LISTS = ALL_FIELDS.values()