Merge branch 'stable-2.16' into stable-2.17
[ganeti-github.git] / lib / ovf.py
1 #!/usr/bin/python
2 #
3
4 # Copyright (C) 2011, 2012 Google Inc.
5 # All rights reserved.
6 #
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted provided that the following conditions are
9 # met:
10 #
11 # 1. Redistributions of source code must retain the above copyright notice,
12 # this list of conditions and the following disclaimer.
13 #
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31 """Converter tools between ovf and ganeti config file
32
33 """
34
35 # pylint: disable=F0401, E1101, C0413
36
37 # F0401 because ElementTree is not default for python 2.4
38 # E1101 makes no sense - pylint assumes that ElementTree object is a tuple
39 # C0413 Wrong import position
40
41
42 import ConfigParser
43 import errno
44 import logging
45 import os
46 import os.path
47 import re
48 import shutil
49 import tarfile
50 import tempfile
51 import xml.dom.minidom
52 import xml.parsers.expat
53 try:
54 import xml.etree.ElementTree as ET
55 except ImportError:
56 import elementtree.ElementTree as ET
57
58 try:
59 ParseError = ET.ParseError # pylint: disable=E1103
60 except AttributeError:
61 ParseError = None
62
63 from ganeti import constants
64 from ganeti import errors
65 from ganeti import utils
66 from ganeti import pathutils
67
68
69 # Schemas used in OVF format
70 GANETI_SCHEMA = "http://ganeti"
71 OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1"
72 RASD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/"
73 "CIM_ResourceAllocationSettingData")
74 VSSD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/"
75 "CIM_VirtualSystemSettingData")
76 XML_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance"
77
78 # File extensions in OVF package
79 OVA_EXT = ".ova"
80 OVF_EXT = ".ovf"
81 MF_EXT = ".mf"
82 CERT_EXT = ".cert"
83 COMPRESSION_EXT = ".gz"
84 FILE_EXTENSIONS = [
85 OVF_EXT,
86 MF_EXT,
87 CERT_EXT,
88 ]
89
90 COMPRESSION_TYPE = "gzip"
91 NO_COMPRESSION = [None, "identity"]
92 COMPRESS = "compression"
93 DECOMPRESS = "decompression"
94 ALLOWED_ACTIONS = [COMPRESS, DECOMPRESS]
95
96 VMDK = "vmdk"
97 RAW = "raw"
98 COW = "cow"
99 ALLOWED_FORMATS = [RAW, COW, VMDK]
100
101 # ResourceType values
102 RASD_TYPE = {
103 "vcpus": "3",
104 "memory": "4",
105 "scsi-controller": "6",
106 "ethernet-adapter": "10",
107 "disk": "17",
108 }
109
110 SCSI_SUBTYPE = "lsilogic"
111 VS_TYPE = {
112 "ganeti": "ganeti-ovf",
113 "external": "vmx-04",
114 }
115
116 # AllocationUnits values and conversion
117 ALLOCATION_UNITS = {
118 "b": ["bytes", "b"],
119 "kb": ["kilobytes", "kb", "byte * 2^10", "kibibytes", "kib"],
120 "mb": ["megabytes", "mb", "byte * 2^20", "mebibytes", "mib"],
121 "gb": ["gigabytes", "gb", "byte * 2^30", "gibibytes", "gib"],
122 }
123 CONVERT_UNITS_TO_MB = {
124 "b": lambda x: x / (1024 * 1024),
125 "kb": lambda x: x / 1024,
126 "mb": lambda x: x,
127 "gb": lambda x: x * 1024,
128 }
129
130 # Names of the config fields
131 NAME = "name"
132 OS = "os"
133 HYPERV = "hypervisor"
134 VCPUS = "vcpus"
135 MEMORY = "memory"
136 AUTO_BALANCE = "auto_balance"
137 DISK_TEMPLATE = "disk_template"
138 TAGS = "tags"
139 VERSION = "version"
140
141 # Instance IDs of System and SCSI controller
142 INSTANCE_ID = {
143 "system": 0,
144 "vcpus": 1,
145 "memory": 2,
146 "scsi": 3,
147 }
148
149 # Disk format descriptions
150 DISK_FORMAT = {
151 RAW: "http://en.wikipedia.org/wiki/Byte",
152 VMDK: "http://www.vmware.com/interfaces/specifications/vmdk.html"
153 "#monolithicSparse",
154 COW: "http://www.gnome.org/~markmc/qcow-image-format.html",
155 }
156
157
158 def CheckQemuImg():
159 """ Make sure that qemu-img is present before performing operations.
160
161 @raise errors.OpPrereqError: when qemu-img was not found in the system
162
163 """
164 if not constants.QEMUIMG_PATH:
165 raise errors.OpPrereqError("qemu-img not found at build time, unable"
166 " to continue", errors.ECODE_STATE)
167
168
169 def LinkFile(old_path, prefix=None, suffix=None, directory=None):
170 """Create link with a given prefix and suffix.
171
172 This is a wrapper over os.link. It tries to create a hard link for given file,
173 but instead of rising error when file exists, the function changes the name
174 a little bit.
175
176 @type old_path:string
177 @param old_path: path to the file that is to be linked
178 @type prefix: string
179 @param prefix: prefix of filename for the link
180 @type suffix: string
181 @param suffix: suffix of the filename for the link
182 @type directory: string
183 @param directory: directory of the link
184
185 @raise errors.OpPrereqError: when error on linking is different than
186 "File exists"
187
188 """
189 assert(prefix is not None or suffix is not None)
190 if directory is None:
191 directory = os.getcwd()
192 new_path = utils.PathJoin(directory, "%s%s" % (prefix, suffix))
193 counter = 1
194 while True:
195 try:
196 os.link(old_path, new_path)
197 break
198 except OSError, err:
199 if err.errno == errno.EEXIST:
200 new_path = utils.PathJoin(directory,
201 "%s_%s%s" % (prefix, counter, suffix))
202 counter += 1
203 else:
204 raise errors.OpPrereqError("Error moving the file %s to %s location:"
205 " %s" % (old_path, new_path, err),
206 errors.ECODE_ENVIRON)
207 return new_path
208
209
210 class OVFReader(object):
211 """Reader class for OVF files.
212
213 @type files_list: list
214 @ivar files_list: list of files in the OVF package
215 @type tree: ET.ElementTree
216 @ivar tree: XML tree of the .ovf file
217 @type schema_name: string
218 @ivar schema_name: name of the .ovf file
219 @type input_dir: string
220 @ivar input_dir: directory in which the .ovf file resides
221
222 """
223 def __init__(self, input_path):
224 """Initialiaze the reader - load the .ovf file to XML parser.
225
226 It is assumed that names of manifesto (.mf), certificate (.cert) and ovf
227 files are the same. In order to account any other files as part of the ovf
228 package, they have to be explicitly mentioned in the Resources section
229 of the .ovf file.
230
231 @type input_path: string
232 @param input_path: absolute path to the .ovf file
233
234 @raise errors.OpPrereqError: when .ovf file is not a proper XML file or some
235 of the files mentioned in Resources section do not exist
236
237 """
238 self.tree = ET.ElementTree()
239 try:
240 self.tree.parse(input_path)
241 except (ParseError, xml.parsers.expat.ExpatError), err:
242 raise errors.OpPrereqError("Error while reading %s file: %s" %
243 (OVF_EXT, err), errors.ECODE_ENVIRON)
244
245 # Create a list of all files in the OVF package
246 (input_dir, input_file) = os.path.split(input_path)
247 (input_name, _) = os.path.splitext(input_file)
248 files_directory = utils.ListVisibleFiles(input_dir)
249 files_list = []
250 for file_name in files_directory:
251 (name, extension) = os.path.splitext(file_name)
252 if extension in FILE_EXTENSIONS and name == input_name:
253 files_list.append(file_name)
254 files_list += self._GetAttributes("{%s}References/{%s}File" %
255 (OVF_SCHEMA, OVF_SCHEMA),
256 "{%s}href" % OVF_SCHEMA)
257 for file_name in files_list:
258 file_path = utils.PathJoin(input_dir, file_name)
259 if not os.path.exists(file_path):
260 raise errors.OpPrereqError("File does not exist: %s" % file_path,
261 errors.ECODE_ENVIRON)
262 logging.info("Files in the OVF package: %s", " ".join(files_list))
263 self.files_list = files_list
264 self.input_dir = input_dir
265 self.schema_name = input_name
266
267 def _GetAttributes(self, path, attribute):
268 """Get specified attribute from all nodes accessible using given path.
269
270 Function follows the path from root node to the desired tags using path,
271 then reads the apropriate attribute values.
272
273 @type path: string
274 @param path: path of nodes to visit
275 @type attribute: string
276 @param attribute: attribute for which we gather the information
277 @rtype: list
278 @return: for each accessible tag with the attribute value set, value of the
279 attribute
280
281 """
282 current_list = self.tree.findall(path)
283 results = [x.get(attribute) for x in current_list]
284 return filter(None, results)
285
286 def _GetElementMatchingAttr(self, path, match_attr):
287 """Searches for element on a path that matches certain attribute value.
288
289 Function follows the path from root node to the desired tags using path,
290 then searches for the first one matching the attribute value.
291
292 @type path: string
293 @param path: path of nodes to visit
294 @type match_attr: tuple
295 @param match_attr: pair (attribute, value) for which we search
296 @rtype: ET.ElementTree or None
297 @return: first element matching match_attr or None if nothing matches
298
299 """
300 potential_elements = self.tree.findall(path)
301 (attr, val) = match_attr
302 for elem in potential_elements:
303 if elem.get(attr) == val:
304 return elem
305 return None
306
307 def _GetElementMatchingText(self, path, match_text):
308 """Searches for element on a path that matches certain text value.
309
310 Function follows the path from root node to the desired tags using path,
311 then searches for the first one matching the text value.
312
313 @type path: string
314 @param path: path of nodes to visit
315 @type match_text: tuple
316 @param match_text: pair (node, text) for which we search
317 @rtype: ET.ElementTree or None
318 @return: first element matching match_text or None if nothing matches
319
320 """
321 potential_elements = self.tree.findall(path)
322 (node, text) = match_text
323 for elem in potential_elements:
324 if elem.findtext(node) == text:
325 return elem
326 return None
327
328 @staticmethod
329 def _GetDictParameters(root, schema):
330 """Reads text in all children and creates the dictionary from the contents.
331
332 @type root: ET.ElementTree or None
333 @param root: father of the nodes we want to collect data about
334 @type schema: string
335 @param schema: schema name to be removed from the tag
336 @rtype: dict
337 @return: dictionary containing tags and their text contents, tags have their
338 schema fragment removed or empty dictionary, when root is None
339
340 """
341 if root is None:
342 return {}
343 results = {}
344 for element in list(root):
345 pref_len = len("{%s}" % schema)
346 assert(schema in element.tag)
347 tag = element.tag[pref_len:]
348 results[tag] = element.text
349 return results
350
351 def VerifyManifest(self):
352 """Verifies manifest for the OVF package, if one is given.
353
354 @raise errors.OpPrereqError: if SHA1 checksums do not match
355
356 """
357 if "%s%s" % (self.schema_name, MF_EXT) in self.files_list:
358 logging.warning("Verifying SHA1 checksums, this may take a while")
359 manifest_filename = "%s%s" % (self.schema_name, MF_EXT)
360 manifest_path = utils.PathJoin(self.input_dir, manifest_filename)
361 manifest_content = utils.ReadFile(manifest_path).splitlines()
362 manifest_files = {}
363 regexp = r"SHA1\((\S+)\)= (\S+)"
364 for line in manifest_content:
365 match = re.match(regexp, line)
366 if match:
367 file_name = match.group(1)
368 sha1_sum = match.group(2)
369 manifest_files[file_name] = sha1_sum
370 files_with_paths = [utils.PathJoin(self.input_dir, file_name)
371 for file_name in self.files_list]
372 sha1_sums = utils.FingerprintFiles(files_with_paths)
373 for file_name, value in manifest_files.iteritems():
374 if sha1_sums.get(utils.PathJoin(self.input_dir, file_name)) != value:
375 raise errors.OpPrereqError("SHA1 checksum of %s does not match the"
376 " value in manifest file" % file_name,
377 errors.ECODE_ENVIRON)
378 logging.info("SHA1 checksums verified")
379
380 def GetInstanceName(self):
381 """Provides information about instance name.
382
383 @rtype: string
384 @return: instance name string
385
386 """
387 find_name = "{%s}VirtualSystem/{%s}Name" % (OVF_SCHEMA, OVF_SCHEMA)
388 return self.tree.findtext(find_name)
389
390 def GetDiskTemplate(self):
391 """Returns disk template from .ovf file
392
393 @rtype: string or None
394 @return: name of the template
395 """
396 find_template = ("{%s}GanetiSection/{%s}DiskTemplate" %
397 (GANETI_SCHEMA, GANETI_SCHEMA))
398 return self.tree.findtext(find_template)
399
400 def GetHypervisorData(self):
401 """Provides hypervisor information - hypervisor name and options.
402
403 @rtype: dict
404 @return: dictionary containing name of the used hypervisor and all the
405 specified options
406
407 """
408 hypervisor_search = ("{%s}GanetiSection/{%s}Hypervisor" %
409 (GANETI_SCHEMA, GANETI_SCHEMA))
410 hypervisor_data = self.tree.find(hypervisor_search)
411 if hypervisor_data is None:
412 return {"hypervisor_name": constants.VALUE_AUTO}
413 results = {
414 "hypervisor_name": hypervisor_data.findtext("{%s}Name" % GANETI_SCHEMA,
415 default=constants.VALUE_AUTO),
416 }
417 parameters = hypervisor_data.find("{%s}Parameters" % GANETI_SCHEMA)
418 results.update(self._GetDictParameters(parameters, GANETI_SCHEMA))
419 return results
420
421 def GetOSData(self):
422 """ Provides operating system information - os name and options.
423
424 @rtype: dict
425 @return: dictionary containing name and options for the chosen OS
426
427 """
428 results = {}
429 os_search = ("{%s}GanetiSection/{%s}OperatingSystem" %
430 (GANETI_SCHEMA, GANETI_SCHEMA))
431 os_data = self.tree.find(os_search)
432 if os_data is not None:
433 results["os_name"] = os_data.findtext("{%s}Name" % GANETI_SCHEMA)
434 parameters = os_data.find("{%s}Parameters" % GANETI_SCHEMA)
435 results.update(self._GetDictParameters(parameters, GANETI_SCHEMA))
436 return results
437
438 def GetBackendData(self):
439 """ Provides backend information - vcpus, memory, auto balancing options.
440
441 @rtype: dict
442 @return: dictionary containing options for vcpus, memory and auto balance
443 settings
444
445 """
446 results = {}
447
448 find_vcpus = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" %
449 (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA))
450 match_vcpus = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["vcpus"])
451 vcpus = self._GetElementMatchingText(find_vcpus, match_vcpus)
452 if vcpus is not None:
453 vcpus_count = vcpus.findtext("{%s}VirtualQuantity" % RASD_SCHEMA,
454 default=constants.VALUE_AUTO)
455 else:
456 vcpus_count = constants.VALUE_AUTO
457 results["vcpus"] = str(vcpus_count)
458
459 find_memory = find_vcpus
460 match_memory = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["memory"])
461 memory = self._GetElementMatchingText(find_memory, match_memory)
462 memory_raw = None
463 if memory is not None:
464 alloc_units = memory.findtext("{%s}AllocationUnits" % RASD_SCHEMA)
465 matching_units = [units for units, variants in ALLOCATION_UNITS.items()
466 if alloc_units.lower() in variants]
467 if matching_units == []:
468 raise errors.OpPrereqError("Unit %s for RAM memory unknown" %
469 alloc_units, errors.ECODE_INVAL)
470 units = matching_units[0]
471 memory_raw = int(memory.findtext("{%s}VirtualQuantity" % RASD_SCHEMA,
472 default=constants.VALUE_AUTO))
473 memory_count = CONVERT_UNITS_TO_MB[units](memory_raw)
474 else:
475 memory_count = constants.VALUE_AUTO
476 results["memory"] = str(memory_count)
477
478 find_balance = ("{%s}GanetiSection/{%s}AutoBalance" %
479 (GANETI_SCHEMA, GANETI_SCHEMA))
480 balance = self.tree.findtext(find_balance, default=constants.VALUE_AUTO)
481 results["auto_balance"] = balance
482
483 return results
484
485 def GetTagsData(self):
486 """Provides tags information for instance.
487
488 @rtype: string or None
489 @return: string of comma-separated tags for the instance
490
491 """
492 find_tags = "{%s}GanetiSection/{%s}Tags" % (GANETI_SCHEMA, GANETI_SCHEMA)
493 results = self.tree.findtext(find_tags)
494 if results:
495 return results
496 else:
497 return None
498
499 def GetVersionData(self):
500 """Provides version number read from .ovf file
501
502 @rtype: string
503 @return: string containing the version number
504
505 """
506 find_version = ("{%s}GanetiSection/{%s}Version" %
507 (GANETI_SCHEMA, GANETI_SCHEMA))
508 return self.tree.findtext(find_version)
509
510 def GetNetworkData(self):
511 """Provides data about the network in the OVF instance.
512
513 The method gathers the data about networks used by OVF instance. It assumes
514 that 'name' tag means something - in essence, if it contains one of the
515 words 'bridged' or 'routed' then that will be the mode of this network in
516 Ganeti. The information about the network can be either in GanetiSection or
517 VirtualHardwareSection.
518
519 @rtype: dict
520 @return: dictionary containing all the network information
521
522 """
523 results = {}
524 networks_search = ("{%s}NetworkSection/{%s}Network" %
525 (OVF_SCHEMA, OVF_SCHEMA))
526 network_names = self._GetAttributes(networks_search,
527 "{%s}name" % OVF_SCHEMA)
528 required = ["ip", "mac", "link", "mode", "network"]
529 for (counter, network_name) in enumerate(network_names):
530 network_search = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item"
531 % (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA))
532 ganeti_search = ("{%s}GanetiSection/{%s}Network/{%s}Nic" %
533 (GANETI_SCHEMA, GANETI_SCHEMA, GANETI_SCHEMA))
534 network_match = ("{%s}Connection" % RASD_SCHEMA, network_name)
535 ganeti_match = ("{%s}name" % OVF_SCHEMA, network_name)
536 network_data = self._GetElementMatchingText(network_search, network_match)
537 network_ganeti_data = self._GetElementMatchingAttr(ganeti_search,
538 ganeti_match)
539
540 ganeti_data = {}
541 if network_ganeti_data is not None:
542 ganeti_data["mode"] = network_ganeti_data.findtext("{%s}Mode" %
543 GANETI_SCHEMA)
544 ganeti_data["mac"] = network_ganeti_data.findtext("{%s}MACAddress" %
545 GANETI_SCHEMA)
546 ganeti_data["ip"] = network_ganeti_data.findtext("{%s}IPAddress" %
547 GANETI_SCHEMA)
548 ganeti_data["link"] = network_ganeti_data.findtext("{%s}Link" %
549 GANETI_SCHEMA)
550 ganeti_data["network"] = network_ganeti_data.findtext("{%s}Net" %
551 GANETI_SCHEMA)
552 mac_data = None
553 if network_data is not None:
554 mac_data = network_data.findtext("{%s}Address" % RASD_SCHEMA)
555
556 network_name = network_name.lower()
557
558 # First, some not Ganeti-specific information is collected
559 if constants.NIC_MODE_BRIDGED in network_name:
560 results["nic%s_mode" % counter] = "bridged"
561 elif constants.NIC_MODE_ROUTED in network_name:
562 results["nic%s_mode" % counter] = "routed"
563 results["nic%s_mac" % counter] = mac_data
564
565 # GanetiSection data overrides 'manually' collected data
566 for name, value in ganeti_data.iteritems():
567 results["nic%s_%s" % (counter, name)] = value
568
569 # Bridged network has no IP - unless specifically stated otherwise
570 if (results.get("nic%s_mode" % counter) == "bridged" and
571 not results.get("nic%s_ip" % counter)):
572 results["nic%s_ip" % counter] = constants.VALUE_NONE
573
574 for option in required:
575 if not results.get("nic%s_%s" % (counter, option)):
576 results["nic%s_%s" % (counter, option)] = constants.VALUE_AUTO
577
578 if network_names:
579 results["nic_count"] = str(len(network_names))
580 return results
581
582 def GetDisksNames(self):
583 """Provides list of file names for the disks used by the instance.
584
585 @rtype: list
586 @return: list of file names, as referenced in .ovf file
587
588 """
589 results = []
590 disks_search = "{%s}DiskSection/{%s}Disk" % (OVF_SCHEMA, OVF_SCHEMA)
591 disk_ids = self._GetAttributes(disks_search, "{%s}fileRef" % OVF_SCHEMA)
592 for disk in disk_ids:
593 disk_search = "{%s}References/{%s}File" % (OVF_SCHEMA, OVF_SCHEMA)
594 disk_match = ("{%s}id" % OVF_SCHEMA, disk)
595 disk_elem = self._GetElementMatchingAttr(disk_search, disk_match)
596 if disk_elem is None:
597 raise errors.OpPrereqError("%s file corrupted - disk %s not found in"
598 " references" % (OVF_EXT, disk),
599 errors.ECODE_ENVIRON)
600 disk_name = disk_elem.get("{%s}href" % OVF_SCHEMA)
601 disk_compression = disk_elem.get("{%s}compression" % OVF_SCHEMA)
602 results.append((disk_name, disk_compression))
603 return results
604
605
606 def SubElementText(parent, tag, text, attrib={}, **extra):
607 # pylint: disable=W0102
608 """This is just a wrapper on ET.SubElement that always has text content.
609
610 """
611 if text is None:
612 return None
613 elem = ET.SubElement(parent, tag, attrib=attrib, **extra)
614 elem.text = str(text)
615 return elem
616
617
618 class OVFWriter(object):
619 """Writer class for OVF files.
620
621 @type tree: ET.ElementTree
622 @ivar tree: XML tree that we are constructing
623 @type virtual_system_type: string
624 @ivar virtual_system_type: value of vssd:VirtualSystemType, for external usage
625 in VMWare this requires to be vmx
626 @type hardware_list: list
627 @ivar hardware_list: list of items prepared for VirtualHardwareSection
628 @type next_instance_id: int
629 @ivar next_instance_id: next instance id to be used when creating elements on
630 hardware_list
631
632 """
633 def __init__(self, has_gnt_section):
634 """Initialize the writer - set the top element.
635
636 @type has_gnt_section: bool
637 @param has_gnt_section: if the Ganeti schema should be added - i.e. this
638 means that Ganeti section will be present
639
640 """
641 env_attribs = {
642 "xmlns:xsi": XML_SCHEMA,
643 "xmlns:vssd": VSSD_SCHEMA,
644 "xmlns:rasd": RASD_SCHEMA,
645 "xmlns:ovf": OVF_SCHEMA,
646 "xmlns": OVF_SCHEMA,
647 "xml:lang": "en-US",
648 }
649 if has_gnt_section:
650 env_attribs["xmlns:gnt"] = GANETI_SCHEMA
651 self.virtual_system_type = VS_TYPE["ganeti"]
652 else:
653 self.virtual_system_type = VS_TYPE["external"]
654 self.tree = ET.Element("Envelope", attrib=env_attribs)
655 self.hardware_list = []
656 # INSTANCE_ID contains statically assigned IDs, starting from 0
657 self.next_instance_id = len(INSTANCE_ID) # FIXME: hackish
658
659 def SaveDisksData(self, disks):
660 """Convert disk information to certain OVF sections.
661
662 @type disks: list
663 @param disks: list of dictionaries of disk options from config.ini
664
665 """
666 references = ET.SubElement(self.tree, "References")
667 disk_section = ET.SubElement(self.tree, "DiskSection")
668 SubElementText(disk_section, "Info", "Virtual disk information")
669 for counter, disk in enumerate(disks):
670 file_id = "file%s" % counter
671 disk_id = "disk%s" % counter
672 file_attribs = {
673 "ovf:href": disk["path"],
674 "ovf:size": str(disk["real-size"]),
675 "ovf:id": file_id,
676 }
677 disk_attribs = {
678 "ovf:capacity": str(disk["virt-size"]),
679 "ovf:diskId": disk_id,
680 "ovf:fileRef": file_id,
681 "ovf:format": DISK_FORMAT.get(disk["format"], disk["format"]),
682 }
683 if "compression" in disk:
684 file_attribs["ovf:compression"] = disk["compression"]
685 ET.SubElement(references, "File", attrib=file_attribs)
686 ET.SubElement(disk_section, "Disk", attrib=disk_attribs)
687
688 # Item in VirtualHardwareSection creation
689 disk_item = ET.Element("Item")
690 SubElementText(disk_item, "rasd:ElementName", disk_id)
691 SubElementText(disk_item, "rasd:HostResource", "ovf:/disk/%s" % disk_id)
692 SubElementText(disk_item, "rasd:InstanceID", self.next_instance_id)
693 SubElementText(disk_item, "rasd:Parent", INSTANCE_ID["scsi"])
694 SubElementText(disk_item, "rasd:ResourceType", RASD_TYPE["disk"])
695 self.hardware_list.append(disk_item)
696 self.next_instance_id += 1
697
698 def SaveNetworksData(self, networks):
699 """Convert network information to NetworkSection.
700
701 @type networks: list
702 @param networks: list of dictionaries of network options form config.ini
703
704 """
705 network_section = ET.SubElement(self.tree, "NetworkSection")
706 SubElementText(network_section, "Info", "List of logical networks")
707 for counter, network in enumerate(networks):
708 network_name = "%s%s" % (network["mode"], counter)
709 network_attrib = {"ovf:name": network_name}
710 ET.SubElement(network_section, "Network", attrib=network_attrib)
711
712 # Item in VirtualHardwareSection creation
713 network_item = ET.Element("Item")
714 SubElementText(network_item, "rasd:Address", network["mac"])
715 SubElementText(network_item, "rasd:Connection", network_name)
716 SubElementText(network_item, "rasd:ElementName", network_name)
717 SubElementText(network_item, "rasd:InstanceID", self.next_instance_id)
718 SubElementText(network_item, "rasd:ResourceType",
719 RASD_TYPE["ethernet-adapter"])
720 self.hardware_list.append(network_item)
721 self.next_instance_id += 1
722
723 @staticmethod
724 def _SaveNameAndParams(root, data):
725 """Save name and parameters information under root using data.
726
727 @type root: ET.Element
728 @param root: root element for the Name and Parameters
729 @type data: dict
730 @param data: data from which we gather the values
731
732 """
733 assert(data.get("name"))
734 name = SubElementText(root, "gnt:Name", data["name"])
735 params = ET.SubElement(root, "gnt:Parameters")
736 for name, value in data.iteritems():
737 if name != "name":
738 SubElementText(params, "gnt:%s" % name, value)
739
740 def SaveGanetiData(self, ganeti, networks):
741 """Convert Ganeti-specific information to GanetiSection.
742
743 @type ganeti: dict
744 @param ganeti: dictionary of Ganeti-specific options from config.ini
745 @type networks: list
746 @param networks: list of dictionaries of network options form config.ini
747
748 """
749 ganeti_section = ET.SubElement(self.tree, "gnt:GanetiSection")
750
751 SubElementText(ganeti_section, "gnt:Version", ganeti.get("version"))
752 SubElementText(ganeti_section, "gnt:DiskTemplate",
753 ganeti.get("disk_template"))
754 SubElementText(ganeti_section, "gnt:AutoBalance",
755 ganeti.get("auto_balance"))
756 SubElementText(ganeti_section, "gnt:Tags", ganeti.get("tags"))
757
758 osys = ET.SubElement(ganeti_section, "gnt:OperatingSystem")
759 self._SaveNameAndParams(osys, ganeti["os"])
760
761 hypervisor = ET.SubElement(ganeti_section, "gnt:Hypervisor")
762 self._SaveNameAndParams(hypervisor, ganeti["hypervisor"])
763
764 network_section = ET.SubElement(ganeti_section, "gnt:Network")
765 for counter, network in enumerate(networks):
766 network_name = "%s%s" % (network["mode"], counter)
767 nic_attrib = {"ovf:name": network_name}
768 nic = ET.SubElement(network_section, "gnt:Nic", attrib=nic_attrib)
769 SubElementText(nic, "gnt:Mode", network["mode"])
770 SubElementText(nic, "gnt:MACAddress", network["mac"])
771 SubElementText(nic, "gnt:IPAddress", network["ip"])
772 SubElementText(nic, "gnt:Link", network["link"])
773 SubElementText(nic, "gnt:Net", network["network"])
774
775 def SaveVirtualSystemData(self, name, vcpus, memory):
776 """Convert virtual system information to OVF sections.
777
778 @type name: string
779 @param name: name of the instance
780 @type vcpus: int
781 @param vcpus: number of VCPUs
782 @type memory: int
783 @param memory: RAM memory in MB
784
785 """
786 assert(vcpus > 0)
787 assert(memory > 0)
788 vs_attrib = {"ovf:id": name}
789 virtual_system = ET.SubElement(self.tree, "VirtualSystem", attrib=vs_attrib)
790 SubElementText(virtual_system, "Info", "A virtual machine")
791
792 name_section = ET.SubElement(virtual_system, "Name")
793 name_section.text = name
794 os_attrib = {"ovf:id": "0"}
795 os_section = ET.SubElement(virtual_system, "OperatingSystemSection",
796 attrib=os_attrib)
797 SubElementText(os_section, "Info", "Installed guest operating system")
798 hardware_section = ET.SubElement(virtual_system, "VirtualHardwareSection")
799 SubElementText(hardware_section, "Info", "Virtual hardware requirements")
800
801 # System description
802 system = ET.SubElement(hardware_section, "System")
803 SubElementText(system, "vssd:ElementName", "Virtual Hardware Family")
804 SubElementText(system, "vssd:InstanceID", INSTANCE_ID["system"])
805 SubElementText(system, "vssd:VirtualSystemIdentifier", name)
806 SubElementText(system, "vssd:VirtualSystemType", self.virtual_system_type)
807
808 # Item for vcpus
809 vcpus_item = ET.SubElement(hardware_section, "Item")
810 SubElementText(vcpus_item, "rasd:ElementName",
811 "%s virtual CPU(s)" % vcpus)
812 SubElementText(vcpus_item, "rasd:InstanceID", INSTANCE_ID["vcpus"])
813 SubElementText(vcpus_item, "rasd:ResourceType", RASD_TYPE["vcpus"])
814 SubElementText(vcpus_item, "rasd:VirtualQuantity", vcpus)
815
816 # Item for memory
817 memory_item = ET.SubElement(hardware_section, "Item")
818 SubElementText(memory_item, "rasd:AllocationUnits", "byte * 2^20")
819 SubElementText(memory_item, "rasd:ElementName", "%sMB of memory" % memory)
820 SubElementText(memory_item, "rasd:InstanceID", INSTANCE_ID["memory"])
821 SubElementText(memory_item, "rasd:ResourceType", RASD_TYPE["memory"])
822 SubElementText(memory_item, "rasd:VirtualQuantity", memory)
823
824 # Item for scsi controller
825 scsi_item = ET.SubElement(hardware_section, "Item")
826 SubElementText(scsi_item, "rasd:Address", INSTANCE_ID["system"])
827 SubElementText(scsi_item, "rasd:ElementName", "scsi_controller0")
828 SubElementText(scsi_item, "rasd:InstanceID", INSTANCE_ID["scsi"])
829 SubElementText(scsi_item, "rasd:ResourceSubType", SCSI_SUBTYPE)
830 SubElementText(scsi_item, "rasd:ResourceType", RASD_TYPE["scsi-controller"])
831
832 # Other items - from self.hardware_list
833 for item in self.hardware_list:
834 hardware_section.append(item)
835
836 def PrettyXmlDump(self):
837 """Formatter of the XML file.
838
839 @rtype: string
840 @return: XML tree in the form of nicely-formatted string
841
842 """
843 raw_string = ET.tostring(self.tree)
844 parsed_xml = xml.dom.minidom.parseString(raw_string)
845 xml_string = parsed_xml.toprettyxml(indent=" ")
846 text_re = re.compile(r">\n\s+([^<>\s].*?)\n\s+</", re.DOTALL)
847 return text_re.sub(r">\g<1></", xml_string)
848
849
850 class Converter(object):
851 """Converter class for OVF packages.
852
853 Converter is a class above both ImporterOVF and ExporterOVF. It's purpose is
854 to provide a common interface for the two.
855
856 @type options: optparse.Values
857 @ivar options: options parsed from the command line
858 @type output_dir: string
859 @ivar output_dir: directory to which the results of conversion shall be
860 written
861 @type temp_file_manager: L{utils.TemporaryFileManager}
862 @ivar temp_file_manager: container for temporary files created during
863 conversion
864 @type temp_dir: string
865 @ivar temp_dir: temporary directory created then we deal with OVA
866
867 """
868 def __init__(self, input_path, options):
869 """Initialize the converter.
870
871 @type input_path: string
872 @param input_path: path to the Converter input file
873 @type options: optparse.Values
874 @param options: command line options
875
876 @raise errors.OpPrereqError: if file does not exist
877
878 """
879 input_path = os.path.abspath(input_path)
880 if not os.path.isfile(input_path):
881 raise errors.OpPrereqError("File does not exist: %s" % input_path,
882 errors.ECODE_ENVIRON)
883 self.options = options
884 self.temp_file_manager = utils.TemporaryFileManager()
885 self.temp_dir = None
886 self.output_dir = None
887 self._ReadInputData(input_path)
888
889 def _ReadInputData(self, input_path):
890 """Reads the data on which the conversion will take place.
891
892 @type input_path: string
893 @param input_path: absolute path to the Converter input file
894
895 """
896 raise NotImplementedError()
897
898 def _CompressDisk(self, disk_path, compression, action):
899 """Performs (de)compression on the disk and returns the new path
900
901 @type disk_path: string
902 @param disk_path: path to the disk
903 @type compression: string
904 @param compression: compression type
905 @type action: string
906 @param action: whether the action is compression or decompression
907 @rtype: string
908 @return: new disk path after (de)compression
909
910 @raise errors.OpPrereqError: disk (de)compression failed or "compression"
911 is not supported
912
913 """
914 assert(action in ALLOWED_ACTIONS)
915 # For now we only support gzip, as it is used in ovftool
916 if compression != COMPRESSION_TYPE:
917 raise errors.OpPrereqError("Unsupported compression type: %s"
918 % compression, errors.ECODE_INVAL)
919 disk_file = os.path.basename(disk_path)
920 if action == DECOMPRESS:
921 (disk_name, _) = os.path.splitext(disk_file)
922 prefix = disk_name
923 elif action == COMPRESS:
924 prefix = disk_file
925 new_path = utils.GetClosedTempfile(suffix=COMPRESSION_EXT, prefix=prefix,
926 dir=self.output_dir)
927 self.temp_file_manager.Add(new_path)
928 args = ["gzip", "-c", disk_path]
929 run_result = utils.RunCmd(args, output=new_path)
930 if run_result.failed:
931 raise errors.OpPrereqError("Disk %s failed with output: %s"
932 % (action, run_result.stderr),
933 errors.ECODE_ENVIRON)
934 logging.info("The %s of the disk is completed", action)
935 return (COMPRESSION_EXT, new_path)
936
937 def _ConvertDisk(self, disk_format, disk_path):
938 """Performes conversion to specified format.
939
940 @type disk_format: string
941 @param disk_format: format to which the disk should be converted
942 @type disk_path: string
943 @param disk_path: path to the disk that should be converted
944 @rtype: string
945 @return path to the output disk
946
947 @raise errors.OpPrereqError: convertion of the disk failed
948
949 """
950 CheckQemuImg()
951 disk_file = os.path.basename(disk_path)
952 (disk_name, disk_extension) = os.path.splitext(disk_file)
953 if disk_extension != disk_format:
954 logging.warning("Conversion of disk image to %s format, this may take"
955 " a while", disk_format)
956
957 new_disk_path = utils.GetClosedTempfile(
958 suffix=".%s" % disk_format, prefix=disk_name, dir=self.output_dir)
959 self.temp_file_manager.Add(new_disk_path)
960 args = [
961 constants.QEMUIMG_PATH,
962 "convert",
963 "-O",
964 disk_format,
965 disk_path,
966 new_disk_path,
967 ]
968 run_result = utils.RunCmd(args, cwd=os.getcwd())
969 if run_result.failed:
970 raise errors.OpPrereqError("Convertion to %s failed, qemu-img output was"
971 ": %s" % (disk_format, run_result.stderr),
972 errors.ECODE_ENVIRON)
973 return (".%s" % disk_format, new_disk_path)
974
975 @staticmethod
976 def _GetDiskQemuInfo(disk_path, regexp):
977 """Figures out some information of the disk using qemu-img.
978
979 @type disk_path: string
980 @param disk_path: path to the disk we want to know the format of
981 @type regexp: string
982 @param regexp: string that has to be matched, it has to contain one group
983 @rtype: string
984 @return: disk format
985
986 @raise errors.OpPrereqError: format information cannot be retrieved
987
988 """
989 CheckQemuImg()
990 args = [constants.QEMUIMG_PATH, "info", disk_path]
991 run_result = utils.RunCmd(args, cwd=os.getcwd())
992 if run_result.failed:
993 raise errors.OpPrereqError("Gathering info about the disk using qemu-img"
994 " failed, output was: %s" % run_result.stderr,
995 errors.ECODE_ENVIRON)
996 result = run_result.output
997 regexp = r"%s" % regexp
998 match = re.search(regexp, result)
999 if match:
1000 disk_format = match.group(1)
1001 else:
1002 raise errors.OpPrereqError("No file information matching %s found in:"
1003 " %s" % (regexp, result),
1004 errors.ECODE_ENVIRON)
1005 return disk_format
1006
1007 def Parse(self):
1008 """Parses the data and creates a structure containing all required info.
1009
1010 """
1011 raise NotImplementedError()
1012
1013 def Save(self):
1014 """Saves the gathered configuration in an apropriate format.
1015
1016 """
1017 raise NotImplementedError()
1018
1019 def Cleanup(self):
1020 """Cleans the temporary directory, if one was created.
1021
1022 """
1023 self.temp_file_manager.Cleanup()
1024 if self.temp_dir:
1025 shutil.rmtree(self.temp_dir)
1026 self.temp_dir = None
1027
1028
1029 class OVFImporter(Converter):
1030 """Converter from OVF to Ganeti config file.
1031
1032 @type input_dir: string
1033 @ivar input_dir: directory in which the .ovf file resides
1034 @type output_dir: string
1035 @ivar output_dir: directory to which the results of conversion shall be
1036 written
1037 @type input_path: string
1038 @ivar input_path: complete path to the .ovf file
1039 @type ovf_reader: L{OVFReader}
1040 @ivar ovf_reader: OVF reader instance collects data from .ovf file
1041 @type results_name: string
1042 @ivar results_name: name of imported instance
1043 @type results_template: string
1044 @ivar results_template: disk template read from .ovf file or command line
1045 arguments
1046 @type results_hypervisor: dict
1047 @ivar results_hypervisor: hypervisor information gathered from .ovf file or
1048 command line arguments
1049 @type results_os: dict
1050 @ivar results_os: operating system information gathered from .ovf file or
1051 command line arguments
1052 @type results_backend: dict
1053 @ivar results_backend: backend information gathered from .ovf file or
1054 command line arguments
1055 @type results_tags: string
1056 @ivar results_tags: string containing instance-specific tags
1057 @type results_version: string
1058 @ivar results_version: version as required by Ganeti import
1059 @type results_network: dict
1060 @ivar results_network: network information gathered from .ovf file or command
1061 line arguments
1062 @type results_disk: dict
1063 @ivar results_disk: disk information gathered from .ovf file or command line
1064 arguments
1065
1066 """
1067 def _ReadInputData(self, input_path):
1068 """Reads the data on which the conversion will take place.
1069
1070 @type input_path: string
1071 @param input_path: absolute path to the .ovf or .ova input file
1072
1073 @raise errors.OpPrereqError: if input file is neither .ovf nor .ova
1074
1075 """
1076 (input_dir, input_file) = os.path.split(input_path)
1077 (_, input_extension) = os.path.splitext(input_file)
1078
1079 if input_extension == OVF_EXT:
1080 logging.info("%s file extension found, no unpacking necessary", OVF_EXT)
1081 self.input_dir = input_dir
1082 self.input_path = input_path
1083 self.temp_dir = None
1084 elif input_extension == OVA_EXT:
1085 logging.info("%s file extension found, proceeding to unpacking", OVA_EXT)
1086 self._UnpackOVA(input_path)
1087 else:
1088 raise errors.OpPrereqError("Unknown file extension; expected %s or %s"
1089 " file" % (OVA_EXT, OVF_EXT),
1090 errors.ECODE_INVAL)
1091 assert ((input_extension == OVA_EXT and self.temp_dir) or
1092 (input_extension == OVF_EXT and not self.temp_dir))
1093 assert self.input_dir in self.input_path
1094
1095 if self.options.output_dir:
1096 self.output_dir = os.path.abspath(self.options.output_dir)
1097 if (os.path.commonprefix([pathutils.EXPORT_DIR, self.output_dir]) !=
1098 pathutils.EXPORT_DIR):
1099 logging.warning("Export path is not under %s directory, import to"
1100 " Ganeti using gnt-backup may fail",
1101 pathutils.EXPORT_DIR)
1102 else:
1103 self.output_dir = pathutils.EXPORT_DIR
1104
1105 self.ovf_reader = OVFReader(self.input_path)
1106 self.ovf_reader.VerifyManifest()
1107
1108 def _UnpackOVA(self, input_path):
1109 """Unpacks the .ova package into temporary directory.
1110
1111 @type input_path: string
1112 @param input_path: path to the .ova package file
1113
1114 @raise errors.OpPrereqError: if file is not a proper tarball, one of the
1115 files in the archive seem malicious (e.g. path starts with '../') or
1116 .ova package does not contain .ovf file
1117
1118 """
1119 input_name = None
1120 if not tarfile.is_tarfile(input_path):
1121 raise errors.OpPrereqError("The provided %s file is not a proper tar"
1122 " archive" % OVA_EXT, errors.ECODE_ENVIRON)
1123 ova_content = tarfile.open(input_path)
1124 temp_dir = tempfile.mkdtemp()
1125 self.temp_dir = temp_dir
1126 for file_name in ova_content.getnames():
1127 file_normname = os.path.normpath(file_name)
1128 try:
1129 utils.PathJoin(temp_dir, file_normname)
1130 except ValueError, err:
1131 raise errors.OpPrereqError("File %s inside %s package is not safe" %
1132 (file_name, OVA_EXT), errors.ECODE_ENVIRON)
1133 if file_name.endswith(OVF_EXT):
1134 input_name = file_name
1135 if not input_name:
1136 raise errors.OpPrereqError("No %s file in %s package found" %
1137 (OVF_EXT, OVA_EXT), errors.ECODE_ENVIRON)
1138 logging.warning("Unpacking the %s archive, this may take a while",
1139 input_path)
1140 self.input_dir = temp_dir
1141 self.input_path = utils.PathJoin(self.temp_dir, input_name)
1142 try:
1143 try:
1144 extract = ova_content.extractall
1145 except AttributeError:
1146 # This is a prehistorical case of using python < 2.5
1147 for member in ova_content.getmembers():
1148 ova_content.extract(member, path=self.temp_dir)
1149 else:
1150 extract(self.temp_dir)
1151 except tarfile.TarError, err:
1152 raise errors.OpPrereqError("Error while extracting %s archive: %s" %
1153 (OVA_EXT, err), errors.ECODE_ENVIRON)
1154 logging.info("OVA package extracted to %s directory", self.temp_dir)
1155
1156 def Parse(self):
1157 """Parses the data and creates a structure containing all required info.
1158
1159 The method reads the information given either as a command line option or as
1160 a part of the OVF description.
1161
1162 @raise errors.OpPrereqError: if some required part of the description of
1163 virtual instance is missing or unable to create output directory
1164
1165 """
1166 self.results_name = self._GetInfo("instance name", self.options.name,
1167 self._ParseNameOptions,
1168 self.ovf_reader.GetInstanceName)
1169 if not self.results_name:
1170 raise errors.OpPrereqError("Name of instance not provided",
1171 errors.ECODE_INVAL)
1172
1173 self.output_dir = utils.PathJoin(self.output_dir, self.results_name)
1174 try:
1175 utils.Makedirs(self.output_dir)
1176 except OSError, err:
1177 raise errors.OpPrereqError("Failed to create directory %s: %s" %
1178 (self.output_dir, err), errors.ECODE_ENVIRON)
1179
1180 self.results_template = self._GetInfo(
1181 "disk template", self.options.disk_template, self._ParseTemplateOptions,
1182 self.ovf_reader.GetDiskTemplate)
1183 if not self.results_template:
1184 logging.info("Disk template not given")
1185
1186 self.results_hypervisor = self._GetInfo(
1187 "hypervisor", self.options.hypervisor, self._ParseHypervisorOptions,
1188 self.ovf_reader.GetHypervisorData)
1189 assert self.results_hypervisor["hypervisor_name"]
1190 if self.results_hypervisor["hypervisor_name"] == constants.VALUE_AUTO:
1191 logging.debug("Default hypervisor settings from the cluster will be used")
1192
1193 self.results_os = self._GetInfo(
1194 "OS", self.options.os, self._ParseOSOptions, self.ovf_reader.GetOSData)
1195 if not self.results_os.get("os_name"):
1196 raise errors.OpPrereqError("OS name must be provided",
1197 errors.ECODE_INVAL)
1198
1199 self.results_backend = self._GetInfo(
1200 "backend", self.options.beparams,
1201 self._ParseBackendOptions, self.ovf_reader.GetBackendData)
1202 assert self.results_backend.get("vcpus")
1203 assert self.results_backend.get("memory")
1204 assert self.results_backend.get("auto_balance") is not None
1205
1206 self.results_tags = self._GetInfo(
1207 "tags", self.options.tags, self._ParseTags, self.ovf_reader.GetTagsData)
1208
1209 ovf_version = self.ovf_reader.GetVersionData()
1210 if ovf_version:
1211 self.results_version = ovf_version
1212 else:
1213 self.results_version = constants.EXPORT_VERSION
1214
1215 self.results_network = self._GetInfo(
1216 "network", self.options.nics, self._ParseNicOptions,
1217 self.ovf_reader.GetNetworkData, ignore_test=self.options.no_nics)
1218
1219 self.results_disk = self._GetInfo(
1220 "disk", self.options.disks, self._ParseDiskOptions, self._GetDiskInfo,
1221 ignore_test=self.results_template == constants.DT_DISKLESS)
1222
1223 if not self.results_disk and not self.results_network:
1224 raise errors.OpPrereqError("Either disk specification or network"
1225 " description must be present",
1226 errors.ECODE_STATE)
1227
1228 @staticmethod
1229 def _GetInfo(name, cmd_arg, cmd_function, nocmd_function,
1230 ignore_test=False):
1231 """Get information about some section - e.g. disk, network, hypervisor.
1232
1233 @type name: string
1234 @param name: name of the section
1235 @type cmd_arg: dict
1236 @param cmd_arg: command line argument specific for section 'name'
1237 @type cmd_function: callable
1238 @param cmd_function: function to call if 'cmd_args' exists
1239 @type nocmd_function: callable
1240 @param nocmd_function: function to call if 'cmd_args' is not there
1241
1242 """
1243 if ignore_test:
1244 logging.info("Information for %s will be ignored", name)
1245 return {}
1246 if cmd_arg:
1247 logging.info("Information for %s will be parsed from command line", name)
1248 results = cmd_function()
1249 else:
1250 logging.info("Information for %s will be parsed from %s file",
1251 name, OVF_EXT)
1252 results = nocmd_function()
1253 logging.info("Options for %s were succesfully read", name)
1254 return results
1255
1256 def _ParseNameOptions(self):
1257 """Returns name if one was given in command line.
1258
1259 @rtype: string
1260 @return: name of an instance
1261
1262 """
1263 return self.options.name
1264
1265 def _ParseTemplateOptions(self):
1266 """Returns disk template if one was given in command line.
1267
1268 @rtype: string
1269 @return: disk template name
1270
1271 """
1272 return self.options.disk_template
1273
1274 def _ParseHypervisorOptions(self):
1275 """Parses hypervisor options given in a command line.
1276
1277 @rtype: dict
1278 @return: dictionary containing name of the chosen hypervisor and all the
1279 options
1280
1281 """
1282 assert isinstance(self.options.hypervisor, tuple)
1283 assert len(self.options.hypervisor) == 2
1284 results = {}
1285 if self.options.hypervisor[0]:
1286 results["hypervisor_name"] = self.options.hypervisor[0]
1287 else:
1288 results["hypervisor_name"] = constants.VALUE_AUTO
1289 results.update(self.options.hypervisor[1])
1290 return results
1291
1292 def _ParseOSOptions(self):
1293 """Parses OS options given in command line.
1294
1295 @rtype: dict
1296 @return: dictionary containing name of chosen OS and all its options
1297
1298 """
1299 assert self.options.os
1300 results = {}
1301 results["os_name"] = self.options.os
1302 results.update(self.options.osparams)
1303 return results
1304
1305 def _ParseBackendOptions(self):
1306 """Parses backend options given in command line.
1307
1308 @rtype: dict
1309 @return: dictionary containing vcpus, memory and auto-balance options
1310
1311 """
1312 assert self.options.beparams
1313 backend = {}
1314 backend.update(self.options.beparams)
1315 must_contain = ["vcpus", "memory", "auto_balance"]
1316 for element in must_contain:
1317 if backend.get(element) is None:
1318 backend[element] = constants.VALUE_AUTO
1319 return backend
1320
1321 def _ParseTags(self):
1322 """Returns tags list given in command line.
1323
1324 @rtype: string
1325 @return: string containing comma-separated tags
1326
1327 """
1328 return self.options.tags
1329
1330 def _ParseNicOptions(self):
1331 """Parses network options given in a command line or as a dictionary.
1332
1333 @rtype: dict
1334 @return: dictionary of network-related options
1335
1336 """
1337 assert self.options.nics
1338 results = {}
1339 for (nic_id, nic_desc) in self.options.nics:
1340 results["nic%s_mode" % nic_id] = \
1341 nic_desc.get("mode", constants.VALUE_AUTO)
1342 results["nic%s_mac" % nic_id] = nic_desc.get("mac", constants.VALUE_AUTO)
1343 results["nic%s_link" % nic_id] = \
1344 nic_desc.get("link", constants.VALUE_AUTO)
1345 results["nic%s_network" % nic_id] = \
1346 nic_desc.get("network", constants.VALUE_AUTO)
1347 if nic_desc.get("mode") == "bridged":
1348 results["nic%s_ip" % nic_id] = constants.VALUE_NONE
1349 else:
1350 results["nic%s_ip" % nic_id] = constants.VALUE_AUTO
1351 results["nic_count"] = str(len(self.options.nics))
1352 return results
1353
1354 def _ParseDiskOptions(self):
1355 """Parses disk options given in a command line.
1356
1357 @rtype: dict
1358 @return: dictionary of disk-related options
1359
1360 @raise errors.OpPrereqError: disk description does not contain size
1361 information or size information is invalid or creation failed
1362
1363 """
1364 CheckQemuImg()
1365 assert self.options.disks
1366 results = {}
1367 for (disk_id, disk_desc) in self.options.disks:
1368 results["disk%s_ivname" % disk_id] = "disk/%s" % disk_id
1369 if disk_desc.get("size"):
1370 try:
1371 disk_size = utils.ParseUnit(disk_desc["size"])
1372 except ValueError:
1373 raise errors.OpPrereqError("Invalid disk size for disk %s: %s" %
1374 (disk_id, disk_desc["size"]),
1375 errors.ECODE_INVAL)
1376 new_path = utils.PathJoin(self.output_dir, str(disk_id))
1377 args = [
1378 constants.QEMUIMG_PATH,
1379 "create",
1380 "-f",
1381 "raw",
1382 new_path,
1383 disk_size,
1384 ]
1385 run_result = utils.RunCmd(args)
1386 if run_result.failed:
1387 raise errors.OpPrereqError("Creation of disk %s failed, output was:"
1388 " %s" % (new_path, run_result.stderr),
1389 errors.ECODE_ENVIRON)
1390 results["disk%s_size" % disk_id] = str(disk_size)
1391 results["disk%s_dump" % disk_id] = "disk%s.raw" % disk_id
1392 else:
1393 raise errors.OpPrereqError("Disks created for import must have their"
1394 " size specified",
1395 errors.ECODE_INVAL)
1396 results["disk_count"] = str(len(self.options.disks))
1397 return results
1398
1399 def _GetDiskInfo(self):
1400 """Gathers information about disks used by instance, perfomes conversion.
1401
1402 @rtype: dict
1403 @return: dictionary of disk-related options
1404
1405 @raise errors.OpPrereqError: disk is not in the same directory as .ovf file
1406
1407 """
1408 results = {}
1409 disks_list = self.ovf_reader.GetDisksNames()
1410 for (counter, (disk_name, disk_compression)) in enumerate(disks_list):
1411 if os.path.dirname(disk_name):
1412 raise errors.OpPrereqError("Disks are not allowed to have absolute"
1413 " paths or paths outside main OVF"
1414 " directory", errors.ECODE_ENVIRON)
1415 disk, _ = os.path.splitext(disk_name)
1416 disk_path = utils.PathJoin(self.input_dir, disk_name)
1417 if disk_compression not in NO_COMPRESSION:
1418 _, disk_path = self._CompressDisk(disk_path, disk_compression,
1419 DECOMPRESS)
1420 disk, _ = os.path.splitext(disk)
1421 if self._GetDiskQemuInfo(disk_path, r"file format: (\S+)") != "raw":
1422 logging.info("Conversion to raw format is required")
1423 ext, new_disk_path = self._ConvertDisk("raw", disk_path)
1424
1425 final_disk_path = LinkFile(new_disk_path, prefix=disk, suffix=ext,
1426 directory=self.output_dir)
1427 final_name = os.path.basename(final_disk_path)
1428 disk_size = os.path.getsize(final_disk_path) / (1024 * 1024)
1429 results["disk%s_dump" % counter] = final_name
1430 results["disk%s_size" % counter] = str(disk_size)
1431 results["disk%s_ivname" % counter] = "disk/%s" % str(counter)
1432 if disks_list:
1433 results["disk_count"] = str(len(disks_list))
1434 return results
1435
1436 def Save(self):
1437 """Saves all the gathered information in a constant.EXPORT_CONF_FILE file.
1438
1439 @raise errors.OpPrereqError: when saving to config file failed
1440
1441 """
1442 logging.info("Conversion was succesfull, saving %s in %s directory",
1443 constants.EXPORT_CONF_FILE, self.output_dir)
1444 results = {
1445 constants.INISECT_INS: {},
1446 constants.INISECT_BEP: {},
1447 constants.INISECT_EXP: {},
1448 constants.INISECT_OSP: {},
1449 constants.INISECT_HYP: {},
1450 }
1451
1452 results[constants.INISECT_INS].update(self.results_disk)
1453 results[constants.INISECT_INS].update(self.results_network)
1454 results[constants.INISECT_INS]["hypervisor"] = \
1455 self.results_hypervisor["hypervisor_name"]
1456 results[constants.INISECT_INS]["name"] = self.results_name
1457 if self.results_template:
1458 results[constants.INISECT_INS]["disk_template"] = self.results_template
1459 if self.results_tags:
1460 results[constants.INISECT_INS]["tags"] = self.results_tags
1461
1462 results[constants.INISECT_BEP].update(self.results_backend)
1463
1464 results[constants.INISECT_EXP]["os"] = self.results_os["os_name"]
1465 results[constants.INISECT_EXP]["version"] = self.results_version
1466
1467 del self.results_os["os_name"]
1468 results[constants.INISECT_OSP].update(self.results_os)
1469
1470 del self.results_hypervisor["hypervisor_name"]
1471 results[constants.INISECT_HYP].update(self.results_hypervisor)
1472
1473 output_file_name = utils.PathJoin(self.output_dir,
1474 constants.EXPORT_CONF_FILE)
1475
1476 output = []
1477 for section, options in results.iteritems():
1478 output.append("[%s]" % section)
1479 for name, value in options.iteritems():
1480 if value is None:
1481 value = ""
1482 output.append("%s = %s" % (name, value))
1483 output.append("")
1484 output_contents = "\n".join(output)
1485
1486 try:
1487 utils.WriteFile(output_file_name, data=output_contents)
1488 except errors.ProgrammerError, err:
1489 raise errors.OpPrereqError("Saving the config file failed: %s" % err,
1490 errors.ECODE_ENVIRON)
1491
1492 self.Cleanup()
1493
1494
1495 class ConfigParserWithDefaults(ConfigParser.SafeConfigParser):
1496 """This is just a wrapper on SafeConfigParser, that uses default values
1497
1498 """
1499 def get(self, section, options, raw=None, vars=None): # pylint: disable=W0622
1500 try:
1501 result = ConfigParser.SafeConfigParser.get(self, section, options,
1502 raw=raw, vars=vars)
1503 except ConfigParser.NoOptionError:
1504 result = None
1505 return result
1506
1507 def getint(self, section, options):
1508 try:
1509 result = ConfigParser.SafeConfigParser.get(self, section, options)
1510 except ConfigParser.NoOptionError:
1511 result = 0
1512 return int(result)
1513
1514
1515 class OVFExporter(Converter):
1516 """Converter from Ganeti config file to OVF
1517
1518 @type input_dir: string
1519 @ivar input_dir: directory in which the config.ini file resides
1520 @type output_dir: string
1521 @ivar output_dir: directory to which the results of conversion shall be
1522 written
1523 @type packed_dir: string
1524 @ivar packed_dir: if we want OVA package, this points to the real (i.e. not
1525 temp) output directory
1526 @type input_path: string
1527 @ivar input_path: complete path to the config.ini file
1528 @type output_path: string
1529 @ivar output_path: complete path to .ovf file
1530 @type config_parser: L{ConfigParserWithDefaults}
1531 @ivar config_parser: parser for the config.ini file
1532 @type reference_files: list
1533 @ivar reference_files: files referenced in the ovf file
1534 @type results_disk: list
1535 @ivar results_disk: list of dictionaries of disk options from config.ini
1536 @type results_network: list
1537 @ivar results_network: list of dictionaries of network options form config.ini
1538 @type results_name: string
1539 @ivar results_name: name of the instance
1540 @type results_vcpus: string
1541 @ivar results_vcpus: number of VCPUs
1542 @type results_memory: string
1543 @ivar results_memory: RAM memory in MB
1544 @type results_ganeti: dict
1545 @ivar results_ganeti: dictionary of Ganeti-specific options from config.ini
1546
1547 """
1548 def _ReadInputData(self, input_path):
1549 """Reads the data on which the conversion will take place.
1550
1551 @type input_path: string
1552 @param input_path: absolute path to the config.ini input file
1553
1554 @raise errors.OpPrereqError: error when reading the config file
1555
1556 """
1557 input_dir = os.path.dirname(input_path)
1558 self.input_path = input_path
1559 self.input_dir = input_dir
1560 if self.options.output_dir:
1561 self.output_dir = os.path.abspath(self.options.output_dir)
1562 else:
1563 self.output_dir = input_dir
1564 self.config_parser = ConfigParserWithDefaults()
1565 logging.info("Reading configuration from %s file", input_path)
1566 try:
1567 self.config_parser.read(input_path)
1568 except ConfigParser.MissingSectionHeaderError, err:
1569 raise errors.OpPrereqError("Error when trying to read %s: %s" %
1570 (input_path, err), errors.ECODE_ENVIRON)
1571 if self.options.ova_package:
1572 self.temp_dir = tempfile.mkdtemp()
1573 self.packed_dir = self.output_dir
1574 self.output_dir = self.temp_dir
1575
1576 self.ovf_writer = OVFWriter(not self.options.ext_usage)
1577
1578 def _ParseName(self):
1579 """Parses name from command line options or config file.
1580
1581 @rtype: string
1582 @return: name of Ganeti instance
1583
1584 @raise errors.OpPrereqError: if name of the instance is not provided
1585
1586 """
1587 if self.options.name:
1588 name = self.options.name
1589 else:
1590 name = self.config_parser.get(constants.INISECT_INS, NAME)
1591 if name is None:
1592 raise errors.OpPrereqError("No instance name found",
1593 errors.ECODE_ENVIRON)
1594 return name
1595
1596 def _ParseVCPUs(self):
1597 """Parses vcpus number from config file.
1598
1599 @rtype: int
1600 @return: number of virtual CPUs
1601
1602 @raise errors.OpPrereqError: if number of VCPUs equals 0
1603
1604 """
1605 vcpus = self.config_parser.getint(constants.INISECT_BEP, VCPUS)
1606 if vcpus == 0:
1607 raise errors.OpPrereqError("No CPU information found",
1608 errors.ECODE_ENVIRON)
1609 return vcpus
1610
1611 def _ParseMemory(self):
1612 """Parses vcpus number from config file.
1613
1614 @rtype: int
1615 @return: amount of memory in MB
1616
1617 @raise errors.OpPrereqError: if amount of memory equals 0
1618
1619 """
1620 memory = self.config_parser.getint(constants.INISECT_BEP, MEMORY)
1621 if memory == 0:
1622 raise errors.OpPrereqError("No memory information found",
1623 errors.ECODE_ENVIRON)
1624 return memory
1625
1626 def _ParseGaneti(self):
1627 """Parses Ganeti data from config file.
1628
1629 @rtype: dictionary
1630 @return: dictionary of Ganeti-specific options
1631
1632 """
1633 results = {}
1634 # hypervisor
1635 results["hypervisor"] = {}
1636 hyp_name = self.config_parser.get(constants.INISECT_INS, HYPERV)
1637 if hyp_name is None:
1638 raise errors.OpPrereqError("No hypervisor information found",
1639 errors.ECODE_ENVIRON)
1640 results["hypervisor"]["name"] = hyp_name
1641 pairs = self.config_parser.items(constants.INISECT_HYP)
1642 for (name, value) in pairs:
1643 results["hypervisor"][name] = value
1644 # os
1645 results["os"] = {}
1646 os_name = self.config_parser.get(constants.INISECT_EXP, OS)
1647 if os_name is None:
1648 raise errors.OpPrereqError("No operating system information found",
1649 errors.ECODE_ENVIRON)
1650 results["os"]["name"] = os_name
1651 pairs = self.config_parser.items(constants.INISECT_OSP)
1652 for (name, value) in pairs:
1653 results["os"][name] = value
1654 # other
1655 others = [
1656 (constants.INISECT_INS, DISK_TEMPLATE, "disk_template"),
1657 (constants.INISECT_BEP, AUTO_BALANCE, "auto_balance"),
1658 (constants.INISECT_INS, TAGS, "tags"),
1659 (constants.INISECT_EXP, VERSION, "version"),
1660 ]
1661 for (section, element, name) in others:
1662 results[name] = self.config_parser.get(section, element)
1663 return results
1664
1665 def _ParseNetworks(self):
1666 """Parses network data from config file.
1667
1668 @rtype: list
1669 @return: list of dictionaries of network options
1670
1671 @raise errors.OpPrereqError: then network mode is not recognized
1672
1673 """
1674 results = []
1675 counter = 0
1676 while True:
1677 data_link = \
1678 self.config_parser.get(constants.INISECT_INS,
1679 "nic%s_link" % counter)
1680 if data_link is None:
1681 break
1682 results.append({
1683 "mode": self.config_parser.get(constants.INISECT_INS,
1684 "nic%s_mode" % counter),
1685 "mac": self.config_parser.get(constants.INISECT_INS,
1686 "nic%s_mac" % counter),
1687 "ip": self.config_parser.get(constants.INISECT_INS,
1688 "nic%s_ip" % counter),
1689 "network": self.config_parser.get(constants.INISECT_INS,
1690 "nic%s_network" % counter),
1691 "link": data_link,
1692 })
1693 if results[counter]["mode"] not in constants.NIC_VALID_MODES:
1694 raise errors.OpPrereqError("Network mode %s not recognized"
1695 % results[counter]["mode"],
1696 errors.ECODE_INVAL)
1697 counter += 1
1698 return results
1699
1700 def _GetDiskOptions(self, disk_file, compression):
1701 """Convert the disk and gather disk info for .ovf file.
1702
1703 @type disk_file: string
1704 @param disk_file: name of the disk (without the full path)
1705 @type compression: bool
1706 @param compression: whether the disk should be compressed or not
1707
1708 @raise errors.OpPrereqError: when disk image does not exist
1709
1710 """
1711 disk_path = utils.PathJoin(self.input_dir, disk_file)
1712 results = {}
1713 if not os.path.isfile(disk_path):
1714 raise errors.OpPrereqError("Disk image does not exist: %s" % disk_path,
1715 errors.ECODE_ENVIRON)
1716 if os.path.dirname(disk_file):
1717 raise errors.OpPrereqError("Path for the disk: %s contains a directory"
1718 " name" % disk_path, errors.ECODE_ENVIRON)
1719 disk_name, _ = os.path.splitext(disk_file)
1720 ext, new_disk_path = self._ConvertDisk(self.options.disk_format, disk_path)
1721 results["format"] = self.options.disk_format
1722 results["virt-size"] = self._GetDiskQemuInfo(
1723 new_disk_path, r"virtual size: \S+ \((\d+) bytes\)")
1724 if compression:
1725 ext2, new_disk_path = self._CompressDisk(new_disk_path, "gzip",
1726 COMPRESS)
1727 disk_name, _ = os.path.splitext(disk_name)
1728 results["compression"] = "gzip"
1729 ext += ext2
1730 final_disk_path = LinkFile(new_disk_path, prefix=disk_name, suffix=ext,
1731 directory=self.output_dir)
1732 final_disk_name = os.path.basename(final_disk_path)
1733 results["real-size"] = os.path.getsize(final_disk_path)
1734 results["path"] = final_disk_name
1735 self.references_files.append(final_disk_path)
1736 return results
1737
1738 def _ParseDisks(self):
1739 """Parses disk data from config file.
1740
1741 @rtype: list
1742 @return: list of dictionaries of disk options
1743
1744 """
1745 results = []
1746 counter = 0
1747 while True:
1748 disk_file = \
1749 self.config_parser.get(constants.INISECT_INS, "disk%s_dump" % counter)
1750 if disk_file is None:
1751 break
1752 results.append(self._GetDiskOptions(disk_file, self.options.compression))
1753 counter += 1
1754 return results
1755
1756 def Parse(self):
1757 """Parses the data and creates a structure containing all required info.
1758
1759 """
1760 try:
1761 utils.Makedirs(self.output_dir)
1762 except OSError, err:
1763 raise errors.OpPrereqError("Failed to create directory %s: %s" %
1764 (self.output_dir, err), errors.ECODE_ENVIRON)
1765
1766 self.references_files = []
1767 self.results_name = self._ParseName()
1768 self.results_vcpus = self._ParseVCPUs()
1769 self.results_memory = self._ParseMemory()
1770 if not self.options.ext_usage:
1771 self.results_ganeti = self._ParseGaneti()
1772 self.results_network = self._ParseNetworks()
1773 self.results_disk = self._ParseDisks()
1774
1775 def _PrepareManifest(self, path):
1776 """Creates manifest for all the files in OVF package.
1777
1778 @type path: string
1779 @param path: path to manifesto file
1780
1781 @raise errors.OpPrereqError: if error occurs when writing file
1782
1783 """
1784 logging.info("Preparing manifest for the OVF package")
1785 lines = []
1786 files_list = [self.output_path]
1787 files_list.extend(self.references_files)
1788 logging.warning("Calculating SHA1 checksums, this may take a while")
1789 sha1_sums = utils.FingerprintFiles(files_list)
1790 for file_path, value in sha1_sums.iteritems():
1791 file_name = os.path.basename(file_path)
1792 lines.append("SHA1(%s)= %s" % (file_name, value))
1793 lines.append("")
1794 data = "\n".join(lines)
1795 try:
1796 utils.WriteFile(path, data=data)
1797 except errors.ProgrammerError, err:
1798 raise errors.OpPrereqError("Saving the manifest file failed: %s" % err,
1799 errors.ECODE_ENVIRON)
1800
1801 @staticmethod
1802 def _PrepareTarFile(tar_path, files_list):
1803 """Creates tarfile from the files in OVF package.
1804
1805 @type tar_path: string
1806 @param tar_path: path to the resulting file
1807 @type files_list: list
1808 @param files_list: list of files in the OVF package
1809
1810 """
1811 logging.info("Preparing tarball for the OVF package")
1812 open(tar_path, mode="w").close()
1813 ova_package = tarfile.open(name=tar_path, mode="w")
1814 for file_path in files_list:
1815 file_name = os.path.basename(file_path)
1816 ova_package.add(file_path, arcname=file_name)
1817 ova_package.close()
1818
1819 def Save(self):
1820 """Saves the gathered configuration in an apropriate format.
1821
1822 @raise errors.OpPrereqError: if unable to create output directory
1823
1824 """
1825 output_file = "%s%s" % (self.results_name, OVF_EXT)
1826 output_path = utils.PathJoin(self.output_dir, output_file)
1827 self.ovf_writer = OVFWriter(not self.options.ext_usage)
1828 logging.info("Saving read data to %s", output_path)
1829
1830 self.output_path = utils.PathJoin(self.output_dir, output_file)
1831 files_list = [self.output_path]
1832
1833 self.ovf_writer.SaveDisksData(self.results_disk)
1834 self.ovf_writer.SaveNetworksData(self.results_network)
1835 if not self.options.ext_usage:
1836 self.ovf_writer.SaveGanetiData(self.results_ganeti, self.results_network)
1837
1838 self.ovf_writer.SaveVirtualSystemData(self.results_name, self.results_vcpus,
1839 self.results_memory)
1840
1841 data = self.ovf_writer.PrettyXmlDump()
1842 utils.WriteFile(self.output_path, data=data)
1843
1844 manifest_file = "%s%s" % (self.results_name, MF_EXT)
1845 manifest_path = utils.PathJoin(self.output_dir, manifest_file)
1846 self._PrepareManifest(manifest_path)
1847 files_list.append(manifest_path)
1848
1849 files_list.extend(self.references_files)
1850
1851 if self.options.ova_package:
1852 ova_file = "%s%s" % (self.results_name, OVA_EXT)
1853 packed_path = utils.PathJoin(self.packed_dir, ova_file)
1854 try:
1855 utils.Makedirs(self.packed_dir)
1856 except OSError, err:
1857 raise errors.OpPrereqError("Failed to create directory %s: %s" %
1858 (self.packed_dir, err),
1859 errors.ECODE_ENVIRON)
1860 self._PrepareTarFile(packed_path, files_list)
1861 logging.info("Creation of the OVF package was successfull")
1862 self.Cleanup()