Placate warnings on ganeti.outils_unittest.py
[ganeti-github.git] / lib / ovf.py
1 #!/usr/bin/python
2 #
3
4 # Copyright (C) 2011, 2012 Google Inc.
5 #
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 # 02110-1301, USA.
20
21
22 """Converter tools between ovf and ganeti config file
23
24 """
25
26 # pylint: disable=F0401, E1101
27
28 # F0401 because ElementTree is not default for python 2.4
29 # E1101 makes no sense - pylint assumes that ElementTree object is a tuple
30
31
32 import ConfigParser
33 import errno
34 import logging
35 import os
36 import os.path
37 import re
38 import shutil
39 import tarfile
40 import tempfile
41 import xml.dom.minidom
42 import xml.parsers.expat
43 try:
44 import xml.etree.ElementTree as ET
45 except ImportError:
46 import elementtree.ElementTree as ET
47
48 try:
49 ParseError = ET.ParseError # pylint: disable=E1103
50 except AttributeError:
51 ParseError = None
52
53 from ganeti import constants
54 from ganeti import errors
55 from ganeti import utils
56 from ganeti import pathutils
57
58
59 # Schemas used in OVF format
60 GANETI_SCHEMA = "http://ganeti"
61 OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1"
62 RASD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/"
63 "CIM_ResourceAllocationSettingData")
64 VSSD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/"
65 "CIM_VirtualSystemSettingData")
66 XML_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance"
67
68 # File extensions in OVF package
69 OVA_EXT = ".ova"
70 OVF_EXT = ".ovf"
71 MF_EXT = ".mf"
72 CERT_EXT = ".cert"
73 COMPRESSION_EXT = ".gz"
74 FILE_EXTENSIONS = [
75 OVF_EXT,
76 MF_EXT,
77 CERT_EXT,
78 ]
79
80 COMPRESSION_TYPE = "gzip"
81 NO_COMPRESSION = [None, "identity"]
82 COMPRESS = "compression"
83 DECOMPRESS = "decompression"
84 ALLOWED_ACTIONS = [COMPRESS, DECOMPRESS]
85
86 VMDK = "vmdk"
87 RAW = "raw"
88 COW = "cow"
89 ALLOWED_FORMATS = [RAW, COW, VMDK]
90
91 # ResourceType values
92 RASD_TYPE = {
93 "vcpus": "3",
94 "memory": "4",
95 "scsi-controller": "6",
96 "ethernet-adapter": "10",
97 "disk": "17",
98 }
99
100 SCSI_SUBTYPE = "lsilogic"
101 VS_TYPE = {
102 "ganeti": "ganeti-ovf",
103 "external": "vmx-04",
104 }
105
106 # AllocationUnits values and conversion
107 ALLOCATION_UNITS = {
108 "b": ["bytes", "b"],
109 "kb": ["kilobytes", "kb", "byte * 2^10", "kibibytes", "kib"],
110 "mb": ["megabytes", "mb", "byte * 2^20", "mebibytes", "mib"],
111 "gb": ["gigabytes", "gb", "byte * 2^30", "gibibytes", "gib"],
112 }
113 CONVERT_UNITS_TO_MB = {
114 "b": lambda x: x / (1024 * 1024),
115 "kb": lambda x: x / 1024,
116 "mb": lambda x: x,
117 "gb": lambda x: x * 1024,
118 }
119
120 # Names of the config fields
121 NAME = "name"
122 OS = "os"
123 HYPERV = "hypervisor"
124 VCPUS = "vcpus"
125 MEMORY = "memory"
126 AUTO_BALANCE = "auto_balance"
127 DISK_TEMPLATE = "disk_template"
128 TAGS = "tags"
129 VERSION = "version"
130
131 # Instance IDs of System and SCSI controller
132 INSTANCE_ID = {
133 "system": 0,
134 "vcpus": 1,
135 "memory": 2,
136 "scsi": 3,
137 }
138
139 # Disk format descriptions
140 DISK_FORMAT = {
141 RAW: "http://en.wikipedia.org/wiki/Byte",
142 VMDK: "http://www.vmware.com/interfaces/specifications/vmdk.html"
143 "#monolithicSparse",
144 COW: "http://www.gnome.org/~markmc/qcow-image-format.html",
145 }
146
147
148 def CheckQemuImg():
149 """ Make sure that qemu-img is present before performing operations.
150
151 @raise errors.OpPrereqError: when qemu-img was not found in the system
152
153 """
154 if not constants.QEMUIMG_PATH:
155 raise errors.OpPrereqError("qemu-img not found at build time, unable"
156 " to continue", errors.ECODE_STATE)
157
158
159 def LinkFile(old_path, prefix=None, suffix=None, directory=None):
160 """Create link with a given prefix and suffix.
161
162 This is a wrapper over os.link. It tries to create a hard link for given file,
163 but instead of rising error when file exists, the function changes the name
164 a little bit.
165
166 @type old_path:string
167 @param old_path: path to the file that is to be linked
168 @type prefix: string
169 @param prefix: prefix of filename for the link
170 @type suffix: string
171 @param suffix: suffix of the filename for the link
172 @type directory: string
173 @param directory: directory of the link
174
175 @raise errors.OpPrereqError: when error on linking is different than
176 "File exists"
177
178 """
179 assert(prefix is not None or suffix is not None)
180 if directory is None:
181 directory = os.getcwd()
182 new_path = utils.PathJoin(directory, "%s%s" % (prefix, suffix))
183 counter = 1
184 while True:
185 try:
186 os.link(old_path, new_path)
187 break
188 except OSError, err:
189 if err.errno == errno.EEXIST:
190 new_path = utils.PathJoin(directory,
191 "%s_%s%s" % (prefix, counter, suffix))
192 counter += 1
193 else:
194 raise errors.OpPrereqError("Error moving the file %s to %s location:"
195 " %s" % (old_path, new_path, err),
196 errors.ECODE_ENVIRON)
197 return new_path
198
199
200 class OVFReader(object):
201 """Reader class for OVF files.
202
203 @type files_list: list
204 @ivar files_list: list of files in the OVF package
205 @type tree: ET.ElementTree
206 @ivar tree: XML tree of the .ovf file
207 @type schema_name: string
208 @ivar schema_name: name of the .ovf file
209 @type input_dir: string
210 @ivar input_dir: directory in which the .ovf file resides
211
212 """
213 def __init__(self, input_path):
214 """Initialiaze the reader - load the .ovf file to XML parser.
215
216 It is assumed that names of manifesto (.mf), certificate (.cert) and ovf
217 files are the same. In order to account any other files as part of the ovf
218 package, they have to be explicitly mentioned in the Resources section
219 of the .ovf file.
220
221 @type input_path: string
222 @param input_path: absolute path to the .ovf file
223
224 @raise errors.OpPrereqError: when .ovf file is not a proper XML file or some
225 of the files mentioned in Resources section do not exist
226
227 """
228 self.tree = ET.ElementTree()
229 try:
230 self.tree.parse(input_path)
231 except (ParseError, xml.parsers.expat.ExpatError), err:
232 raise errors.OpPrereqError("Error while reading %s file: %s" %
233 (OVF_EXT, err), errors.ECODE_ENVIRON)
234
235 # Create a list of all files in the OVF package
236 (input_dir, input_file) = os.path.split(input_path)
237 (input_name, _) = os.path.splitext(input_file)
238 files_directory = utils.ListVisibleFiles(input_dir)
239 files_list = []
240 for file_name in files_directory:
241 (name, extension) = os.path.splitext(file_name)
242 if extension in FILE_EXTENSIONS and name == input_name:
243 files_list.append(file_name)
244 files_list += self._GetAttributes("{%s}References/{%s}File" %
245 (OVF_SCHEMA, OVF_SCHEMA),
246 "{%s}href" % OVF_SCHEMA)
247 for file_name in files_list:
248 file_path = utils.PathJoin(input_dir, file_name)
249 if not os.path.exists(file_path):
250 raise errors.OpPrereqError("File does not exist: %s" % file_path,
251 errors.ECODE_ENVIRON)
252 logging.info("Files in the OVF package: %s", " ".join(files_list))
253 self.files_list = files_list
254 self.input_dir = input_dir
255 self.schema_name = input_name
256
257 def _GetAttributes(self, path, attribute):
258 """Get specified attribute from all nodes accessible using given path.
259
260 Function follows the path from root node to the desired tags using path,
261 then reads the apropriate attribute values.
262
263 @type path: string
264 @param path: path of nodes to visit
265 @type attribute: string
266 @param attribute: attribute for which we gather the information
267 @rtype: list
268 @return: for each accessible tag with the attribute value set, value of the
269 attribute
270
271 """
272 current_list = self.tree.findall(path)
273 results = [x.get(attribute) for x in current_list]
274 return filter(None, results)
275
276 def _GetElementMatchingAttr(self, path, match_attr):
277 """Searches for element on a path that matches certain attribute value.
278
279 Function follows the path from root node to the desired tags using path,
280 then searches for the first one matching the attribute value.
281
282 @type path: string
283 @param path: path of nodes to visit
284 @type match_attr: tuple
285 @param match_attr: pair (attribute, value) for which we search
286 @rtype: ET.ElementTree or None
287 @return: first element matching match_attr or None if nothing matches
288
289 """
290 potential_elements = self.tree.findall(path)
291 (attr, val) = match_attr
292 for elem in potential_elements:
293 if elem.get(attr) == val:
294 return elem
295 return None
296
297 def _GetElementMatchingText(self, path, match_text):
298 """Searches for element on a path that matches certain text value.
299
300 Function follows the path from root node to the desired tags using path,
301 then searches for the first one matching the text value.
302
303 @type path: string
304 @param path: path of nodes to visit
305 @type match_text: tuple
306 @param match_text: pair (node, text) for which we search
307 @rtype: ET.ElementTree or None
308 @return: first element matching match_text or None if nothing matches
309
310 """
311 potential_elements = self.tree.findall(path)
312 (node, text) = match_text
313 for elem in potential_elements:
314 if elem.findtext(node) == text:
315 return elem
316 return None
317
318 @staticmethod
319 def _GetDictParameters(root, schema):
320 """Reads text in all children and creates the dictionary from the contents.
321
322 @type root: ET.ElementTree or None
323 @param root: father of the nodes we want to collect data about
324 @type schema: string
325 @param schema: schema name to be removed from the tag
326 @rtype: dict
327 @return: dictionary containing tags and their text contents, tags have their
328 schema fragment removed or empty dictionary, when root is None
329
330 """
331 if root is None:
332 return {}
333 results = {}
334 for element in list(root):
335 pref_len = len("{%s}" % schema)
336 assert(schema in element.tag)
337 tag = element.tag[pref_len:]
338 results[tag] = element.text
339 return results
340
341 def VerifyManifest(self):
342 """Verifies manifest for the OVF package, if one is given.
343
344 @raise errors.OpPrereqError: if SHA1 checksums do not match
345
346 """
347 if "%s%s" % (self.schema_name, MF_EXT) in self.files_list:
348 logging.warning("Verifying SHA1 checksums, this may take a while")
349 manifest_filename = "%s%s" % (self.schema_name, MF_EXT)
350 manifest_path = utils.PathJoin(self.input_dir, manifest_filename)
351 manifest_content = utils.ReadFile(manifest_path).splitlines()
352 manifest_files = {}
353 regexp = r"SHA1\((\S+)\)= (\S+)"
354 for line in manifest_content:
355 match = re.match(regexp, line)
356 if match:
357 file_name = match.group(1)
358 sha1_sum = match.group(2)
359 manifest_files[file_name] = sha1_sum
360 files_with_paths = [utils.PathJoin(self.input_dir, file_name)
361 for file_name in self.files_list]
362 sha1_sums = utils.FingerprintFiles(files_with_paths)
363 for file_name, value in manifest_files.iteritems():
364 if sha1_sums.get(utils.PathJoin(self.input_dir, file_name)) != value:
365 raise errors.OpPrereqError("SHA1 checksum of %s does not match the"
366 " value in manifest file" % file_name,
367 errors.ECODE_ENVIRON)
368 logging.info("SHA1 checksums verified")
369
370 def GetInstanceName(self):
371 """Provides information about instance name.
372
373 @rtype: string
374 @return: instance name string
375
376 """
377 find_name = "{%s}VirtualSystem/{%s}Name" % (OVF_SCHEMA, OVF_SCHEMA)
378 return self.tree.findtext(find_name)
379
380 def GetDiskTemplate(self):
381 """Returns disk template from .ovf file
382
383 @rtype: string or None
384 @return: name of the template
385 """
386 find_template = ("{%s}GanetiSection/{%s}DiskTemplate" %
387 (GANETI_SCHEMA, GANETI_SCHEMA))
388 return self.tree.findtext(find_template)
389
390 def GetHypervisorData(self):
391 """Provides hypervisor information - hypervisor name and options.
392
393 @rtype: dict
394 @return: dictionary containing name of the used hypervisor and all the
395 specified options
396
397 """
398 hypervisor_search = ("{%s}GanetiSection/{%s}Hypervisor" %
399 (GANETI_SCHEMA, GANETI_SCHEMA))
400 hypervisor_data = self.tree.find(hypervisor_search)
401 if hypervisor_data is None:
402 return {"hypervisor_name": constants.VALUE_AUTO}
403 results = {
404 "hypervisor_name": hypervisor_data.findtext("{%s}Name" % GANETI_SCHEMA,
405 default=constants.VALUE_AUTO),
406 }
407 parameters = hypervisor_data.find("{%s}Parameters" % GANETI_SCHEMA)
408 results.update(self._GetDictParameters(parameters, GANETI_SCHEMA))
409 return results
410
411 def GetOSData(self):
412 """ Provides operating system information - os name and options.
413
414 @rtype: dict
415 @return: dictionary containing name and options for the chosen OS
416
417 """
418 results = {}
419 os_search = ("{%s}GanetiSection/{%s}OperatingSystem" %
420 (GANETI_SCHEMA, GANETI_SCHEMA))
421 os_data = self.tree.find(os_search)
422 if os_data is not None:
423 results["os_name"] = os_data.findtext("{%s}Name" % GANETI_SCHEMA)
424 parameters = os_data.find("{%s}Parameters" % GANETI_SCHEMA)
425 results.update(self._GetDictParameters(parameters, GANETI_SCHEMA))
426 return results
427
428 def GetBackendData(self):
429 """ Provides backend information - vcpus, memory, auto balancing options.
430
431 @rtype: dict
432 @return: dictionary containing options for vcpus, memory and auto balance
433 settings
434
435 """
436 results = {}
437
438 find_vcpus = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" %
439 (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA))
440 match_vcpus = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["vcpus"])
441 vcpus = self._GetElementMatchingText(find_vcpus, match_vcpus)
442 if vcpus is not None:
443 vcpus_count = vcpus.findtext("{%s}VirtualQuantity" % RASD_SCHEMA,
444 default=constants.VALUE_AUTO)
445 else:
446 vcpus_count = constants.VALUE_AUTO
447 results["vcpus"] = str(vcpus_count)
448
449 find_memory = find_vcpus
450 match_memory = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["memory"])
451 memory = self._GetElementMatchingText(find_memory, match_memory)
452 memory_raw = None
453 if memory is not None:
454 alloc_units = memory.findtext("{%s}AllocationUnits" % RASD_SCHEMA)
455 matching_units = [units for units, variants in ALLOCATION_UNITS.items()
456 if alloc_units.lower() in variants]
457 if matching_units == []:
458 raise errors.OpPrereqError("Unit %s for RAM memory unknown" %
459 alloc_units, errors.ECODE_INVAL)
460 units = matching_units[0]
461 memory_raw = int(memory.findtext("{%s}VirtualQuantity" % RASD_SCHEMA,
462 default=constants.VALUE_AUTO))
463 memory_count = CONVERT_UNITS_TO_MB[units](memory_raw)
464 else:
465 memory_count = constants.VALUE_AUTO
466 results["memory"] = str(memory_count)
467
468 find_balance = ("{%s}GanetiSection/{%s}AutoBalance" %
469 (GANETI_SCHEMA, GANETI_SCHEMA))
470 balance = self.tree.findtext(find_balance, default=constants.VALUE_AUTO)
471 results["auto_balance"] = balance
472
473 return results
474
475 def GetTagsData(self):
476 """Provides tags information for instance.
477
478 @rtype: string or None
479 @return: string of comma-separated tags for the instance
480
481 """
482 find_tags = "{%s}GanetiSection/{%s}Tags" % (GANETI_SCHEMA, GANETI_SCHEMA)
483 results = self.tree.findtext(find_tags)
484 if results:
485 return results
486 else:
487 return None
488
489 def GetVersionData(self):
490 """Provides version number read from .ovf file
491
492 @rtype: string
493 @return: string containing the version number
494
495 """
496 find_version = ("{%s}GanetiSection/{%s}Version" %
497 (GANETI_SCHEMA, GANETI_SCHEMA))
498 return self.tree.findtext(find_version)
499
500 def GetNetworkData(self):
501 """Provides data about the network in the OVF instance.
502
503 The method gathers the data about networks used by OVF instance. It assumes
504 that 'name' tag means something - in essence, if it contains one of the
505 words 'bridged' or 'routed' then that will be the mode of this network in
506 Ganeti. The information about the network can be either in GanetiSection or
507 VirtualHardwareSection.
508
509 @rtype: dict
510 @return: dictionary containing all the network information
511
512 """
513 results = {}
514 networks_search = ("{%s}NetworkSection/{%s}Network" %
515 (OVF_SCHEMA, OVF_SCHEMA))
516 network_names = self._GetAttributes(networks_search,
517 "{%s}name" % OVF_SCHEMA)
518 required = ["ip", "mac", "link", "mode", "network"]
519 for (counter, network_name) in enumerate(network_names):
520 network_search = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item"
521 % (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA))
522 ganeti_search = ("{%s}GanetiSection/{%s}Network/{%s}Nic" %
523 (GANETI_SCHEMA, GANETI_SCHEMA, GANETI_SCHEMA))
524 network_match = ("{%s}Connection" % RASD_SCHEMA, network_name)
525 ganeti_match = ("{%s}name" % OVF_SCHEMA, network_name)
526 network_data = self._GetElementMatchingText(network_search, network_match)
527 network_ganeti_data = self._GetElementMatchingAttr(ganeti_search,
528 ganeti_match)
529
530 ganeti_data = {}
531 if network_ganeti_data is not None:
532 ganeti_data["mode"] = network_ganeti_data.findtext("{%s}Mode" %
533 GANETI_SCHEMA)
534 ganeti_data["mac"] = network_ganeti_data.findtext("{%s}MACAddress" %
535 GANETI_SCHEMA)
536 ganeti_data["ip"] = network_ganeti_data.findtext("{%s}IPAddress" %
537 GANETI_SCHEMA)
538 ganeti_data["link"] = network_ganeti_data.findtext("{%s}Link" %
539 GANETI_SCHEMA)
540 ganeti_data["network"] = network_ganeti_data.findtext("{%s}Net" %
541 GANETI_SCHEMA)
542 mac_data = None
543 if network_data is not None:
544 mac_data = network_data.findtext("{%s}Address" % RASD_SCHEMA)
545
546 network_name = network_name.lower()
547
548 # First, some not Ganeti-specific information is collected
549 if constants.NIC_MODE_BRIDGED in network_name:
550 results["nic%s_mode" % counter] = "bridged"
551 elif constants.NIC_MODE_ROUTED in network_name:
552 results["nic%s_mode" % counter] = "routed"
553 results["nic%s_mac" % counter] = mac_data
554
555 # GanetiSection data overrides 'manually' collected data
556 for name, value in ganeti_data.iteritems():
557 results["nic%s_%s" % (counter, name)] = value
558
559 # Bridged network has no IP - unless specifically stated otherwise
560 if (results.get("nic%s_mode" % counter) == "bridged" and
561 not results.get("nic%s_ip" % counter)):
562 results["nic%s_ip" % counter] = constants.VALUE_NONE
563
564 for option in required:
565 if not results.get("nic%s_%s" % (counter, option)):
566 results["nic%s_%s" % (counter, option)] = constants.VALUE_AUTO
567
568 if network_names:
569 results["nic_count"] = str(len(network_names))
570 return results
571
572 def GetDisksNames(self):
573 """Provides list of file names for the disks used by the instance.
574
575 @rtype: list
576 @return: list of file names, as referenced in .ovf file
577
578 """
579 results = []
580 disks_search = "{%s}DiskSection/{%s}Disk" % (OVF_SCHEMA, OVF_SCHEMA)
581 disk_ids = self._GetAttributes(disks_search, "{%s}fileRef" % OVF_SCHEMA)
582 for disk in disk_ids:
583 disk_search = "{%s}References/{%s}File" % (OVF_SCHEMA, OVF_SCHEMA)
584 disk_match = ("{%s}id" % OVF_SCHEMA, disk)
585 disk_elem = self._GetElementMatchingAttr(disk_search, disk_match)
586 if disk_elem is None:
587 raise errors.OpPrereqError("%s file corrupted - disk %s not found in"
588 " references" % (OVF_EXT, disk),
589 errors.ECODE_ENVIRON)
590 disk_name = disk_elem.get("{%s}href" % OVF_SCHEMA)
591 disk_compression = disk_elem.get("{%s}compression" % OVF_SCHEMA)
592 results.append((disk_name, disk_compression))
593 return results
594
595
596 def SubElementText(parent, tag, text, attrib={}, **extra):
597 # pylint: disable=W0102
598 """This is just a wrapper on ET.SubElement that always has text content.
599
600 """
601 if text is None:
602 return None
603 elem = ET.SubElement(parent, tag, attrib=attrib, **extra)
604 elem.text = str(text)
605 return elem
606
607
608 class OVFWriter(object):
609 """Writer class for OVF files.
610
611 @type tree: ET.ElementTree
612 @ivar tree: XML tree that we are constructing
613 @type virtual_system_type: string
614 @ivar virtual_system_type: value of vssd:VirtualSystemType, for external usage
615 in VMWare this requires to be vmx
616 @type hardware_list: list
617 @ivar hardware_list: list of items prepared for VirtualHardwareSection
618 @type next_instance_id: int
619 @ivar next_instance_id: next instance id to be used when creating elements on
620 hardware_list
621
622 """
623 def __init__(self, has_gnt_section):
624 """Initialize the writer - set the top element.
625
626 @type has_gnt_section: bool
627 @param has_gnt_section: if the Ganeti schema should be added - i.e. this
628 means that Ganeti section will be present
629
630 """
631 env_attribs = {
632 "xmlns:xsi": XML_SCHEMA,
633 "xmlns:vssd": VSSD_SCHEMA,
634 "xmlns:rasd": RASD_SCHEMA,
635 "xmlns:ovf": OVF_SCHEMA,
636 "xmlns": OVF_SCHEMA,
637 "xml:lang": "en-US",
638 }
639 if has_gnt_section:
640 env_attribs["xmlns:gnt"] = GANETI_SCHEMA
641 self.virtual_system_type = VS_TYPE["ganeti"]
642 else:
643 self.virtual_system_type = VS_TYPE["external"]
644 self.tree = ET.Element("Envelope", attrib=env_attribs)
645 self.hardware_list = []
646 # INSTANCE_ID contains statically assigned IDs, starting from 0
647 self.next_instance_id = len(INSTANCE_ID) # FIXME: hackish
648
649 def SaveDisksData(self, disks):
650 """Convert disk information to certain OVF sections.
651
652 @type disks: list
653 @param disks: list of dictionaries of disk options from config.ini
654
655 """
656 references = ET.SubElement(self.tree, "References")
657 disk_section = ET.SubElement(self.tree, "DiskSection")
658 SubElementText(disk_section, "Info", "Virtual disk information")
659 for counter, disk in enumerate(disks):
660 file_id = "file%s" % counter
661 disk_id = "disk%s" % counter
662 file_attribs = {
663 "ovf:href": disk["path"],
664 "ovf:size": str(disk["real-size"]),
665 "ovf:id": file_id,
666 }
667 disk_attribs = {
668 "ovf:capacity": str(disk["virt-size"]),
669 "ovf:diskId": disk_id,
670 "ovf:fileRef": file_id,
671 "ovf:format": DISK_FORMAT.get(disk["format"], disk["format"]),
672 }
673 if "compression" in disk:
674 file_attribs["ovf:compression"] = disk["compression"]
675 ET.SubElement(references, "File", attrib=file_attribs)
676 ET.SubElement(disk_section, "Disk", attrib=disk_attribs)
677
678 # Item in VirtualHardwareSection creation
679 disk_item = ET.Element("Item")
680 SubElementText(disk_item, "rasd:ElementName", disk_id)
681 SubElementText(disk_item, "rasd:HostResource", "ovf:/disk/%s" % disk_id)
682 SubElementText(disk_item, "rasd:InstanceID", self.next_instance_id)
683 SubElementText(disk_item, "rasd:Parent", INSTANCE_ID["scsi"])
684 SubElementText(disk_item, "rasd:ResourceType", RASD_TYPE["disk"])
685 self.hardware_list.append(disk_item)
686 self.next_instance_id += 1
687
688 def SaveNetworksData(self, networks):
689 """Convert network information to NetworkSection.
690
691 @type networks: list
692 @param networks: list of dictionaries of network options form config.ini
693
694 """
695 network_section = ET.SubElement(self.tree, "NetworkSection")
696 SubElementText(network_section, "Info", "List of logical networks")
697 for counter, network in enumerate(networks):
698 network_name = "%s%s" % (network["mode"], counter)
699 network_attrib = {"ovf:name": network_name}
700 ET.SubElement(network_section, "Network", attrib=network_attrib)
701
702 # Item in VirtualHardwareSection creation
703 network_item = ET.Element("Item")
704 SubElementText(network_item, "rasd:Address", network["mac"])
705 SubElementText(network_item, "rasd:Connection", network_name)
706 SubElementText(network_item, "rasd:ElementName", network_name)
707 SubElementText(network_item, "rasd:InstanceID", self.next_instance_id)
708 SubElementText(network_item, "rasd:ResourceType",
709 RASD_TYPE["ethernet-adapter"])
710 self.hardware_list.append(network_item)
711 self.next_instance_id += 1
712
713 @staticmethod
714 def _SaveNameAndParams(root, data):
715 """Save name and parameters information under root using data.
716
717 @type root: ET.Element
718 @param root: root element for the Name and Parameters
719 @type data: dict
720 @param data: data from which we gather the values
721
722 """
723 assert(data.get("name"))
724 name = SubElementText(root, "gnt:Name", data["name"])
725 params = ET.SubElement(root, "gnt:Parameters")
726 for name, value in data.iteritems():
727 if name != "name":
728 SubElementText(params, "gnt:%s" % name, value)
729
730 def SaveGanetiData(self, ganeti, networks):
731 """Convert Ganeti-specific information to GanetiSection.
732
733 @type ganeti: dict
734 @param ganeti: dictionary of Ganeti-specific options from config.ini
735 @type networks: list
736 @param networks: list of dictionaries of network options form config.ini
737
738 """
739 ganeti_section = ET.SubElement(self.tree, "gnt:GanetiSection")
740
741 SubElementText(ganeti_section, "gnt:Version", ganeti.get("version"))
742 SubElementText(ganeti_section, "gnt:DiskTemplate",
743 ganeti.get("disk_template"))
744 SubElementText(ganeti_section, "gnt:AutoBalance",
745 ganeti.get("auto_balance"))
746 SubElementText(ganeti_section, "gnt:Tags", ganeti.get("tags"))
747
748 osys = ET.SubElement(ganeti_section, "gnt:OperatingSystem")
749 self._SaveNameAndParams(osys, ganeti["os"])
750
751 hypervisor = ET.SubElement(ganeti_section, "gnt:Hypervisor")
752 self._SaveNameAndParams(hypervisor, ganeti["hypervisor"])
753
754 network_section = ET.SubElement(ganeti_section, "gnt:Network")
755 for counter, network in enumerate(networks):
756 network_name = "%s%s" % (network["mode"], counter)
757 nic_attrib = {"ovf:name": network_name}
758 nic = ET.SubElement(network_section, "gnt:Nic", attrib=nic_attrib)
759 SubElementText(nic, "gnt:Mode", network["mode"])
760 SubElementText(nic, "gnt:MACAddress", network["mac"])
761 SubElementText(nic, "gnt:IPAddress", network["ip"])
762 SubElementText(nic, "gnt:Link", network["link"])
763 SubElementText(nic, "gnt:Net", network["network"])
764
765 def SaveVirtualSystemData(self, name, vcpus, memory):
766 """Convert virtual system information to OVF sections.
767
768 @type name: string
769 @param name: name of the instance
770 @type vcpus: int
771 @param vcpus: number of VCPUs
772 @type memory: int
773 @param memory: RAM memory in MB
774
775 """
776 assert(vcpus > 0)
777 assert(memory > 0)
778 vs_attrib = {"ovf:id": name}
779 virtual_system = ET.SubElement(self.tree, "VirtualSystem", attrib=vs_attrib)
780 SubElementText(virtual_system, "Info", "A virtual machine")
781
782 name_section = ET.SubElement(virtual_system, "Name")
783 name_section.text = name
784 os_attrib = {"ovf:id": "0"}
785 os_section = ET.SubElement(virtual_system, "OperatingSystemSection",
786 attrib=os_attrib)
787 SubElementText(os_section, "Info", "Installed guest operating system")
788 hardware_section = ET.SubElement(virtual_system, "VirtualHardwareSection")
789 SubElementText(hardware_section, "Info", "Virtual hardware requirements")
790
791 # System description
792 system = ET.SubElement(hardware_section, "System")
793 SubElementText(system, "vssd:ElementName", "Virtual Hardware Family")
794 SubElementText(system, "vssd:InstanceID", INSTANCE_ID["system"])
795 SubElementText(system, "vssd:VirtualSystemIdentifier", name)
796 SubElementText(system, "vssd:VirtualSystemType", self.virtual_system_type)
797
798 # Item for vcpus
799 vcpus_item = ET.SubElement(hardware_section, "Item")
800 SubElementText(vcpus_item, "rasd:ElementName",
801 "%s virtual CPU(s)" % vcpus)
802 SubElementText(vcpus_item, "rasd:InstanceID", INSTANCE_ID["vcpus"])
803 SubElementText(vcpus_item, "rasd:ResourceType", RASD_TYPE["vcpus"])
804 SubElementText(vcpus_item, "rasd:VirtualQuantity", vcpus)
805
806 # Item for memory
807 memory_item = ET.SubElement(hardware_section, "Item")
808 SubElementText(memory_item, "rasd:AllocationUnits", "byte * 2^20")
809 SubElementText(memory_item, "rasd:ElementName", "%sMB of memory" % memory)
810 SubElementText(memory_item, "rasd:InstanceID", INSTANCE_ID["memory"])
811 SubElementText(memory_item, "rasd:ResourceType", RASD_TYPE["memory"])
812 SubElementText(memory_item, "rasd:VirtualQuantity", memory)
813
814 # Item for scsi controller
815 scsi_item = ET.SubElement(hardware_section, "Item")
816 SubElementText(scsi_item, "rasd:Address", INSTANCE_ID["system"])
817 SubElementText(scsi_item, "rasd:ElementName", "scsi_controller0")
818 SubElementText(scsi_item, "rasd:InstanceID", INSTANCE_ID["scsi"])
819 SubElementText(scsi_item, "rasd:ResourceSubType", SCSI_SUBTYPE)
820 SubElementText(scsi_item, "rasd:ResourceType", RASD_TYPE["scsi-controller"])
821
822 # Other items - from self.hardware_list
823 for item in self.hardware_list:
824 hardware_section.append(item)
825
826 def PrettyXmlDump(self):
827 """Formatter of the XML file.
828
829 @rtype: string
830 @return: XML tree in the form of nicely-formatted string
831
832 """
833 raw_string = ET.tostring(self.tree)
834 parsed_xml = xml.dom.minidom.parseString(raw_string)
835 xml_string = parsed_xml.toprettyxml(indent=" ")
836 text_re = re.compile(">\n\s+([^<>\s].*?)\n\s+</", re.DOTALL)
837 return text_re.sub(">\g<1></", xml_string)
838
839
840 class Converter(object):
841 """Converter class for OVF packages.
842
843 Converter is a class above both ImporterOVF and ExporterOVF. It's purpose is
844 to provide a common interface for the two.
845
846 @type options: optparse.Values
847 @ivar options: options parsed from the command line
848 @type output_dir: string
849 @ivar output_dir: directory to which the results of conversion shall be
850 written
851 @type temp_file_manager: L{utils.TemporaryFileManager}
852 @ivar temp_file_manager: container for temporary files created during
853 conversion
854 @type temp_dir: string
855 @ivar temp_dir: temporary directory created then we deal with OVA
856
857 """
858 def __init__(self, input_path, options):
859 """Initialize the converter.
860
861 @type input_path: string
862 @param input_path: path to the Converter input file
863 @type options: optparse.Values
864 @param options: command line options
865
866 @raise errors.OpPrereqError: if file does not exist
867
868 """
869 input_path = os.path.abspath(input_path)
870 if not os.path.isfile(input_path):
871 raise errors.OpPrereqError("File does not exist: %s" % input_path,
872 errors.ECODE_ENVIRON)
873 self.options = options
874 self.temp_file_manager = utils.TemporaryFileManager()
875 self.temp_dir = None
876 self.output_dir = None
877 self._ReadInputData(input_path)
878
879 def _ReadInputData(self, input_path):
880 """Reads the data on which the conversion will take place.
881
882 @type input_path: string
883 @param input_path: absolute path to the Converter input file
884
885 """
886 raise NotImplementedError()
887
888 def _CompressDisk(self, disk_path, compression, action):
889 """Performs (de)compression on the disk and returns the new path
890
891 @type disk_path: string
892 @param disk_path: path to the disk
893 @type compression: string
894 @param compression: compression type
895 @type action: string
896 @param action: whether the action is compression or decompression
897 @rtype: string
898 @return: new disk path after (de)compression
899
900 @raise errors.OpPrereqError: disk (de)compression failed or "compression"
901 is not supported
902
903 """
904 assert(action in ALLOWED_ACTIONS)
905 # For now we only support gzip, as it is used in ovftool
906 if compression != COMPRESSION_TYPE:
907 raise errors.OpPrereqError("Unsupported compression type: %s"
908 % compression, errors.ECODE_INVAL)
909 disk_file = os.path.basename(disk_path)
910 if action == DECOMPRESS:
911 (disk_name, _) = os.path.splitext(disk_file)
912 prefix = disk_name
913 elif action == COMPRESS:
914 prefix = disk_file
915 new_path = utils.GetClosedTempfile(suffix=COMPRESSION_EXT, prefix=prefix,
916 dir=self.output_dir)
917 self.temp_file_manager.Add(new_path)
918 args = ["gzip", "-c", disk_path]
919 run_result = utils.RunCmd(args, output=new_path)
920 if run_result.failed:
921 raise errors.OpPrereqError("Disk %s failed with output: %s"
922 % (action, run_result.stderr),
923 errors.ECODE_ENVIRON)
924 logging.info("The %s of the disk is completed", action)
925 return (COMPRESSION_EXT, new_path)
926
927 def _ConvertDisk(self, disk_format, disk_path):
928 """Performes conversion to specified format.
929
930 @type disk_format: string
931 @param disk_format: format to which the disk should be converted
932 @type disk_path: string
933 @param disk_path: path to the disk that should be converted
934 @rtype: string
935 @return path to the output disk
936
937 @raise errors.OpPrereqError: convertion of the disk failed
938
939 """
940 CheckQemuImg()
941 disk_file = os.path.basename(disk_path)
942 (disk_name, disk_extension) = os.path.splitext(disk_file)
943 if disk_extension != disk_format:
944 logging.warning("Conversion of disk image to %s format, this may take"
945 " a while", disk_format)
946
947 new_disk_path = utils.GetClosedTempfile(
948 suffix=".%s" % disk_format, prefix=disk_name, dir=self.output_dir)
949 self.temp_file_manager.Add(new_disk_path)
950 args = [
951 constants.QEMUIMG_PATH,
952 "convert",
953 "-O",
954 disk_format,
955 disk_path,
956 new_disk_path,
957 ]
958 run_result = utils.RunCmd(args, cwd=os.getcwd())
959 if run_result.failed:
960 raise errors.OpPrereqError("Convertion to %s failed, qemu-img output was"
961 ": %s" % (disk_format, run_result.stderr),
962 errors.ECODE_ENVIRON)
963 return (".%s" % disk_format, new_disk_path)
964
965 @staticmethod
966 def _GetDiskQemuInfo(disk_path, regexp):
967 """Figures out some information of the disk using qemu-img.
968
969 @type disk_path: string
970 @param disk_path: path to the disk we want to know the format of
971 @type regexp: string
972 @param regexp: string that has to be matched, it has to contain one group
973 @rtype: string
974 @return: disk format
975
976 @raise errors.OpPrereqError: format information cannot be retrieved
977
978 """
979 CheckQemuImg()
980 args = [constants.QEMUIMG_PATH, "info", disk_path]
981 run_result = utils.RunCmd(args, cwd=os.getcwd())
982 if run_result.failed:
983 raise errors.OpPrereqError("Gathering info about the disk using qemu-img"
984 " failed, output was: %s" % run_result.stderr,
985 errors.ECODE_ENVIRON)
986 result = run_result.output
987 regexp = r"%s" % regexp
988 match = re.search(regexp, result)
989 if match:
990 disk_format = match.group(1)
991 else:
992 raise errors.OpPrereqError("No file information matching %s found in:"
993 " %s" % (regexp, result),
994 errors.ECODE_ENVIRON)
995 return disk_format
996
997 def Parse(self):
998 """Parses the data and creates a structure containing all required info.
999
1000 """
1001 raise NotImplementedError()
1002
1003 def Save(self):
1004 """Saves the gathered configuration in an apropriate format.
1005
1006 """
1007 raise NotImplementedError()
1008
1009 def Cleanup(self):
1010 """Cleans the temporary directory, if one was created.
1011
1012 """
1013 self.temp_file_manager.Cleanup()
1014 if self.temp_dir:
1015 shutil.rmtree(self.temp_dir)
1016 self.temp_dir = None
1017
1018
1019 class OVFImporter(Converter):
1020 """Converter from OVF to Ganeti config file.
1021
1022 @type input_dir: string
1023 @ivar input_dir: directory in which the .ovf file resides
1024 @type output_dir: string
1025 @ivar output_dir: directory to which the results of conversion shall be
1026 written
1027 @type input_path: string
1028 @ivar input_path: complete path to the .ovf file
1029 @type ovf_reader: L{OVFReader}
1030 @ivar ovf_reader: OVF reader instance collects data from .ovf file
1031 @type results_name: string
1032 @ivar results_name: name of imported instance
1033 @type results_template: string
1034 @ivar results_template: disk template read from .ovf file or command line
1035 arguments
1036 @type results_hypervisor: dict
1037 @ivar results_hypervisor: hypervisor information gathered from .ovf file or
1038 command line arguments
1039 @type results_os: dict
1040 @ivar results_os: operating system information gathered from .ovf file or
1041 command line arguments
1042 @type results_backend: dict
1043 @ivar results_backend: backend information gathered from .ovf file or
1044 command line arguments
1045 @type results_tags: string
1046 @ivar results_tags: string containing instance-specific tags
1047 @type results_version: string
1048 @ivar results_version: version as required by Ganeti import
1049 @type results_network: dict
1050 @ivar results_network: network information gathered from .ovf file or command
1051 line arguments
1052 @type results_disk: dict
1053 @ivar results_disk: disk information gathered from .ovf file or command line
1054 arguments
1055
1056 """
1057 def _ReadInputData(self, input_path):
1058 """Reads the data on which the conversion will take place.
1059
1060 @type input_path: string
1061 @param input_path: absolute path to the .ovf or .ova input file
1062
1063 @raise errors.OpPrereqError: if input file is neither .ovf nor .ova
1064
1065 """
1066 (input_dir, input_file) = os.path.split(input_path)
1067 (_, input_extension) = os.path.splitext(input_file)
1068
1069 if input_extension == OVF_EXT:
1070 logging.info("%s file extension found, no unpacking necessary", OVF_EXT)
1071 self.input_dir = input_dir
1072 self.input_path = input_path
1073 self.temp_dir = None
1074 elif input_extension == OVA_EXT:
1075 logging.info("%s file extension found, proceeding to unpacking", OVA_EXT)
1076 self._UnpackOVA(input_path)
1077 else:
1078 raise errors.OpPrereqError("Unknown file extension; expected %s or %s"
1079 " file" % (OVA_EXT, OVF_EXT),
1080 errors.ECODE_INVAL)
1081 assert ((input_extension == OVA_EXT and self.temp_dir) or
1082 (input_extension == OVF_EXT and not self.temp_dir))
1083 assert self.input_dir in self.input_path
1084
1085 if self.options.output_dir:
1086 self.output_dir = os.path.abspath(self.options.output_dir)
1087 if (os.path.commonprefix([pathutils.EXPORT_DIR, self.output_dir]) !=
1088 pathutils.EXPORT_DIR):
1089 logging.warning("Export path is not under %s directory, import to"
1090 " Ganeti using gnt-backup may fail",
1091 pathutils.EXPORT_DIR)
1092 else:
1093 self.output_dir = pathutils.EXPORT_DIR
1094
1095 self.ovf_reader = OVFReader(self.input_path)
1096 self.ovf_reader.VerifyManifest()
1097
1098 def _UnpackOVA(self, input_path):
1099 """Unpacks the .ova package into temporary directory.
1100
1101 @type input_path: string
1102 @param input_path: path to the .ova package file
1103
1104 @raise errors.OpPrereqError: if file is not a proper tarball, one of the
1105 files in the archive seem malicious (e.g. path starts with '../') or
1106 .ova package does not contain .ovf file
1107
1108 """
1109 input_name = None
1110 if not tarfile.is_tarfile(input_path):
1111 raise errors.OpPrereqError("The provided %s file is not a proper tar"
1112 " archive" % OVA_EXT, errors.ECODE_ENVIRON)
1113 ova_content = tarfile.open(input_path)
1114 temp_dir = tempfile.mkdtemp()
1115 self.temp_dir = temp_dir
1116 for file_name in ova_content.getnames():
1117 file_normname = os.path.normpath(file_name)
1118 try:
1119 utils.PathJoin(temp_dir, file_normname)
1120 except ValueError, err:
1121 raise errors.OpPrereqError("File %s inside %s package is not safe" %
1122 (file_name, OVA_EXT), errors.ECODE_ENVIRON)
1123 if file_name.endswith(OVF_EXT):
1124 input_name = file_name
1125 if not input_name:
1126 raise errors.OpPrereqError("No %s file in %s package found" %
1127 (OVF_EXT, OVA_EXT), errors.ECODE_ENVIRON)
1128 logging.warning("Unpacking the %s archive, this may take a while",
1129 input_path)
1130 self.input_dir = temp_dir
1131 self.input_path = utils.PathJoin(self.temp_dir, input_name)
1132 try:
1133 try:
1134 extract = ova_content.extractall
1135 except AttributeError:
1136 # This is a prehistorical case of using python < 2.5
1137 for member in ova_content.getmembers():
1138 ova_content.extract(member, path=self.temp_dir)
1139 else:
1140 extract(self.temp_dir)
1141 except tarfile.TarError, err:
1142 raise errors.OpPrereqError("Error while extracting %s archive: %s" %
1143 (OVA_EXT, err), errors.ECODE_ENVIRON)
1144 logging.info("OVA package extracted to %s directory", self.temp_dir)
1145
1146 def Parse(self):
1147 """Parses the data and creates a structure containing all required info.
1148
1149 The method reads the information given either as a command line option or as
1150 a part of the OVF description.
1151
1152 @raise errors.OpPrereqError: if some required part of the description of
1153 virtual instance is missing or unable to create output directory
1154
1155 """
1156 self.results_name = self._GetInfo("instance name", self.options.name,
1157 self._ParseNameOptions,
1158 self.ovf_reader.GetInstanceName)
1159 if not self.results_name:
1160 raise errors.OpPrereqError("Name of instance not provided",
1161 errors.ECODE_INVAL)
1162
1163 self.output_dir = utils.PathJoin(self.output_dir, self.results_name)
1164 try:
1165 utils.Makedirs(self.output_dir)
1166 except OSError, err:
1167 raise errors.OpPrereqError("Failed to create directory %s: %s" %
1168 (self.output_dir, err), errors.ECODE_ENVIRON)
1169
1170 self.results_template = self._GetInfo(
1171 "disk template", self.options.disk_template, self._ParseTemplateOptions,
1172 self.ovf_reader.GetDiskTemplate)
1173 if not self.results_template:
1174 logging.info("Disk template not given")
1175
1176 self.results_hypervisor = self._GetInfo(
1177 "hypervisor", self.options.hypervisor, self._ParseHypervisorOptions,
1178 self.ovf_reader.GetHypervisorData)
1179 assert self.results_hypervisor["hypervisor_name"]
1180 if self.results_hypervisor["hypervisor_name"] == constants.VALUE_AUTO:
1181 logging.debug("Default hypervisor settings from the cluster will be used")
1182
1183 self.results_os = self._GetInfo(
1184 "OS", self.options.os, self._ParseOSOptions, self.ovf_reader.GetOSData)
1185 if not self.results_os.get("os_name"):
1186 raise errors.OpPrereqError("OS name must be provided",
1187 errors.ECODE_INVAL)
1188
1189 self.results_backend = self._GetInfo(
1190 "backend", self.options.beparams,
1191 self._ParseBackendOptions, self.ovf_reader.GetBackendData)
1192 assert self.results_backend.get("vcpus")
1193 assert self.results_backend.get("memory")
1194 assert self.results_backend.get("auto_balance") is not None
1195
1196 self.results_tags = self._GetInfo(
1197 "tags", self.options.tags, self._ParseTags, self.ovf_reader.GetTagsData)
1198
1199 ovf_version = self.ovf_reader.GetVersionData()
1200 if ovf_version:
1201 self.results_version = ovf_version
1202 else:
1203 self.results_version = constants.EXPORT_VERSION
1204
1205 self.results_network = self._GetInfo(
1206 "network", self.options.nics, self._ParseNicOptions,
1207 self.ovf_reader.GetNetworkData, ignore_test=self.options.no_nics)
1208
1209 self.results_disk = self._GetInfo(
1210 "disk", self.options.disks, self._ParseDiskOptions, self._GetDiskInfo,
1211 ignore_test=self.results_template == constants.DT_DISKLESS)
1212
1213 if not self.results_disk and not self.results_network:
1214 raise errors.OpPrereqError("Either disk specification or network"
1215 " description must be present",
1216 errors.ECODE_STATE)
1217
1218 @staticmethod
1219 def _GetInfo(name, cmd_arg, cmd_function, nocmd_function,
1220 ignore_test=False):
1221 """Get information about some section - e.g. disk, network, hypervisor.
1222
1223 @type name: string
1224 @param name: name of the section
1225 @type cmd_arg: dict
1226 @param cmd_arg: command line argument specific for section 'name'
1227 @type cmd_function: callable
1228 @param cmd_function: function to call if 'cmd_args' exists
1229 @type nocmd_function: callable
1230 @param nocmd_function: function to call if 'cmd_args' is not there
1231
1232 """
1233 if ignore_test:
1234 logging.info("Information for %s will be ignored", name)
1235 return {}
1236 if cmd_arg:
1237 logging.info("Information for %s will be parsed from command line", name)
1238 results = cmd_function()
1239 else:
1240 logging.info("Information for %s will be parsed from %s file",
1241 name, OVF_EXT)
1242 results = nocmd_function()
1243 logging.info("Options for %s were succesfully read", name)
1244 return results
1245
1246 def _ParseNameOptions(self):
1247 """Returns name if one was given in command line.
1248
1249 @rtype: string
1250 @return: name of an instance
1251
1252 """
1253 return self.options.name
1254
1255 def _ParseTemplateOptions(self):
1256 """Returns disk template if one was given in command line.
1257
1258 @rtype: string
1259 @return: disk template name
1260
1261 """
1262 return self.options.disk_template
1263
1264 def _ParseHypervisorOptions(self):
1265 """Parses hypervisor options given in a command line.
1266
1267 @rtype: dict
1268 @return: dictionary containing name of the chosen hypervisor and all the
1269 options
1270
1271 """
1272 assert type(self.options.hypervisor) is tuple
1273 assert len(self.options.hypervisor) == 2
1274 results = {}
1275 if self.options.hypervisor[0]:
1276 results["hypervisor_name"] = self.options.hypervisor[0]
1277 else:
1278 results["hypervisor_name"] = constants.VALUE_AUTO
1279 results.update(self.options.hypervisor[1])
1280 return results
1281
1282 def _ParseOSOptions(self):
1283 """Parses OS options given in command line.
1284
1285 @rtype: dict
1286 @return: dictionary containing name of chosen OS and all its options
1287
1288 """
1289 assert self.options.os
1290 results = {}
1291 results["os_name"] = self.options.os
1292 results.update(self.options.osparams)
1293 return results
1294
1295 def _ParseBackendOptions(self):
1296 """Parses backend options given in command line.
1297
1298 @rtype: dict
1299 @return: dictionary containing vcpus, memory and auto-balance options
1300
1301 """
1302 assert self.options.beparams
1303 backend = {}
1304 backend.update(self.options.beparams)
1305 must_contain = ["vcpus", "memory", "auto_balance"]
1306 for element in must_contain:
1307 if backend.get(element) is None:
1308 backend[element] = constants.VALUE_AUTO
1309 return backend
1310
1311 def _ParseTags(self):
1312 """Returns tags list given in command line.
1313
1314 @rtype: string
1315 @return: string containing comma-separated tags
1316
1317 """
1318 return self.options.tags
1319
1320 def _ParseNicOptions(self):
1321 """Parses network options given in a command line or as a dictionary.
1322
1323 @rtype: dict
1324 @return: dictionary of network-related options
1325
1326 """
1327 assert self.options.nics
1328 results = {}
1329 for (nic_id, nic_desc) in self.options.nics:
1330 results["nic%s_mode" % nic_id] = \
1331 nic_desc.get("mode", constants.VALUE_AUTO)
1332 results["nic%s_mac" % nic_id] = nic_desc.get("mac", constants.VALUE_AUTO)
1333 results["nic%s_link" % nic_id] = \
1334 nic_desc.get("link", constants.VALUE_AUTO)
1335 results["nic%s_network" % nic_id] = \
1336 nic_desc.get("network", constants.VALUE_AUTO)
1337 if nic_desc.get("mode") == "bridged":
1338 results["nic%s_ip" % nic_id] = constants.VALUE_NONE
1339 else:
1340 results["nic%s_ip" % nic_id] = constants.VALUE_AUTO
1341 results["nic_count"] = str(len(self.options.nics))
1342 return results
1343
1344 def _ParseDiskOptions(self):
1345 """Parses disk options given in a command line.
1346
1347 @rtype: dict
1348 @return: dictionary of disk-related options
1349
1350 @raise errors.OpPrereqError: disk description does not contain size
1351 information or size information is invalid or creation failed
1352
1353 """
1354 CheckQemuImg()
1355 assert self.options.disks
1356 results = {}
1357 for (disk_id, disk_desc) in self.options.disks:
1358 results["disk%s_ivname" % disk_id] = "disk/%s" % disk_id
1359 if disk_desc.get("size"):
1360 try:
1361 disk_size = utils.ParseUnit(disk_desc["size"])
1362 except ValueError:
1363 raise errors.OpPrereqError("Invalid disk size for disk %s: %s" %
1364 (disk_id, disk_desc["size"]),
1365 errors.ECODE_INVAL)
1366 new_path = utils.PathJoin(self.output_dir, str(disk_id))
1367 args = [
1368 constants.QEMUIMG_PATH,
1369 "create",
1370 "-f",
1371 "raw",
1372 new_path,
1373 disk_size,
1374 ]
1375 run_result = utils.RunCmd(args)
1376 if run_result.failed:
1377 raise errors.OpPrereqError("Creation of disk %s failed, output was:"
1378 " %s" % (new_path, run_result.stderr),
1379 errors.ECODE_ENVIRON)
1380 results["disk%s_size" % disk_id] = str(disk_size)
1381 results["disk%s_dump" % disk_id] = "disk%s.raw" % disk_id
1382 else:
1383 raise errors.OpPrereqError("Disks created for import must have their"
1384 " size specified",
1385 errors.ECODE_INVAL)
1386 results["disk_count"] = str(len(self.options.disks))
1387 return results
1388
1389 def _GetDiskInfo(self):
1390 """Gathers information about disks used by instance, perfomes conversion.
1391
1392 @rtype: dict
1393 @return: dictionary of disk-related options
1394
1395 @raise errors.OpPrereqError: disk is not in the same directory as .ovf file
1396
1397 """
1398 results = {}
1399 disks_list = self.ovf_reader.GetDisksNames()
1400 for (counter, (disk_name, disk_compression)) in enumerate(disks_list):
1401 if os.path.dirname(disk_name):
1402 raise errors.OpPrereqError("Disks are not allowed to have absolute"
1403 " paths or paths outside main OVF"
1404 " directory", errors.ECODE_ENVIRON)
1405 disk, _ = os.path.splitext(disk_name)
1406 disk_path = utils.PathJoin(self.input_dir, disk_name)
1407 if disk_compression not in NO_COMPRESSION:
1408 _, disk_path = self._CompressDisk(disk_path, disk_compression,
1409 DECOMPRESS)
1410 disk, _ = os.path.splitext(disk)
1411 if self._GetDiskQemuInfo(disk_path, "file format: (\S+)") != "raw":
1412 logging.info("Conversion to raw format is required")
1413 ext, new_disk_path = self._ConvertDisk("raw", disk_path)
1414
1415 final_disk_path = LinkFile(new_disk_path, prefix=disk, suffix=ext,
1416 directory=self.output_dir)
1417 final_name = os.path.basename(final_disk_path)
1418 disk_size = os.path.getsize(final_disk_path) / (1024 * 1024)
1419 results["disk%s_dump" % counter] = final_name
1420 results["disk%s_size" % counter] = str(disk_size)
1421 results["disk%s_ivname" % counter] = "disk/%s" % str(counter)
1422 if disks_list:
1423 results["disk_count"] = str(len(disks_list))
1424 return results
1425
1426 def Save(self):
1427 """Saves all the gathered information in a constant.EXPORT_CONF_FILE file.
1428
1429 @raise errors.OpPrereqError: when saving to config file failed
1430
1431 """
1432 logging.info("Conversion was succesfull, saving %s in %s directory",
1433 constants.EXPORT_CONF_FILE, self.output_dir)
1434 results = {
1435 constants.INISECT_INS: {},
1436 constants.INISECT_BEP: {},
1437 constants.INISECT_EXP: {},
1438 constants.INISECT_OSP: {},
1439 constants.INISECT_HYP: {},
1440 }
1441
1442 results[constants.INISECT_INS].update(self.results_disk)
1443 results[constants.INISECT_INS].update(self.results_network)
1444 results[constants.INISECT_INS]["hypervisor"] = \
1445 self.results_hypervisor["hypervisor_name"]
1446 results[constants.INISECT_INS]["name"] = self.results_name
1447 if self.results_template:
1448 results[constants.INISECT_INS]["disk_template"] = self.results_template
1449 if self.results_tags:
1450 results[constants.INISECT_INS]["tags"] = self.results_tags
1451
1452 results[constants.INISECT_BEP].update(self.results_backend)
1453
1454 results[constants.INISECT_EXP]["os"] = self.results_os["os_name"]
1455 results[constants.INISECT_EXP]["version"] = self.results_version
1456
1457 del self.results_os["os_name"]
1458 results[constants.INISECT_OSP].update(self.results_os)
1459
1460 del self.results_hypervisor["hypervisor_name"]
1461 results[constants.INISECT_HYP].update(self.results_hypervisor)
1462
1463 output_file_name = utils.PathJoin(self.output_dir,
1464 constants.EXPORT_CONF_FILE)
1465
1466 output = []
1467 for section, options in results.iteritems():
1468 output.append("[%s]" % section)
1469 for name, value in options.iteritems():
1470 if value is None:
1471 value = ""
1472 output.append("%s = %s" % (name, value))
1473 output.append("")
1474 output_contents = "\n".join(output)
1475
1476 try:
1477 utils.WriteFile(output_file_name, data=output_contents)
1478 except errors.ProgrammerError, err:
1479 raise errors.OpPrereqError("Saving the config file failed: %s" % err,
1480 errors.ECODE_ENVIRON)
1481
1482 self.Cleanup()
1483
1484
1485 class ConfigParserWithDefaults(ConfigParser.SafeConfigParser):
1486 """This is just a wrapper on SafeConfigParser, that uses default values
1487
1488 """
1489 def get(self, section, options, raw=None, vars=None): # pylint: disable=W0622
1490 try:
1491 result = ConfigParser.SafeConfigParser.get(self, section, options,
1492 raw=raw, vars=vars)
1493 except ConfigParser.NoOptionError:
1494 result = None
1495 return result
1496
1497 def getint(self, section, options):
1498 try:
1499 result = ConfigParser.SafeConfigParser.get(self, section, options)
1500 except ConfigParser.NoOptionError:
1501 result = 0
1502 return int(result)
1503
1504
1505 class OVFExporter(Converter):
1506 """Converter from Ganeti config file to OVF
1507
1508 @type input_dir: string
1509 @ivar input_dir: directory in which the config.ini file resides
1510 @type output_dir: string
1511 @ivar output_dir: directory to which the results of conversion shall be
1512 written
1513 @type packed_dir: string
1514 @ivar packed_dir: if we want OVA package, this points to the real (i.e. not
1515 temp) output directory
1516 @type input_path: string
1517 @ivar input_path: complete path to the config.ini file
1518 @type output_path: string
1519 @ivar output_path: complete path to .ovf file
1520 @type config_parser: L{ConfigParserWithDefaults}
1521 @ivar config_parser: parser for the config.ini file
1522 @type reference_files: list
1523 @ivar reference_files: files referenced in the ovf file
1524 @type results_disk: list
1525 @ivar results_disk: list of dictionaries of disk options from config.ini
1526 @type results_network: list
1527 @ivar results_network: list of dictionaries of network options form config.ini
1528 @type results_name: string
1529 @ivar results_name: name of the instance
1530 @type results_vcpus: string
1531 @ivar results_vcpus: number of VCPUs
1532 @type results_memory: string
1533 @ivar results_memory: RAM memory in MB
1534 @type results_ganeti: dict
1535 @ivar results_ganeti: dictionary of Ganeti-specific options from config.ini
1536
1537 """
1538 def _ReadInputData(self, input_path):
1539 """Reads the data on which the conversion will take place.
1540
1541 @type input_path: string
1542 @param input_path: absolute path to the config.ini input file
1543
1544 @raise errors.OpPrereqError: error when reading the config file
1545
1546 """
1547 input_dir = os.path.dirname(input_path)
1548 self.input_path = input_path
1549 self.input_dir = input_dir
1550 if self.options.output_dir:
1551 self.output_dir = os.path.abspath(self.options.output_dir)
1552 else:
1553 self.output_dir = input_dir
1554 self.config_parser = ConfigParserWithDefaults()
1555 logging.info("Reading configuration from %s file", input_path)
1556 try:
1557 self.config_parser.read(input_path)
1558 except ConfigParser.MissingSectionHeaderError, err:
1559 raise errors.OpPrereqError("Error when trying to read %s: %s" %
1560 (input_path, err), errors.ECODE_ENVIRON)
1561 if self.options.ova_package:
1562 self.temp_dir = tempfile.mkdtemp()
1563 self.packed_dir = self.output_dir
1564 self.output_dir = self.temp_dir
1565
1566 self.ovf_writer = OVFWriter(not self.options.ext_usage)
1567
1568 def _ParseName(self):
1569 """Parses name from command line options or config file.
1570
1571 @rtype: string
1572 @return: name of Ganeti instance
1573
1574 @raise errors.OpPrereqError: if name of the instance is not provided
1575
1576 """
1577 if self.options.name:
1578 name = self.options.name
1579 else:
1580 name = self.config_parser.get(constants.INISECT_INS, NAME)
1581 if name is None:
1582 raise errors.OpPrereqError("No instance name found",
1583 errors.ECODE_ENVIRON)
1584 return name
1585
1586 def _ParseVCPUs(self):
1587 """Parses vcpus number from config file.
1588
1589 @rtype: int
1590 @return: number of virtual CPUs
1591
1592 @raise errors.OpPrereqError: if number of VCPUs equals 0
1593
1594 """
1595 vcpus = self.config_parser.getint(constants.INISECT_BEP, VCPUS)
1596 if vcpus == 0:
1597 raise errors.OpPrereqError("No CPU information found",
1598 errors.ECODE_ENVIRON)
1599 return vcpus
1600
1601 def _ParseMemory(self):
1602 """Parses vcpus number from config file.
1603
1604 @rtype: int
1605 @return: amount of memory in MB
1606
1607 @raise errors.OpPrereqError: if amount of memory equals 0
1608
1609 """
1610 memory = self.config_parser.getint(constants.INISECT_BEP, MEMORY)
1611 if memory == 0:
1612 raise errors.OpPrereqError("No memory information found",
1613 errors.ECODE_ENVIRON)
1614 return memory
1615
1616 def _ParseGaneti(self):
1617 """Parses Ganeti data from config file.
1618
1619 @rtype: dictionary
1620 @return: dictionary of Ganeti-specific options
1621
1622 """
1623 results = {}
1624 # hypervisor
1625 results["hypervisor"] = {}
1626 hyp_name = self.config_parser.get(constants.INISECT_INS, HYPERV)
1627 if hyp_name is None:
1628 raise errors.OpPrereqError("No hypervisor information found",
1629 errors.ECODE_ENVIRON)
1630 results["hypervisor"]["name"] = hyp_name
1631 pairs = self.config_parser.items(constants.INISECT_HYP)
1632 for (name, value) in pairs:
1633 results["hypervisor"][name] = value
1634 # os
1635 results["os"] = {}
1636 os_name = self.config_parser.get(constants.INISECT_EXP, OS)
1637 if os_name is None:
1638 raise errors.OpPrereqError("No operating system information found",
1639 errors.ECODE_ENVIRON)
1640 results["os"]["name"] = os_name
1641 pairs = self.config_parser.items(constants.INISECT_OSP)
1642 for (name, value) in pairs:
1643 results["os"][name] = value
1644 # other
1645 others = [
1646 (constants.INISECT_INS, DISK_TEMPLATE, "disk_template"),
1647 (constants.INISECT_BEP, AUTO_BALANCE, "auto_balance"),
1648 (constants.INISECT_INS, TAGS, "tags"),
1649 (constants.INISECT_EXP, VERSION, "version"),
1650 ]
1651 for (section, element, name) in others:
1652 results[name] = self.config_parser.get(section, element)
1653 return results
1654
1655 def _ParseNetworks(self):
1656 """Parses network data from config file.
1657
1658 @rtype: list
1659 @return: list of dictionaries of network options
1660
1661 @raise errors.OpPrereqError: then network mode is not recognized
1662
1663 """
1664 results = []
1665 counter = 0
1666 while True:
1667 data_link = \
1668 self.config_parser.get(constants.INISECT_INS,
1669 "nic%s_link" % counter)
1670 if data_link is None:
1671 break
1672 results.append({
1673 "mode": self.config_parser.get(constants.INISECT_INS,
1674 "nic%s_mode" % counter),
1675 "mac": self.config_parser.get(constants.INISECT_INS,
1676 "nic%s_mac" % counter),
1677 "ip": self.config_parser.get(constants.INISECT_INS,
1678 "nic%s_ip" % counter),
1679 "network": self.config_parser.get(constants.INISECT_INS,
1680 "nic%s_network" % counter),
1681 "link": data_link,
1682 })
1683 if results[counter]["mode"] not in constants.NIC_VALID_MODES:
1684 raise errors.OpPrereqError("Network mode %s not recognized"
1685 % results[counter]["mode"],
1686 errors.ECODE_INVAL)
1687 counter += 1
1688 return results
1689
1690 def _GetDiskOptions(self, disk_file, compression):
1691 """Convert the disk and gather disk info for .ovf file.
1692
1693 @type disk_file: string
1694 @param disk_file: name of the disk (without the full path)
1695 @type compression: bool
1696 @param compression: whether the disk should be compressed or not
1697
1698 @raise errors.OpPrereqError: when disk image does not exist
1699
1700 """
1701 disk_path = utils.PathJoin(self.input_dir, disk_file)
1702 results = {}
1703 if not os.path.isfile(disk_path):
1704 raise errors.OpPrereqError("Disk image does not exist: %s" % disk_path,
1705 errors.ECODE_ENVIRON)
1706 if os.path.dirname(disk_file):
1707 raise errors.OpPrereqError("Path for the disk: %s contains a directory"
1708 " name" % disk_path, errors.ECODE_ENVIRON)
1709 disk_name, _ = os.path.splitext(disk_file)
1710 ext, new_disk_path = self._ConvertDisk(self.options.disk_format, disk_path)
1711 results["format"] = self.options.disk_format
1712 results["virt-size"] = self._GetDiskQemuInfo(
1713 new_disk_path, "virtual size: \S+ \((\d+) bytes\)")
1714 if compression:
1715 ext2, new_disk_path = self._CompressDisk(new_disk_path, "gzip",
1716 COMPRESS)
1717 disk_name, _ = os.path.splitext(disk_name)
1718 results["compression"] = "gzip"
1719 ext += ext2
1720 final_disk_path = LinkFile(new_disk_path, prefix=disk_name, suffix=ext,
1721 directory=self.output_dir)
1722 final_disk_name = os.path.basename(final_disk_path)
1723 results["real-size"] = os.path.getsize(final_disk_path)
1724 results["path"] = final_disk_name
1725 self.references_files.append(final_disk_path)
1726 return results
1727
1728 def _ParseDisks(self):
1729 """Parses disk data from config file.
1730
1731 @rtype: list
1732 @return: list of dictionaries of disk options
1733
1734 """
1735 results = []
1736 counter = 0
1737 while True:
1738 disk_file = \
1739 self.config_parser.get(constants.INISECT_INS, "disk%s_dump" % counter)
1740 if disk_file is None:
1741 break
1742 results.append(self._GetDiskOptions(disk_file, self.options.compression))
1743 counter += 1
1744 return results
1745
1746 def Parse(self):
1747 """Parses the data and creates a structure containing all required info.
1748
1749 """
1750 try:
1751 utils.Makedirs(self.output_dir)
1752 except OSError, err:
1753 raise errors.OpPrereqError("Failed to create directory %s: %s" %
1754 (self.output_dir, err), errors.ECODE_ENVIRON)
1755
1756 self.references_files = []
1757 self.results_name = self._ParseName()
1758 self.results_vcpus = self._ParseVCPUs()
1759 self.results_memory = self._ParseMemory()
1760 if not self.options.ext_usage:
1761 self.results_ganeti = self._ParseGaneti()
1762 self.results_network = self._ParseNetworks()
1763 self.results_disk = self._ParseDisks()
1764
1765 def _PrepareManifest(self, path):
1766 """Creates manifest for all the files in OVF package.
1767
1768 @type path: string
1769 @param path: path to manifesto file
1770
1771 @raise errors.OpPrereqError: if error occurs when writing file
1772
1773 """
1774 logging.info("Preparing manifest for the OVF package")
1775 lines = []
1776 files_list = [self.output_path]
1777 files_list.extend(self.references_files)
1778 logging.warning("Calculating SHA1 checksums, this may take a while")
1779 sha1_sums = utils.FingerprintFiles(files_list)
1780 for file_path, value in sha1_sums.iteritems():
1781 file_name = os.path.basename(file_path)
1782 lines.append("SHA1(%s)= %s" % (file_name, value))
1783 lines.append("")
1784 data = "\n".join(lines)
1785 try:
1786 utils.WriteFile(path, data=data)
1787 except errors.ProgrammerError, err:
1788 raise errors.OpPrereqError("Saving the manifest file failed: %s" % err,
1789 errors.ECODE_ENVIRON)
1790
1791 @staticmethod
1792 def _PrepareTarFile(tar_path, files_list):
1793 """Creates tarfile from the files in OVF package.
1794
1795 @type tar_path: string
1796 @param tar_path: path to the resulting file
1797 @type files_list: list
1798 @param files_list: list of files in the OVF package
1799
1800 """
1801 logging.info("Preparing tarball for the OVF package")
1802 open(tar_path, mode="w").close()
1803 ova_package = tarfile.open(name=tar_path, mode="w")
1804 for file_path in files_list:
1805 file_name = os.path.basename(file_path)
1806 ova_package.add(file_path, arcname=file_name)
1807 ova_package.close()
1808
1809 def Save(self):
1810 """Saves the gathered configuration in an apropriate format.
1811
1812 @raise errors.OpPrereqError: if unable to create output directory
1813
1814 """
1815 output_file = "%s%s" % (self.results_name, OVF_EXT)
1816 output_path = utils.PathJoin(self.output_dir, output_file)
1817 self.ovf_writer = OVFWriter(not self.options.ext_usage)
1818 logging.info("Saving read data to %s", output_path)
1819
1820 self.output_path = utils.PathJoin(self.output_dir, output_file)
1821 files_list = [self.output_path]
1822
1823 self.ovf_writer.SaveDisksData(self.results_disk)
1824 self.ovf_writer.SaveNetworksData(self.results_network)
1825 if not self.options.ext_usage:
1826 self.ovf_writer.SaveGanetiData(self.results_ganeti, self.results_network)
1827
1828 self.ovf_writer.SaveVirtualSystemData(self.results_name, self.results_vcpus,
1829 self.results_memory)
1830
1831 data = self.ovf_writer.PrettyXmlDump()
1832 utils.WriteFile(self.output_path, data=data)
1833
1834 manifest_file = "%s%s" % (self.results_name, MF_EXT)
1835 manifest_path = utils.PathJoin(self.output_dir, manifest_file)
1836 self._PrepareManifest(manifest_path)
1837 files_list.append(manifest_path)
1838
1839 files_list.extend(self.references_files)
1840
1841 if self.options.ova_package:
1842 ova_file = "%s%s" % (self.results_name, OVA_EXT)
1843 packed_path = utils.PathJoin(self.packed_dir, ova_file)
1844 try:
1845 utils.Makedirs(self.packed_dir)
1846 except OSError, err:
1847 raise errors.OpPrereqError("Failed to create directory %s: %s" %
1848 (self.packed_dir, err),
1849 errors.ECODE_ENVIRON)
1850 self._PrepareTarFile(packed_path, files_list)
1851 logging.info("Creation of the OVF package was successfull")
1852 self.Cleanup()