Add the compression tools parameter

This patch makes the myriad of changes necessary for the compression
tool parameter to be added. The filtering of compression tools for
suspicious entries has been added for this exact purpose.

Signed-off-by: Hrvoje Ribicic <riba@google.com>
Reviewed-by: Thomas Thrainer <thomasth@google.com>

diff --git a/lib/bootstrap.py b/lib/bootstrap.py
index 05b5a7b..bc3e90d 100644 (file)
--- a/lib/bootstrap.py
+++ b/lib/bootstrap.py
@@ -553,7 +553,7 @@ def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
                 primary_ip_version=None, ipolicy=None,
                 prealloc_wipe_disks=False, use_external_mip_script=False,
                 hv_state=None, disk_state=None, enabled_disk_templates=None,
-                zeroing_image=None):
+                zeroing_image=None, compression_tools=None):
   """Initialise the cluster.
 
   @type candidate_pool_size: int
@@ -780,6 +780,9 @@ def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
 
   now = time.time()
 
+  if compression_tools is not None:
+    cluster.CheckCompressionTools(compression_tools)
+
   # init of cluster config file
   cluster_config = objects.Cluster(
     serial_no=1,
@@ -822,7 +825,8 @@ def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
     candidate_certs=candidate_certs,
     osparams={},
     osparams_private_cluster={},
-    zeroing_image=zeroing_image
+    zeroing_image=zeroing_image,
+    compression_tools=compression_tools
     )
   master_node_config = objects.Node(name=hostname.name,
                                     primary_ip=hostname.ip,

diff --git a/lib/cli.py b/lib/cli.py
index 95461b7..2ca7772 100644 (file)
--- a/lib/cli.py
+++ b/lib/cli.py
@@ -71,6 +71,7 @@ __all__ = [
   "CLUSTER_DOMAIN_SECRET_OPT",
   "CONFIRM_OPT",
   "CP_SIZE_OPT",
+  "COMPRESSION_TOOLS_OPT",
   "DEBUG_OPT",
   "DEBUG_SIMERR_OPT",
   "DISKIDX_OPT",
@@ -1315,6 +1316,12 @@ INSTANCE_COMMUNICATION_NETWORK_OPT = \
                type="string",
                help="Set the network name for instance communication")
 
+COMPRESSION_TOOLS_OPT = \
+    cli_option("--compression-tools",
+               dest="compression_tools", type="string", default=None,
+               help="Comma-separated list of compression tools which are"
+                    " allowed to be used by Ganeti in various operations")
+
 VG_NAME_OPT = cli_option("--vg-name", dest="vg_name",
                          help=("Enables LVM and specifies the volume group"
                                " name (cluster-wide) for disk allocation"

diff --git a/lib/client/gnt_cluster.py b/lib/client/gnt_cluster.py
index f6e38f0..6669a19 100644 (file)
--- a/lib/client/gnt_cluster.py
+++ b/lib/client/gnt_cluster.py
@@ -270,6 +270,8 @@ def InitCluster(opts, args):
   else:
     zeroing_image = ""
 
+  compression_tools = _GetCompressionTools(opts)
+
   default_ialloc_params = opts.default_iallocator_params
   bootstrap.InitCluster(cluster_name=args[0],
                         secondary_ip=opts.secondary_ip,
@@ -301,7 +303,8 @@ def InitCluster(opts, args):
                         hv_state=hv_state,
                         disk_state=disk_state,
                         enabled_disk_templates=enabled_disk_templates,
-                        zeroing_image=zeroing_image
+                        zeroing_image=zeroing_image,
+                        compression_tools=compression_tools
                         )
   op = opcodes.OpClusterPostInit()
   SubmitOpCode(op, opts=opts)
@@ -555,6 +558,7 @@ def ShowClusterConfig(opts, args):
       ("instance communication network",
        result["instance_communication_network"]),
       ("zeroing image", result["zeroing_image"]),
+      ("compression tools", result["compression_tools"]),
       ]),
 
     ("Default node parameters",
@@ -1092,6 +1096,18 @@ def _GetDrbdHelper(opts, enabled_disk_templates):
   return drbd_helper
 
 
+def _GetCompressionTools(opts):
+  """Determine the list of custom compression tools.
+
+  """
+  if opts.compression_tools:
+    return opts.compression_tools.split(",")
+  elif opts.compression_tools is None:
+    return None # To note the parameter was not provided
+  else:
+    return constants.IEC_DEFAULT_TOOLS # Resetting to default
+
+
 def SetClusterParams(opts, args):
   """Modify the cluster.
 
@@ -1133,7 +1149,8 @@ def SetClusterParams(opts, args):
           opts.file_storage_dir is not None or
           opts.instance_communication_network is not None or
           opts.zeroing_image is not None or
-          opts.shared_file_storage_dir is not None):
+          opts.shared_file_storage_dir is not None or
+          opts.compression_tools is not None):
     ToStderr("Please give at least one of the parameters.")
     return 1
 
@@ -1214,6 +1231,8 @@ def SetClusterParams(opts, args):
 
   hv_state = dict(opts.hv_state)
 
+  compression_tools = _GetCompressionTools(opts)
+
   op = opcodes.OpClusterSetParams(
     vg_name=vg_name,
     drbd_helper=drbd_helper,
@@ -1248,6 +1267,7 @@ def SetClusterParams(opts, args):
     instance_communication_network=opts.instance_communication_network,
     zeroing_image=opts.zeroing_image,
     shared_file_storage_dir=opts.shared_file_storage_dir,
+    compression_tools=compression_tools
     )
   return base.GetResult(None, opts, SubmitOrSend(op, opts))
 
@@ -2112,7 +2132,8 @@ commands = {
      PRIMARY_IP_VERSION_OPT, PREALLOC_WIPE_DISKS_OPT, NODE_PARAMS_OPT,
      GLOBAL_SHARED_FILEDIR_OPT, USE_EXTERNAL_MIP_SCRIPT, DISK_PARAMS_OPT,
      HV_STATE_OPT, DISK_STATE_OPT, ENABLED_DISK_TEMPLATES_OPT,
-     IPOLICY_STD_SPECS_OPT, GLOBAL_GLUSTER_FILEDIR_OPT, ZEROING_IMAGE_OPT]
+     IPOLICY_STD_SPECS_OPT, GLOBAL_GLUSTER_FILEDIR_OPT, ZEROING_IMAGE_OPT,
+     COMPRESSION_TOOLS_OPT]
      + INSTANCE_POLICY_OPTS + SPLIT_ISPECS_OPTS,
     "[opts...] <cluster_name>", "Initialises a new cluster configuration"),
   "destroy": (
@@ -2197,7 +2218,8 @@ commands = {
      SUBMIT_OPTS +
      [ENABLED_DISK_TEMPLATES_OPT, IPOLICY_STD_SPECS_OPT, MODIFY_ETCHOSTS_OPT] +
      INSTANCE_POLICY_OPTS +
-     [GLOBAL_FILEDIR_OPT, GLOBAL_SHARED_FILEDIR_OPT, ZEROING_IMAGE_OPT],
+     [GLOBAL_FILEDIR_OPT, GLOBAL_SHARED_FILEDIR_OPT, ZEROING_IMAGE_OPT,
+      COMPRESSION_TOOLS_OPT],
     "[opts...]",
     "Alters the parameters of the cluster"),
   "renew-crypto": (

diff --git a/lib/cmdlib/cluster.py b/lib/cmdlib/cluster.py
index 646a1cd..64a1ef3 100644 (file)
--- a/lib/cmdlib/cluster.py
+++ b/lib/cmdlib/cluster.py
@@ -416,6 +416,7 @@ class LUClusterQuery(NoHooksLU):
       "blacklisted_os": cluster.blacklisted_os,
       "enabled_disk_templates": cluster.enabled_disk_templates,
       "instance_communication_network": cluster.instance_communication_network,
+      "compression_tools": cluster.compression_tools,
       }
 
     return result
@@ -767,6 +768,33 @@ def CheckSharedFileStoragePathVsEnabledDiskTemplates(
       constants.DT_SHARED_FILE)
 
 
+def CheckCompressionTools(tools):
+  """Check whether the provided compression tools look like executables.
+
+  @type tools: list of string
+  @param tools: The tools provided as opcode input
+
+  """
+  regex = re.compile('^[-_a-zA-Z0-9]+$')
+  illegal_tools = [t for t in tools if not regex.match(t)]
+
+  if illegal_tools:
+    raise errors.OpPrereqError(
+      "The tools '%s' contain illegal characters: only alphanumeric values,"
+      " dashes, and underscores are allowed" % ", ".join(illegal_tools)
+    )
+
+  if constants.IEC_GZIP not in tools:
+    raise errors.OpPrereqError("For compatibility reasons, the %s utility must"
+                               " be present among the compression tools" %
+                               constants.IEC_GZIP)
+
+  if constants.IEC_NONE in tools:
+    raise errors.OpPrereqError("%s is a reserved value used for no compression,"
+                               " and cannot be used as the name of a tool" %
+                               constants.IEC_NONE)
+
+
 class LUClusterSetParams(LogicalUnit):
   """Change the parameters of the cluster.
 
@@ -1273,6 +1301,9 @@ class LUClusterSetParams(LogicalUnit):
         network = self.cfg.GetNetwork(network_uuid)
         self._CheckInstanceCommunicationNetwork(network, self.LogWarning)
 
+    if self.op.compression_tools:
+      CheckCompressionTools(self.op.compression_tools)
+
   def _BuildOSParams(self, cluster):
     "Calculate the new OS parameters for this operation."
 
@@ -1645,6 +1676,9 @@ class LUClusterSetParams(LogicalUnit):
       result.Warn("Could not re-enable the master ip on the master,"
                   " please restart manually", self.LogWarning)
 
+    if self.op.compression_tools is not None:
+      self.cfg.SetCompressionTools(self.op.compression_tools)
+
     network_name = self.op.instance_communication_network
     if network_name is not None:
       return self._ModifyInstanceCommunicationNetwork(self.cfg,

diff --git a/lib/config.py b/lib/config.py
index dae0e3d..529e6f9 100644 (file)
--- a/lib/config.py
+++ b/lib/config.py
@@ -1612,6 +1612,28 @@ class ConfigWriter(object):
     """
     return self._config_data.cluster.zeroing_image
 
+  @_ConfigSync(shared=1)
+  def GetCompressionTools(self):
+    """Get cluster compression tools
+
+    @rtype: list of string
+    @return: a list of tools that are cleared for use in this cluster for the
+             purpose of compressing data
+
+    """
+    return self._ConfigData().cluster.compression_tools
+
+  @_ConfigSync()
+  def SetCompressionTools(self, tools):
+    """Set cluster compression tools
+
+    @type tools: list of string
+    @param tools: a list of tools that are cleared for use in this cluster for
+                  the purpose of compressing data
+
+    """
+    self._ConfigData().cluster.compression_tools = tools
+
   @_ConfigSync()
   def AddNodeGroup(self, group, ec_id, check_uuid=True):
     """Add a node group to the configuration.

diff --git a/lib/objects.py b/lib/objects.py
index d74a9f8..3d8690d 100644 (file)
--- a/lib/objects.py
+++ b/lib/objects.py
@@ -1589,7 +1589,8 @@ class Cluster(TaggableObject):
     "candidate_certs",
     "max_running_jobs",
     "instance_communication_network",
-    "zeroing_image"
+    "zeroing_image",
+    "compression_tools",
     ] + _TIMESTAMPS + _UUID
 
   def UpgradeConfig(self):
@@ -1727,6 +1728,9 @@ class Cluster(TaggableObject):
     if self.instance_communication_network is None:
       self.instance_communication_network = ""
 
+    if self.compression_tools is None:
+      self.compression_tools = constants.IEC_DEFAULT_TOOLS
+
   @property
   def primary_hypervisor(self):
     """The first hypervisor is the primary.

diff --git a/src/Ganeti/Constants.hs b/src/Ganeti/Constants.hs
index 6510a44..59b564b 100644 (file)
--- a/src/Ganeti/Constants.hs
+++ b/src/Ganeti/Constants.hs
@@ -601,6 +601,9 @@ iecNone = "none"
 iecAll :: [String]
 iecAll = [iecGzip, iecGzipFast, iecGzipSlow, iecLzop, iecNone]
 
+iecDefaultTools :: [String]
+iecDefaultTools = [iecGzip, iecGzipFast, iecGzipSlow]
+
 iecCompressionUtilities :: Map String String
 iecCompressionUtilities =
   Map.fromList

diff --git a/src/Ganeti/Objects.hs b/src/Ganeti/Objects.hs
index 1d12d0d..6b2b344 100644 (file)
--- a/src/Ganeti/Objects.hs
+++ b/src/Ganeti/Objects.hs
@@ -725,6 +725,7 @@ $(buildObject "Cluster" "cluster" $
   , simpleField "max_running_jobs"               [t| Int                    |]
   , simpleField "instance_communication_network" [t| String                 |]
   , simpleField "zeroing_image"                  [t| String                 |]
+  , simpleField "compression_tools"              [t| [String]               |]
  ]
  ++ timeStampFields
  ++ uuidFields

diff --git a/src/Ganeti/OpCodes.hs b/src/Ganeti/OpCodes.hs
index f0f4878..cd48689 100644 (file)
--- a/src/Ganeti/OpCodes.hs
+++ b/src/Ganeti/OpCodes.hs
@@ -241,6 +241,7 @@ $(genOpCode "OpCode"
      , pClusterGlusterStorageDir
      , pInstanceCommunicationNetwork
      , pZeroingImage
+     , pCompressionTools
      ],
      [])
   , ("OpClusterRedistConf",

diff --git a/src/Ganeti/OpParams.hs b/src/Ganeti/OpParams.hs
index eb66b80..97f5b38 100644 (file)
--- a/src/Ganeti/OpParams.hs
+++ b/src/Ganeti/OpParams.hs
@@ -111,6 +111,7 @@ module Ganeti.OpParams
   , pClusterGlusterStorageDir
   , pInstanceCommunicationNetwork
   , pZeroingImage
+  , pCompressionTools
   , pVgName
   , pEnabledHypervisors
   , pHypervisor
@@ -604,6 +605,12 @@ pZeroingImage :: Field
 pZeroingImage =
   optionalStringField "zeroing_image"
 
+-- | The additional tools that can be used to compress data in transit
+pCompressionTools :: Field
+pCompressionTools =
+  withDoc "List of enabled compression tools" . optionalField $
+  simpleField "compression_tools" [t| [NonEmptyString] |]
+
 -- | Volume group name.
 pVgName :: Field
 pVgName =

diff --git a/src/Ganeti/Query/Server.hs b/src/Ganeti/Query/Server.hs
index b1e470a..d0784a0 100644 (file)
--- a/src/Ganeti/Query/Server.hs
+++ b/src/Ganeti/Query/Server.hs
@@ -173,6 +173,8 @@ handleCall _ _ cdata QueryClusterInfo =
             , ("instance_communication_network",
                showJSON (clusterInstanceCommunicationNetwork cluster))
             , ("zeroing_image", showJSON $ clusterZeroingImage cluster)
+            , ("compression_tools",
+               showJSON $ clusterCompressionTools cluster)
             ]
 
   in case master of

diff --git a/test/hs/Test/Ganeti/OpCodes.hs b/test/hs/Test/Ganeti/OpCodes.hs
index a85b30f..e2231db 100644 (file)
--- a/test/hs/Test/Ganeti/OpCodes.hs
+++ b/test/hs/Test/Ganeti/OpCodes.hs
@@ -208,6 +208,7 @@ instance Arbitrary OpCodes.OpCode where
           <*> genMaybe genName             -- gluster_file_storage_dir
           <*> arbitrary                    -- instance_communication_network
           <*> arbitrary                    -- zeroing_image
+          <*> arbitrary                    -- compression_tools
       "OP_CLUSTER_REDIST_CONF" -> pure OpCodes.OpClusterRedistConf
       "OP_CLUSTER_ACTIVATE_MASTER_IP" ->
         pure OpCodes.OpClusterActivateMasterIp

diff --git a/test/py/cfgupgrade_unittest.py b/test/py/cfgupgrade_unittest.py
index 1b48d2b..a5648d4 100755 (executable)
--- a/test/py/cfgupgrade_unittest.py
+++ b/test/py/cfgupgrade_unittest.py
@@ -50,6 +50,7 @@ def GetMinimalConfig():
       "candidate_certs": {},
       "instance_communication_network": "",
       "zeroing_image": "",
+      "compression_tools": constants.IEC_DEFAULT_TOOLS,
     },
     "instances": {},
     "disks": {},

diff --git a/tools/cfgupgrade b/tools/cfgupgrade
index 9c3574d..2e872e7 100755 (executable)
--- a/tools/cfgupgrade
+++ b/tools/cfgupgrade
@@ -156,6 +156,8 @@ def UpgradeCluster(config_data):
     cluster.get("instance_communication_network", "")
   cluster["zeroing_image"] = \
     cluster.get("zeroing_image", "")
+  cluster["compression_tools"] = \
+    cluster.get("compression_tools", constants.IEC_DEFAULT_TOOLS)
 
 
 def UpgradeGroups(config_data):
@@ -455,6 +457,9 @@ def DowngradeCluster(config_data):
   if "zeroing_image" in cluster:
     del cluster["zeroing_image"]
 
+  if "compression_tools" in cluster:
+    del cluster["compression_tools"]
+
 
 def DowngradeGroups(config_data):
   for group in config_data["nodegroups"].values():