Merge branch 'stable-2.11' into stable-2.12

* stable-2.11
  Renew crypto retries for non-master nodes
  Retries for the master's SSL cert renewal
  Unit tests for offline nodes
  De-duplicate testing code regarding pathutils
  Make LURenewCrypto handle unreachable nodes properly
  Error handling on failed SSL cert renewal for master
  Unit test for LURenewCrypto's valid case
  Mock support for pathutils
  Increase timeout of crypto token RPC

 Conflicts:
	lib/cmdlib/cluster.py
	test/py/cmdlib/cluster_unittest.py
	test/py/cmdlib/testsupport/__init__.py

 Resolution:
        for all affected files: update changes to
        configuration handling

Signed-off-by: Helga Velroyen <helgav@google.com>
Reviewed-by: Petr Pudlak <pudlak@google.com>

diff --cc Makefile.am
index a61755b,a384351..c9e64cd
--- 1/Makefile.am
--- 2/Makefile.am
+++ b/Makefile.am
@@@ -1756,8 -1572,9 +1756,9 @@@ python_test_support = 
        test/py/cmdlib/testsupport/cmdlib_testcase.py \
        test/py/cmdlib/testsupport/config_mock.py \
        test/py/cmdlib/testsupport/iallocator_mock.py \
 -      test/py/cmdlib/testsupport/lock_manager_mock.py \
 +      test/py/cmdlib/testsupport/livelock_mock.py \
        test/py/cmdlib/testsupport/netutils_mock.py \
+       test/py/cmdlib/testsupport/pathutils_mock.py \
        test/py/cmdlib/testsupport/processor_mock.py \
        test/py/cmdlib/testsupport/rpc_runner_mock.py \
        test/py/cmdlib/testsupport/ssh_mock.py \

diff --cc lib/cmdlib/cluster.py
index 5f6504a,0f4b2eb..828212f
--- 1/lib/cmdlib/cluster.py
--- 2/lib/cmdlib/cluster.py
+++ b/lib/cmdlib/cluster.py
@@@ -110,8 -111,11 +110,10 @@@ class LUClusterRenewCrypto(NoHooksLU)
    takes care of the renewal of the client SSL certificates.
  
    """
+   _MAX_NUM_RETRIES = 3
+ 
    def Exec(self, feedback_fn):
      master_uuid = self.cfg.GetMasterNode()
 -    cluster = self.cfg.GetClusterInfo()
  
      server_digest = utils.GetCertificateDigest(
        cert_filename=pathutils.NODED_CERT_FILE)
@@@ -125,21 -131,67 +127,60 @@@
      except IOError:
        logging.info("No old certificate available.")
  
-     new_master_digest = _UpdateMasterClientCert(self, self.cfg, master_uuid)
- 
-     self.cfg.AddNodeToCandidateCerts(master_uuid, new_master_digest)
+     for _ in range(self._MAX_NUM_RETRIES):
+       try:
+         # Technically it should not be necessary to set the cert
+         # paths. However, due to a bug in the mock library, we
+         # have to do this to be able to test the function properly.
+         _UpdateMasterClientCert(
 -            self, master_uuid, cluster, feedback_fn,
++            self, self.cfg, master_uuid,
+             client_cert=pathutils.NODED_CLIENT_CERT_FILE,
+             client_cert_tmp=pathutils.NODED_CLIENT_CERT_FILE_TMP)
+         break
+       except errors.OpExecError as e:
+         pass
+     else:
+       feedback_fn("Could not renew the master's client SSL certificate."
+                    " Cleaning up. Error: %s." % e)
+       # Cleaning up temporary certificates
 -      utils.RemoveNodeFromCandidateCerts("%s-SERVER" % master_uuid,
 -                                         cluster.candidate_certs)
 -      utils.RemoveNodeFromCandidateCerts("%s-OLDMASTER" % master_uuid,
 -                                         cluster.candidate_certs)
++      self.cfg.RemoveNodeFromCandidateCerts("%s-SERVER" % master_uuid)
++      self.cfg.RemoveNodeFromCandidateCerts("%s-OLDMASTER" % master_uuid)
+       try:
+         utils.RemoveFile(pathutils.NODED_CLIENT_CERT_FILE_TMP)
+       except IOError:
+         pass
+       return
+ 
+     node_errors = {}
      nodes = self.cfg.GetAllNodesInfo()
      for (node_uuid, node_info) in nodes.items():
        if node_info.offline:
          feedback_fn("* Skipping offline node %s" % node_info.name)
          continue
        if node_uuid != master_uuid:
-         new_digest = CreateNewClientCert(self, node_uuid)
-         if node_info.master_candidate:
-           self.cfg.AddNodeToCandidateCerts(node_uuid, new_digest)
+         for _ in range(self._MAX_NUM_RETRIES):
+           try:
+             new_digest = CreateNewClientCert(self, node_uuid)
+             if node_info.master_candidate:
 -              utils.AddNodeToCandidateCerts(node_uuid,
 -                                            new_digest,
 -                                            cluster.candidate_certs)
++              self.cfg.AddNodeToCandidateCerts(node_uuid,
++                                               new_digest)
+             break
+           except errors.OpExecError as last_exception:
+             pass
+         else:
+           if last_exception:
+             node_errors[node_uuid] = last_exception
+ 
+     if node_errors:
+       msg = ("Some nodes' SSL client certificates could not be renewed."
+              " Please make sure those nodes are reachable and rerun"
+              " the operation. The affected nodes and their errors are:\n")
+       for uuid, e in node_errors.items():
+         msg += "Node %s: %s\n" % (uuid, e)
+       feedback_fn(msg)
+ 
 -    utils.RemoveNodeFromCandidateCerts("%s-SERVER" % master_uuid,
 -                                       cluster.candidate_certs)
 -    utils.RemoveNodeFromCandidateCerts("%s-OLDMASTER" % master_uuid,
 -                                       cluster.candidate_certs)
 -    # Trigger another update of the config now with the new master cert
 -    self.cfg.Update(cluster, feedback_fn)
 +    self.cfg.RemoveNodeFromCandidateCerts("%s-SERVER" % master_uuid)
 +    self.cfg.RemoveNodeFromCandidateCerts("%s-OLDMASTER" % master_uuid)
-     # Trigger another update of the config now with the new master cert
  
  
  class LUClusterActivateMasterIp(NoHooksLU):

diff --cc lib/rpc_defs.py
Simple merge

diff --cc test/py/cmdlib/cluster_unittest.py
index 1a90972,073b8de..eaed548
--- 1/test/py/cmdlib/cluster_unittest.py
--- 2/test/py/cmdlib/cluster_unittest.py
+++ b/test/py/cmdlib/cluster_unittest.py
@@@ -37,7 -37,8 +37,9 @@@ import OpenSS
  import copy
  import unittest
  import operator
 +import re
+ import shutil
+ import os
  
  from ganeti.cmdlib import cluster
  from ganeti import constants

diff --cc test/py/cmdlib/testsupport/__init__.py
index e25fc27,d121c37..a1f0678
--- 1/test/py/cmdlib/testsupport/__init__.py
--- 2/test/py/cmdlib/testsupport/__init__.py
+++ b/test/py/cmdlib/testsupport/__init__.py
@@@ -36,10 -36,11 +36,11 @@@ from cmdlib.testsupport.cmdlib_testcas
    withLockedLU
  from cmdlib.testsupport.config_mock import ConfigMock
  from cmdlib.testsupport.iallocator_mock import patchIAllocator
 +from cmdlib.testsupport.livelock_mock import LiveLockMock
  from cmdlib.testsupport.utils_mock import patchUtils
 -from cmdlib.testsupport.lock_manager_mock import LockManagerMock
  from cmdlib.testsupport.netutils_mock import patchNetutils, HostnameMock
  from cmdlib.testsupport.processor_mock import ProcessorMock
+ from cmdlib.testsupport.pathutils_mock import patchPathutils
  from cmdlib.testsupport.rpc_runner_mock import CreateRpcRunnerMock, \
    RpcResultsBuilder
  from cmdlib.testsupport.ssh_mock import patchSsh
@@@ -54,8 -54,8 +55,9 @@@ __all__ = ["CmdlibTestCase"
             "patchUtils",
             "patchNetutils",
             "patchSsh",
+            "patchPathutils",
 -           "LockManagerMock",
             "ProcessorMock",
             "RpcResultsBuilder",
 +           "LiveLockMock",
 +           "WConfdMock",
             ]