Merge keystone/next into change to get ready for final merge

Makefile

@@ -13,6 +13,7 @@ test:
 
 functional_test:
 	@echo Starting Amulet tests...
+	@tests/setup/00-setup
 	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
 
 bin/charm_helpers_sync.py:

charm-helpers-hooks.yaml

@@ -11,7 +11,7 @@ include:
         - cluster
     - contrib.python
     - contrib.unison
-    - payload.execd
+    - payload
     - contrib.peerstorage
     - contrib.network.ip
     - contrib.python.packages

charmhelpers/contrib/openstack/neutron.py

@@ -237,12 +237,14 @@ def neutron_plugins():
         plugins['midonet']['driver'] = (
             'neutron.plugins.midonet.plugin.MidonetPluginV2')
     if release >= 'liberty':
-        plugins['midonet']['driver'] = (
-            'midonet.neutron.plugin_v1.MidonetPluginV2')
-        plugins['midonet']['server_packages'].remove(
-            'python-neutron-plugin-midonet')
-        plugins['midonet']['server_packages'].append(
-            'python-networking-midonet')
+        midonet_origin = config('midonet-origin')
+        if midonet_origin is not None and midonet_origin[4:5] == '1':
+            plugins['midonet']['driver'] = (
+                'midonet.neutron.plugin_v1.MidonetPluginV2')
+            plugins['midonet']['server_packages'].remove(
+                'python-neutron-plugin-midonet')
+            plugins['midonet']['server_packages'].append(
+                'python-networking-midonet')
     return plugins
 
 

charmhelpers/payload/archive.py

@@ -0,0 +1,73 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import tarfile
+import zipfile
+from charmhelpers.core import (
+    host,
+    hookenv,
+)
+
+
+class ArchiveError(Exception):
+    pass
+
+
+def get_archive_handler(archive_name):
+    if os.path.isfile(archive_name):
+        if tarfile.is_tarfile(archive_name):
+            return extract_tarfile
+        elif zipfile.is_zipfile(archive_name):
+            return extract_zipfile
+    else:
+        # look at the file name
+        for ext in ('.tar', '.tar.gz', '.tgz', 'tar.bz2', '.tbz2', '.tbz'):
+            if archive_name.endswith(ext):
+                return extract_tarfile
+        for ext in ('.zip', '.jar'):
+            if archive_name.endswith(ext):
+                return extract_zipfile
+
+
+def archive_dest_default(archive_name):
+    archive_file = os.path.basename(archive_name)
+    return os.path.join(hookenv.charm_dir(), "archives", archive_file)
+
+
+def extract(archive_name, destpath=None):
+    handler = get_archive_handler(archive_name)
+    if handler:
+        if not destpath:
+            destpath = archive_dest_default(archive_name)
+        if not os.path.isdir(destpath):
+            host.mkdir(destpath)
+        handler(archive_name, destpath)
+        return destpath
+    else:
+        raise ArchiveError("No handler for archive")
+
+
+def extract_tarfile(archive_name, destpath):
+    "Unpack a tar archive, optionally compressed"
+    archive = tarfile.open(archive_name)
+    archive.extractall(destpath)
+
+
+def extract_zipfile(archive_name, destpath):
+    "Unpack a zip file"
+    archive = zipfile.ZipFile(archive_name)
+    archive.extractall(destpath)

config.yaml

@@ -1,11 +1,11 @@
 options:
   debug:
-    default: "false"
-    type: string
+    type: boolean
+    default: False
     description: Enable verbose logging.
   verbose:
-    default: "false"
-    type: string
+    type: boolean
+    default: False
     description: Enable debug logging.
   use-syslog:
     type: boolean

hooks/keystone_context.py

@@ -189,7 +189,7 @@ class KeystoneContext(context.OSContextGenerator):
     def __call__(self):
         from keystone_utils import (
             api_port, set_admin_token, endpoint_url, resolve_address,
-            PUBLIC, ADMIN, PKI_CERTS_DIR, SSH_USER, ensure_permissions,
+            PUBLIC, ADMIN, PKI_CERTS_DIR, ensure_pki_cert_paths,
         )
         ctxt = {}
         ctxt['token'] = set_admin_token(config('admin-token'))
@@ -198,10 +198,8 @@ def __call__(self):
         ctxt['public_port'] = determine_api_port(api_port('keystone-public'),
                                                  singlenode_mode=True)
 
-        debug = config('debug')
-        ctxt['debug'] = debug and bool_from_string(debug)
-        verbose = config('verbose')
-        ctxt['verbose'] = verbose and bool_from_string(verbose)
+        ctxt['debug'] = config('debug')
+        ctxt['verbose'] = config('verbose')
         ctxt['token_expiration'] = config('token-expiration')
 
         ctxt['identity_backend'] = config('identity-backend')
@@ -219,32 +217,16 @@ def __call__(self):
 
         enable_pki = config('enable-pki')
         if enable_pki and bool_from_string(enable_pki):
-            ctxt['signing'] = True
+            log("Enabling PKI", level=DEBUG)
             ctxt['token_provider'] = 'pki'
 
-        if 'token_provider' in ctxt:
-            log("Configuring PKI token cert paths", level=DEBUG)
-            certs = os.path.join(PKI_CERTS_DIR, 'certs')
-            privates = os.path.join(PKI_CERTS_DIR, 'privates')
-            for path in [PKI_CERTS_DIR, certs, privates]:
-                perms = 0o755
-                if not os.path.isdir(path):
-                    mkdir(path=path, owner=SSH_USER, group='keystone',
-                          perms=perms)
-                else:
-                    # Ensure accessible by ssh user and group (for sync).
-                    ensure_permissions(path, user=SSH_USER,
-                                       group='keystone', perms=perms)
-
-            signing_paths = {'certfile': os.path.join(certs,
-                                                      'signing_cert.pem'),
-                             'keyfile': os.path.join(privates,
-                                                     'signing_key.pem'),
-                             'ca_certs': os.path.join(certs, 'ca.pem'),
-                             'ca_key': os.path.join(certs, 'ca_key.pem')}
-
-            for key, val in signing_paths.iteritems():
-                ctxt[key] = val
+        ensure_pki_cert_paths()
+        certs = os.path.join(PKI_CERTS_DIR, 'certs')
+        privates = os.path.join(PKI_CERTS_DIR, 'privates')
+        ctxt.update({'certfile': os.path.join(certs, 'signing_cert.pem'),
+                     'keyfile': os.path.join(privates, 'signing_key.pem'),
+                     'ca_certs': os.path.join(certs, 'ca.pem'),
+                     'ca_key': os.path.join(certs, 'ca_key.pem')})
 
         # Base endpoint URL's which are used in keystone responses
         # to unauthenticated requests to redirect clients to the
@@ -255,6 +237,7 @@ def __call__(self):
         ctxt['admin_endpoint'] = endpoint_url(
             resolve_address(ADMIN),
             api_port('keystone-admin')).rstrip('v2.0')
+
         return ctxt
 
 
@@ -263,7 +246,7 @@ class KeystoneLoggingContext(context.OSContextGenerator):
     def __call__(self):
         ctxt = {}
         debug = config('debug')
-        if debug and bool_from_string(debug):
+        if debug:
             ctxt['root_level'] = 'DEBUG'
 
         return ctxt

hooks/keystone_hooks.py

@@ -74,13 +74,15 @@
     clear_ssl_synced_units,
     is_db_initialised,
     update_certs_if_available,
-    is_pki_enabled,
     ensure_ssl_dir,
     ensure_pki_dir_permissions,
     ensure_permissions,
     force_ssl_sync,
     filter_null,
     ensure_ssl_dirs,
+    ensure_pki_cert_paths,
+    is_service_present,
+    delete_service_entry,
     assess_status,
 )
 
@@ -175,8 +177,7 @@ def config_changed_postupgrade():
     update_nrpe_config()
     CONFIGS.write_all()
 
-    if is_pki_enabled():
-        initialise_pki()
+    initialise_pki()
 
     update_all_identity_relation_units()
 
@@ -192,11 +193,14 @@ def config_changed_postupgrade():
 
 @synchronize_ca_if_changed(fatal=True)
 def initialise_pki():
-    """Create certs and keys required for PKI token signing.
+    """Create certs and keys required for token signing.
+
+    Used for PKI and signing token revocation list.
 
     NOTE: keystone.conf [signing] section must be up-to-date prior to
           executing this.
     """
+    ensure_pki_cert_paths()
     if not peer_units() or is_ssl_cert_master():
         log("Ensuring PKI token certs created", level=DEBUG)
         cmd = ['keystone-manage', 'pki_setup', '--keystone-user', 'keystone',
@@ -337,6 +341,8 @@ def identity_changed(relation_id=None, remote_unit=None):
             return
 
         add_service_to_keystone(relation_id, remote_unit)
+        if is_service_present('neutron', 'network'):
+            delete_service_entry('quantum', 'network')
         settings = relation_get(rid=relation_id, unit=remote_unit)
         service = settings.get('service', None)
         if service:
@@ -377,44 +383,36 @@ def send_ssl_sync_request():
     Note the we do nothing if the setting is already applied.
     """
     unit = local_unit().replace('/', '-')
-    count = 0
+    # Start with core config (e.g. used for signing revoked token list)
+    ssl_config = 0b1
 
     use_https = config('use-https')
     if use_https and bool_from_string(use_https):
-        count += 1
+        ssl_config ^= 0b10
 
     https_service_endpoints = config('https-service-endpoints')
     if (https_service_endpoints and
             bool_from_string(https_service_endpoints)):
-        count += 2
+        ssl_config ^= 0b100
 
     enable_pki = config('enable-pki')
     if enable_pki and bool_from_string(enable_pki):
-        count += 3
+        ssl_config ^= 0b1000
 
     key = 'ssl-sync-required-%s' % (unit)
-    settings = {key: count}
-
-    # If all ssl is disabled ensure this is set to 0 so that cluster hook runs
-    # and endpoints are updated.
-    if not count:
-        log("Setting %s=%s" % (key, count), level=DEBUG)
-        for rid in relation_ids('cluster'):
-            relation_set(relation_id=rid, relation_settings=settings)
-
-        return
+    settings = {key: ssl_config}
 
-    prev = 0
+    prev = 0b0
     rid = None
     for rid in relation_ids('cluster'):
         for unit in related_units(rid):
-            _prev = relation_get(rid=rid, unit=unit, attribute=key) or 0
+            _prev = relation_get(rid=rid, unit=unit, attribute=key) or 0b0
             if _prev and _prev > prev:
-                prev = _prev
+                prev = bin(_prev)
 
-    if rid and prev < count:
+    if rid and prev ^ ssl_config:
         clear_ssl_synced_units()
-        log("Setting %s=%s" % (key, count), level=DEBUG)
+        log("Setting %s=%s" % (key, bin(ssl_config)), level=DEBUG)
         relation_set(relation_id=rid, relation_settings=settings)
 
 
@@ -459,8 +457,7 @@ def cluster_changed():
 
     check_peer_actions()
 
-    if is_pki_enabled():
-        initialise_pki()
+    initialise_pki()
 
     # Figure out if we need to mandate a sync
     units = get_ssl_sync_request_units()