This patchset is for bug:

24394567 Mitaka Keystone should support MySQL Cluster

This fixes the following aspects of Keystone:

1. Implementation of an oslo.db configuration parameter to specify the MySQL

storage engine (mysql_storage_engine).

2. Replacement of hardcoded SQL statements that set the engine to "InnoDB"

to the above configuration value.

3. Logic to handle SQL differences between MySQL InnoDB and MySQL Cluster (NDB).

This includes column lengths, constraints, foreign keys, and indexes.

This has not been committed upstream, but has been filed in launchpad:

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/067_kilo.py.orig 2016-08-04 10:10:34.947867201 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/067_kilo.py 2016-08-04 10:16:37.481333376 -0700

@@ -12,6 +12,7 @@

import migrate

+from oslo_config import cfg

from oslo_log import log

import sqlalchemy as sql

@@ -21,6 +22,7 @@ from keystone.identity.mapping_backends

LOG = log.getLogger(__name__)

+CONF = cfg.CONF

def upgrade(migrate_engine):

@@ -44,7 +46,7 @@ def upgrade(migrate_engine):

sql.Column('blob', ks_sql.JsonBlob, nullable=False),

sql.Column('type', sql.String(length=255), nullable=False),

sql.Column('extra', ks_sql.JsonBlob.impl),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

domain = sql.Table(

@@ -53,7 +55,7 @@ def upgrade(migrate_engine):

sql.Column('name', sql.String(length=64), nullable=False),

sql.Column('enabled', sql.Boolean, default=True, nullable=False),

sql.Column('extra', ks_sql.JsonBlob.impl),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

endpoint = sql.Table(

@@ -67,7 +69,7 @@ def upgrade(migrate_engine):

sql.Column('enabled', sql.Boolean, nullable=False, default=True,

server_default='1'),

sql.Column('region_id', sql.String(length=255), nullable=True),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

group = sql.Table(

@@ -77,7 +79,7 @@ def upgrade(migrate_engine):

sql.Column('name', sql.String(length=64), nullable=False),

sql.Column('description', sql.Text),

sql.Column('extra', ks_sql.JsonBlob.impl),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

policy = sql.Table(

@@ -86,7 +88,7 @@ def upgrade(migrate_engine):

sql.Column('type', sql.String(length=255), nullable=False),

sql.Column('blob', ks_sql.JsonBlob, nullable=False),

sql.Column('extra', ks_sql.JsonBlob.impl),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

project = sql.Table(

@@ -98,7 +100,7 @@ def upgrade(migrate_engine):

sql.Column('enabled', sql.Boolean),

sql.Column('domain_id', sql.String(length=64), nullable=False),

sql.Column('parent_id', sql.String(64), nullable=True),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

role = sql.Table(

@@ -106,7 +108,7 @@ def upgrade(migrate_engine):

sql.Column('id', sql.String(length=64), primary_key=True),

sql.Column('name', sql.String(length=255), nullable=False),

sql.Column('extra', ks_sql.JsonBlob.impl),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

service = sql.Table(

@@ -116,7 +118,7 @@ def upgrade(migrate_engine):

sql.Column('enabled', sql.Boolean, nullable=False, default=True,

server_default='1'),

sql.Column('extra', ks_sql.JsonBlob.impl),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

token = sql.Table(

@@ -127,7 +129,7 @@ def upgrade(migrate_engine):

sql.Column('valid', sql.Boolean, default=True, nullable=False),

sql.Column('trust_id', sql.String(length=64)),

sql.Column('user_id', sql.String(length=64)),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

trust = sql.Table(

@@ -141,7 +143,7 @@ def upgrade(migrate_engine):

sql.Column('expires_at', sql.DateTime),

sql.Column('remaining_uses', sql.Integer, nullable=True),

sql.Column('extra', ks_sql.JsonBlob.impl),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

trust_role = sql.Table(

@@ -150,7 +152,7 @@ def upgrade(migrate_engine):

nullable=False),

sql.Column('role_id', sql.String(length=64), primary_key=True,

nullable=False),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

user = sql.Table(

@@ -162,14 +164,14 @@ def upgrade(migrate_engine):

sql.Column('enabled', sql.Boolean),

sql.Column('domain_id', sql.String(length=64), nullable=False),

sql.Column('default_project_id', sql.String(length=64)),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

user_group_membership = sql.Table(

'user_group_membership', meta,

sql.Column('user_id', sql.String(length=64), primary_key=True),

sql.Column('group_id', sql.String(length=64), primary_key=True),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

region = sql.Table(

@@ -179,7 +181,7 @@ def upgrade(migrate_engine):

sql.Column('description', sql.String(255), nullable=False),

sql.Column('parent_region_id', sql.String(255), nullable=True),

sql.Column('extra', sql.Text()),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

assignment = sql.Table(

@@ -197,7 +199,7 @@ def upgrade(migrate_engine):

sql.Column('role_id', sql.String(64), nullable=False),

sql.Column('inherited', sql.Boolean, default=False, nullable=False),

sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id', 'role_id'),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

mapping = sql.Table(

@@ -211,7 +213,7 @@ def upgrade(migrate_engine):

mapping_backend.EntityType.GROUP,

name='entity_type'),

nullable=False),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

domain_config_whitelist = sql.Table(

@@ -221,7 +223,7 @@ def upgrade(migrate_engine):

sql.Column('group', sql.String(255), primary_key=True),

sql.Column('option', sql.String(255), primary_key=True),

sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

domain_config_sensitive = sql.Table(

@@ -231,7 +233,7 @@ def upgrade(migrate_engine):

sql.Column('group', sql.String(255), primary_key=True),

sql.Column('option', sql.String(255), primary_key=True),

sql.Column('value', ks_sql.JsonBlob.impl, nullable=False),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

# create all tables

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py.orig 2016-08-04 10:10:40.619013471 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/073_insert_assignment_inherited_pk.py 2016-08-04 10:18:06.349087340 -0700

@@ -11,11 +11,13 @@

# under the License.

import migrate

+from oslo_config import cfg

import sqlalchemy as sql

from sqlalchemy.orm import sessionmaker

from keystone.assignment.backends import sql as assignment_sql

+CONF = cfg.CONF

def upgrade(migrate_engine):

"""Inserts inherited column to assignment table PK constraints.

@@ -57,7 +59,7 @@ def upgrade(migrate_engine):

nullable=False),

sql.PrimaryKeyConstraint('type', 'actor_id', 'target_id',

'role_id', 'inherited'),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

# Create the new assignment table

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/075_confirm_config_registration.py.orig 2016-08-04 10:10:47.746776861 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/075_confirm_config_registration.py 2016-08-04 10:19:03.802564813 -0700

@@ -10,9 +10,11 @@

# License for the specific language governing permissions and limitations

# under the License.

+from oslo_config import cfg

import sqlalchemy as sql

REGISTRATION_TABLE = 'config_register'

+CONF = cfg.CONF

def upgrade(migrate_engine):

@@ -24,6 +26,6 @@ def upgrade(migrate_engine):

meta,

sql.Column('type', sql.String(64), primary_key=True),

sql.Column('domain_id', sql.String(64), nullable=False),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

registration_table.create(migrate_engine, checkfirst=True)

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py.orig 2016-08-04 10:10:53.258335033 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py 2016-08-04 10:20:06.575865756 -0700

@@ -12,10 +12,12 @@

# License for the specific language governing permissions and limitations

# under the License.

+from oslo_config import cfg

import sqlalchemy as sql

from keystone.common.sql import migration_helpers

+CONF = cfg.CONF

def upgrade(migrate_engine):

try:

@@ -48,7 +50,7 @@ def upgrade(migrate_engine):

sql.Column('region_id', sql.String(64),

nullable=True),

sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

endpoint_policy_table.create(migrate_engine, checkfirst=True)

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py.orig 2016-08-04 10:10:59.098720956 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py 2016-08-04 10:22:41.994100588 -0700

@@ -45,7 +45,7 @@ def upgrade(migrate_engine):

sql.Column('id', sql.String(64), primary_key=True),

sql.Column('enabled', sql.Boolean, nullable=False),

sql.Column('description', sql.Text(), nullable=True),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

idp_table.create(migrate_engine, checkfirst=True)

@@ -57,7 +57,7 @@ def upgrade(migrate_engine):

sql.ForeignKey('identity_provider.id', ondelete='CASCADE'),

primary_key=True),

sql.Column('mapping_id', sql.String(64), nullable=False),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

federation_protocol_table.create(migrate_engine, checkfirst=True)

@@ -66,7 +66,7 @@ def upgrade(migrate_engine):

meta,

sql.Column('id', sql.String(64), primary_key=True),

sql.Column('rules', sql.Text(), nullable=False),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

mapping_table.create(migrate_engine, checkfirst=True)

@@ -81,7 +81,7 @@ def upgrade(migrate_engine):

sql.Column('sp_url', sql.String(256), nullable=False),

sql.Column(_RELAY_STATE_PREFIX, sql.String(256), nullable=False,

server_default=relay_state_prefix_default),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

sp_table.create(migrate_engine, checkfirst=True)

@@ -92,6 +92,6 @@ def upgrade(migrate_engine):

sql.Column('idp_id', sql.String(64),

sql.ForeignKey('identity_provider.id', ondelete='CASCADE')),

sql.Column('remote_id', sql.String(255), primary_key=True),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

remote_id_table.create(migrate_engine, checkfirst=True)

--- keystone-9.1.0/keystone/common/sql/migrate_repo/versions/087_implied_roles.py.orig 2016-08-04 10:11:04.322293468 -0700

+++ keystone-9.1.0/keystone/common/sql/migrate_repo/versions/087_implied_roles.py 2016-08-04 10:23:34.919749416 -0700

@@ -12,10 +12,12 @@

import migrate

+from oslo_config import cfg

import sqlalchemy as sql

ROLE_TABLE = 'role'

+CONF = cfg.CONF

def upgrade(migrate_engine):

@@ -27,7 +29,7 @@ def upgrade(migrate_engine):

sql.Column('prior_role_id', sql.String(length=64), primary_key=True),

sql.Column(

'implied_role_id', sql.String(length=64), primary_key=True),

- mysql_engine='InnoDB',

+ mysql_engine=CONF.database.mysql_storage_engine,

mysql_charset='utf8')

implied_role.create()

role = sql.Table(ROLE_TABLE, meta, autoload=True)