From 12e12aef412d5a06696a58292088dfe1d01b2e34 Mon Sep 17 00:00:00 2001 From: Andrew Beekhof Date: Mon, 27 Mar 2017 18:56:43 +1100 Subject: [PATCH 1/3] PE: Containers: Allow custom control ports when using the machine's local address --- lib/pengine/container.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pengine/container.c b/lib/pengine/container.c index 6f3a3b9..710f5dc 100644 --- a/lib/pengine/container.c +++ b/lib/pengine/container.c @@ -119,9 +119,9 @@ valid_network(container_variant_data_t *data) if(data->ip_range_start) { return TRUE; } - if(data->control_port && crm_str_eq(data->docker_network, "host", TRUE)) { + if(data->control_port) { if(data->replicas_per_host > 1) { - pe_err("Specifying the 'control-port' with 'internal-network=host' for %s requires 'replicas-per-host=1'", data->prefix); + pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix); data->replicas_per_host = 1; } return TRUE; -- 1.8.3.1 From 2a6abd68c200d9c92e4003e26ff282619a4af346 Mon Sep 17 00:00:00 2001 From: Andrew Beekhof Date: Tue, 28 Mar 2017 12:24:46 +1100 Subject: [PATCH 2/3] PE: Containers: Allow the internal and external ports to optionally differ --- lib/pengine/container.c | 39 +++++++++++++++++++++++++++++---------- lib/pengine/variant.h | 7 +++++++ xml/resources-2.8.rng | 7 ++++++- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/lib/pengine/container.c b/lib/pengine/container.c index 710f5dc..ed63b80 100644 --- a/lib/pengine/container.c +++ b/lib/pengine/container.c @@ -232,13 +232,13 @@ create_docker_resource( } for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) { - char *port = pIter->data; + container_port_t *port = pIter->data; if(tuple->ipaddr) { offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s", - tuple->ipaddr, port, port); + tuple->ipaddr, port->source, port->target); } else { - offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port, port); + offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target); } } @@ -410,6 +410,13 @@ static void mount_free(container_mount_t *mount) free(mount); } +static void port_free(container_port_t *port) +{ + free(port->source); + free(port->target); + free(port); +} + gboolean container_unpack(resource_t * rsc, pe_working_set_t * data_set) { @@ -465,16 +472,24 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) for (xmlNode *xml_child = __xml_first_child_element(xml_obj); xml_child != NULL; xml_child = __xml_next_element(xml_child)) { - char *port = crm_element_value_copy(xml_child, "port"); + container_port_t *port = calloc(1, sizeof(container_port_t)); + port->source = crm_element_value_copy(xml_child, "port"); - if(port == NULL) { - port = crm_element_value_copy(xml_child, "range"); + if(port->source == NULL) { + port->source = crm_element_value_copy(xml_child, "range"); + } else { + port->target = crm_element_value_copy(xml_child, "internal-port"); } - if(port != NULL) { + if(port->source != NULL && strlen(port->source) > 0) { + if(port->target == NULL) { + port->target = strdup(port->source); + } container_data->ports = g_list_append(container_data->ports, port); + } else { pe_err("Invalid port directive %s", ID(xml_child)); + port_free(port); } } } @@ -557,6 +572,7 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) GListPtr childIter = NULL; resource_t *new_rsc = NULL; container_mount_t *mount = NULL; + container_port_t *port = calloc(1, sizeof(container_port_t)); int offset = 0, max = 1024; char *buffer = calloc(1, max+1); @@ -576,11 +592,14 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) container_data->mounts = g_list_append(container_data->mounts, mount); if(container_data->control_port) { - container_data->ports = g_list_append(container_data->ports, strdup(container_data->control_port)); + port->source = strdup(container_data->control_port); } else { - container_data->ports = g_list_append(container_data->ports, crm_itoa(DEFAULT_REMOTE_PORT)); + port->source = crm_itoa(DEFAULT_REMOTE_PORT); } + port->target = strdup(port->source); + container_data->ports = g_list_append(container_data->ports, port); + if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) { pe_err("Failed unpacking resource %s", crm_element_value(rsc->xml, XML_ATTR_ID)); if (new_rsc != NULL && new_rsc->fns != NULL) { @@ -845,7 +864,7 @@ container_free(resource_t * rsc) g_list_free_full(container_data->tuples, (GDestroyNotify)tuple_free); g_list_free_full(container_data->mounts, (GDestroyNotify)mount_free); - g_list_free_full(container_data->ports, free); + g_list_free_full(container_data->ports, (GDestroyNotify)port_free); common_free(rsc); } diff --git a/lib/pengine/variant.h b/lib/pengine/variant.h index 4a9e2fe..aa2a1b7 100644 --- a/lib/pengine/variant.h +++ b/lib/pengine/variant.h @@ -78,6 +78,13 @@ typedef struct } container_mount_t; +typedef struct +{ + char *source; + char *target; + +} container_port_t; + typedef struct container_variant_data_s { int masters; int replicas; diff --git a/xml/resources-2.8.rng b/xml/resources-2.8.rng index d10c666..870e804 100644 --- a/xml/resources-2.8.rng +++ b/xml/resources-2.8.rng @@ -115,7 +115,12 @@ - + + + + + + ([0-9\-]+) -- 1.8.3.1 From ee9c3cea4efc4ff633ef4e6eb1b4a3d49c1695a5 Mon Sep 17 00:00:00 2001 From: Andrew Beekhof Date: Tue, 28 Mar 2017 13:09:08 +1100 Subject: [PATCH 3/3] PE: Containers: Ensure replicas_per_host=1 is respected during rediscovery and startup --- pengine/container.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pengine/container.c b/pengine/container.c index b66b7b0..1623861 100644 --- a/pengine/container.c +++ b/pengine/container.c @@ -300,7 +300,32 @@ container_create_probe(resource_t * rsc, node_t * node, action_t * complete, any_created |= tuple->child->cmds->create_probe(tuple->child, node, complete, force, data_set); } if(tuple->docker) { - any_created |= tuple->docker->cmds->create_probe(tuple->docker, node, complete, force, data_set); + bool created = tuple->docker->cmds->create_probe(tuple->docker, node, complete, force, data_set); + + if(created) { + any_created = TRUE; + /* If we're limited to one replica per host (due to + * the lack of an IP range probably), then we don't + * want any of our peer containers starting until + * we've established that no other copies are already + * running. + * + * Partly this is to ensure that replicas_per_host is + * observed, but also to ensure that the containers + * don't fail to start because the necessary port + * mappings (which wont include an IP for uniqueness) + * are already taken + */ + + for (GListPtr tIter = container_data->tuples; tIter != NULL && container_data->replicas_per_host == 1; tIter = tIter->next) { + container_grouping_t *other = (container_grouping_t *)tIter->data; + if(other != tuple) { + custom_action_order(tuple->docker, generate_op_key(tuple->docker->id, RSC_STATUS, 0), NULL, + other->docker, generate_op_key(other->docker->id, RSC_START, 0), NULL, + pe_order_optional, data_set); + } + } + } } if(FALSE && tuple->remote) { // TODO: Needed? -- 1.8.3.1