commit e8579f6387d9841ce619d836110050fb18117753

Author: John Dennis <jdennis@redhat.com>

Date:   Wed Jun 14 13:56:18 2017 -0400

    Add diagnostic logging

    Field experience with Mellon has demonstrated there are many

    opportunities for deployment problems. Although there are tools such

    as browser plugins which can capture SAML messages it's onerous for

    site personnel to install and capture the relevant information. The

    problem with this approach is further compounded by the fact the

    external SAML messages are not correlated to Mellon's

    requests/responses. Mellon currently can dump the Lasso session and

    SAML Response messages and place them in Apache environment variables,

    however these do not appear in the log file. To get them into the log

    you have to add custom logging to the Apache config. Another issue is

    the dumps are not human readable, they are base64 encoded, anyone

    looking at the logs after setting up the custom logging will have to

    find the base64 text and then manually copy the text into an external

    base64 decoder. At that point you'll discover the XML is not pretty

    printed making human interpretation difficult.

    The Mellon debug messages written to the Apache error are often

    insufficient to diagnose problems. And since the Mellon log messages

    are written to the Apache error log they are interspersed with a lot

    of non-Mellon message.

    Compounding the problem of writing Mellon debug messages to the Apache

    error log is the fact Apache log messages have a fixed maximum length

    (currently 8192) which is insufficient to completely write out such

    things as SAML Assertions, metadata, etc. Apache logging also escapes

    all control characters with the consequence line breaks are not

    preserved and what was a nicely formatted human readable piece of text

    becomes a single line with escape characters and may be truncated.

    It would be really nice if we could capture diagnostic information

    with these properties:

    * All relevant data is collected in exactly one file.

    * Only information relevant to Mellon appears in the file.

    * All information is human readable (pretty printed, decrypted) with

      no need to rely on other tools.

    * The diagnostic information is grouped by requests.

    * The requests can be cross correlated with other Apache logs because

      they utilize the same unique request identifier.

    This patch adds diagnostic logging to a independent Mellon diagnostics

    log file. Every piece of relevant information is captured, including:

    * Request information which includes:

      - Request method

      - Request URL (raw and processed)

      - Scheme

      - Port

      - Request query parameters

      - Server name

      - Unique request ID

      - process ID, thread ID

      - Request headers

    * Mellon per directory configuration

      A complete dump of the entire am_dir_cfg_rec structure keyed using

      both the Mellon directive it is associated with and it's internal

      name. This is emitted once on first use for a given URL.

      The per directory dump includes the pathname of each file read as

      well as the file contents. This includes:

      - IdP metadata

      - SP metadata

      - SP cert

      - SP key

      - IdP public key file

      - IdP CA file

    * All session management operations

      - cookie

      - session lookup

      - session creation

      - session deletion

      - cache management

      - cache entry information

    * All SAML messages

      Each SAML message is decrypted, decoded and XML pretty printed in

      human readable form.

    * Request pipeline operations

      What operations Mellon performs, what decisions it makes and what

      data is being used to make those decisions.

    * Response

      - response status

      - response headers

      - Apache user name

      - auth_type

      - all Apache environment variables

    Diagnostics can be enabled/disabled both at compile time and run

    time. Compile time inclusion of diagnostics is managed with the

    ENABLE_DIAGNOSTICS preprocssor symbol. The configure script now accepts

    the

      --enable-diagnostics and --disable-diagnostics

    option. Building with diagnostics is disabled by default, you must

    specify --enable-diagnostics to enable the run time option of generating

    diagnostics.

    The following server config directives have been added (e.g. may be

    specified in the main server config area or within a <VirtualHost>

    directive). If Mellon was not built with diagnostics enabled then

    these config directives are no-ops and their use will generated a

    warning in the log file indicating they have been ignored and to be

    effective you must builld Mellon with diagnostics enabled.

      MellonDiagnosticsFile:

        The name of the diagnostics file or pipe,

        (default is logs/mellon_diagnostics)

      MellonDiagnosticsEnable:

        Currently either On or Off but it is designed so it can take other

        flags in the future to control what type of information is

        reported.

    Signed-off-by: John Dennis <jdennis@redhat.com>

diff --git a/Makefile.in b/Makefile.in

index 731919e..3656bec 100644

--- a/Makefile.in

+++ b/Makefile.in

@@ -1,8 +1,11 @@

 # Source files. mod_auth_mellon.c must be the first file.

 SRC=mod_auth_mellon.c \

-	auth_mellon_cache.c auth_mellon_config.c \

-	auth_mellon_cookie.c auth_mellon_handler.c \

+	auth_mellon_cache.c \

+	auth_mellon_config.c \

+	auth_mellon_cookie.c \

+	auth_mellon_diagnostics.c \

+	auth_mellon_handler.c \

 	auth_mellon_util.c \

 	auth_mellon_session.c \

 	auth_mellon_httpclient.c

@@ -25,7 +28,7 @@ DISTFILES=$(SRC) \

 all:	mod_auth_mellon.la

 mod_auth_mellon.la: $(SRC) auth_mellon.h auth_mellon_compat.h

-	@APXS2@ -Wc,"-std=c99 @OPENSSL_CFLAGS@ @LASSO_CFLAGS@ @CURL_CFLAGS@ @GLIB_CFLAGS@ @CFLAGS@" -Wl,"@OPENSSL_LIBS@ @LASSO_LIBS@ @CURL_LIBS@ @GLIB_LIBS@" -Wc,-Wall -Wc,-g -c $(SRC)

+	@APXS2@ -Wc,"-std=c99 @MELLON_CFLAGS@ @OPENSSL_CFLAGS@ @LASSO_CFLAGS@ @CURL_CFLAGS@ @GLIB_CFLAGS@ @CFLAGS@" -Wl,"@OPENSSL_LIBS@ @LASSO_LIBS@ @CURL_LIBS@ @GLIB_LIBS@" -Wc,-Wall -Wc,-g -c $(SRC)

 # Building configure (for distribution)

@@ -46,6 +49,7 @@ distfile:	@NAMEVER@.tar.gz

 .PHONY:	clean

 clean:

 	rm -f mod_auth_mellon.la

+	rm -f $(SRC:%.c=%.o)

 	rm -f $(SRC:%.c=%.lo)

 	rm -f $(SRC:%.c=%.slo)

 	rm -rf .libs/

diff --git a/README b/README

index e084ef4..8211d66 100644

--- a/README

+++ b/README

@@ -132,6 +132,24 @@ MellonPostSize 1048576

 # Default: MellonPostCount 100

 MellonPostCount 100

+# MellonDiagnosticsFile If Mellon was built with diagnostic capability

+# then diagnostic is written here, it may be either a filename or a pipe.

+# If it's a filename then the resulting path is  relative to the ServerRoot.

+# If the value is preceeded by the pipe character "|" it should be followed

+# by a path to a program to receive the log information on its standard input.

+# This is a server context directive, hence it may be specified in the

+# main server config area or within a <VirtualHost> directive.

+# Default: logs/mellon_diagnostics

+MellonDiagnosticsFile logs/mellon_diagnostics

+

+# MellonDiagnosticsEnable If Mellon was built with diagnostic capability

+# then this is a list of words controlling diagnostic output.

+# Currently only On and Off are supported.

+# This is a server context directive, hence it may be specified in the

+# main server config area or within a <VirtualHost> directive.

+# Default: Off

+MellonDiagnosticsEnable Off

+

 ###########################################################################

 # End of global configuration for mod_auth_mellon.

 ###########################################################################

diff --git a/auth_mellon.h b/auth_mellon.h

index a6fa34c..6ce6a8e 100644

--- a/auth_mellon.h

+++ b/auth_mellon.h

@@ -85,6 +85,14 @@

 #define AM_ERROR_MISSING_PAOS_MEDIA_TYPE 3

+#ifdef ENABLE_DIAGNOSTICS

+typedef enum {

+    AM_DIAG_FLAG_ENABLED       = (1 << 0),

+    AM_DIAG_FLAG_DISABLE       = 0,

+    AM_DIAG_FLAG_ENABLE_ALL    = ~0,

+} am_diag_flags_t;

+#endif

+

 /* This is the length of the id we use (for session IDs and

  * replaying POST data).

  */

@@ -100,6 +108,9 @@

 #define am_get_req_cfg(r) (am_req_cfg_rec *)ap_get_module_config((r)->request_config, &auth_mellon_module)

+#ifdef ENABLE_DIAGNOSTICS

+#define am_get_diag_cfg(s) (&(am_get_srv_cfg((s)))->diag_cfg)

+#endif

 typedef struct am_mod_cfg_rec {

     int cache_size;

@@ -124,8 +135,20 @@ typedef struct am_mod_cfg_rec {

 } am_mod_cfg_rec;

+#ifdef ENABLE_DIAGNOSTICS

+typedef struct am_diag_cfg_rec {

+    const char *filename;

+    apr_file_t *fd;

+    am_diag_flags_t flags;

+    apr_table_t *dir_cfg_emitted;

+} am_diag_cfg_rec;

+#endif

+

 typedef struct am_srv_cfg_rec {

     am_mod_cfg_rec *mc;

+#ifdef ENABLE_DIAGNOSTICS

+    am_diag_cfg_rec diag_cfg;

+#endif

 } am_srv_cfg_rec;

 typedef enum {

@@ -284,7 +307,6 @@ typedef struct am_dir_cfg_rec {

     /* List of domains we can redirect to. */

     const char * const *redirect_domains;

-

 } am_dir_cfg_rec;

 /* Bitmask for PAOS service options */

@@ -301,6 +323,9 @@ typedef struct am_req_cfg_rec {

     bool ecp_authn_req;

     ECPServiceOptions ecp_service_options;

 #endif /* HAVE_ECP */

+#ifdef ENABLE_DIAGNOSTICS

+    bool diag_emitted;

+#endif

 } am_req_cfg_rec;

 typedef struct am_cache_storage_t {

@@ -393,6 +418,7 @@ static const int inherit_ecp_send_idplist = -1;

 void *auth_mellon_dir_config(apr_pool_t *p, char *d);

 void *auth_mellon_dir_merge(apr_pool_t *p, void *base, void *add);

 void *auth_mellon_server_config(apr_pool_t *p, server_rec *s);

+void *auth_mellon_srv_merge(apr_pool_t *p, void *base, void *add);

 const char *am_cookie_get(request_rec *r);

@@ -503,4 +529,72 @@ int am_httpclient_post_str(request_rec *r, const char *uri,

 extern module AP_MODULE_DECLARE_DATA auth_mellon_module;

+#ifdef ENABLE_DIAGNOSTICS

+

+/* Initializing an apr_time_t to 0x7fffffffffffffffLL yields an

+ * iso 8601 time with 1 second precision of "294247-01-10T04:00:54Z"

+ * this is 22 characters, +1 for null terminator. */

+#define ISO_8601_BUF_SIZE 23

+

+typedef struct {

+    bool req_headers_written;

+} am_diag_request_data;

+

+const char *

+am_diag_cache_key_type_str(am_cache_key_t key_type);

+

+const char *

+am_diag_cond_str(request_rec *r, const am_cond_t *cond);

+

+int

+am_diag_finalize_request(request_rec *r);

+

+const char *

+am_diag_lasso_http_method_str(LassoHttpMethod http_method);

+

+void

+am_diag_log_cache_entry(request_rec *r, int level, am_cache_entry_t *entry,

+                        const char *fmt, ...)

+    __attribute__((format(printf,4,5)));

+

+void

+am_diag_log_file_data(request_rec *r, int level, am_file_data_t *file_data,

+                      const char *fmt, ...)

+    __attribute__((format(printf,4,5)));

+

+int

+am_diag_log_init(apr_pool_t *pc, apr_pool_t *p, apr_pool_t *pt, server_rec *s);

+

+void

+am_diag_log_lasso_node(request_rec *r, int level, LassoNode *node,

+                       const char *fmt, ...)

+    __attribute__((format(printf,4,5)));

+

+void

+am_diag_log_profile(request_rec *r, int level, LassoProfile *profile,

+                    const char *fmt, ...)

+    __attribute__((format(printf,4,5)));

+

+void

+am_diag_printf(request_rec *r, const char *fmt, ...)

+    __attribute__((format(printf,2,3)));

+

+void

+am_diag_rerror(const char *file, int line, int module_index,

+               int level, apr_status_t status,

+               request_rec *r, const char *fmt, ...);

+

+char *

+am_diag_time_t_to_8601(request_rec *r, apr_time_t t);

+

+#else  /* ENABLE_DIAGNOSTICS */

+

+#define am_diag_log_cache_entry(...) do {} while(0)

+#define am_diag_log_file_data(...) do {} while(0)

+#define am_diag_log_lasso_node(...) do {} while(0)

+#define am_diag_log_profile(...) do {} while(0)

+#define am_diag_printf(...) do {} while(0)

+

+#endif /* ENABLE_DIAGNOSTICS */

+

 #endif /* MOD_AUTH_MELLON_H */

diff --git a/auth_mellon_cache.c b/auth_mellon_cache.c

index af5c267..2115acc 100644

--- a/auth_mellon_cache.c

+++ b/auth_mellon_cache.c

@@ -144,11 +144,17 @@ am_cache_entry_t *am_cache_lock(request_rec *r,

             continue;

         if(strcmp(tablekey, key) == 0) {

+            apr_time_t now = apr_time_now();

             /* We found the entry. */

-            if(e->expires > apr_time_now()) {

+            if(e->expires > now) {

                 /* And it hasn't expired. */

                 return e;

             }

+            else {

+                am_diag_log_cache_entry(r, 0, e,

+                                        "found expired session, now %s\n",

+                                        am_diag_time_t_to_8601(r, now));

+            }

         }

     }

@@ -342,6 +348,10 @@ am_cache_entry_t *am_cache_new(request_rec *r,

              * Update 't' and exit loop.

              */

             t = e;

+            am_diag_log_cache_entry(r, 0, e,

+                                    "%s ejecting expired sessions, now %s\n",

+                                    __func__,

+                                    am_diag_time_t_to_8601(r, current_time));

             break;

         }

@@ -400,6 +410,11 @@ am_cache_entry_t *am_cache_new(request_rec *r,

         return NULL;

     }

+    am_diag_printf(r, "%s created new session, id=%s at %s"

+                   " cookie_token=\"%s\"\n",

+                   __func__, t->key, am_diag_time_t_to_8601(r, current_time),

+                   cookie_token);

+

     return t;

 }

diff --git a/auth_mellon_config.c b/auth_mellon_config.c

index c3cb5e0..66966fc 100644

--- a/auth_mellon_config.c

+++ b/auth_mellon_config.c

@@ -78,6 +78,15 @@ static const apr_size_t post_size = 1024 * 1024;

  */

 static const int post_count = 100;

+#ifdef ENABLE_DIAGNOSTICS

+/* Default filename for mellon diagnostics log file.

+ * Relative pathname is relative to server root. */

+static const char *default_diag_filename = "logs/mellon_diagnostics";

+

+/* Default state for diagnostics is off */

+static am_diag_flags_t default_diag_flags = AM_DIAG_FLAG_DISABLE;

+#endif

+

 /* whether to merge env. vars or not

  * the MellonMergeEnvVars configuration directive if you change this.

  */

@@ -470,6 +479,78 @@ static const char *am_set_module_config_int_slot(cmd_parms *cmd,

     return ap_set_int_slot(cmd, am_get_mod_cfg(cmd->server), arg);

 }

+/* This function handles the MellonDiagnosticsFile configuration directive.

+ * It emits as warning in the log file if Mellon is not built with

+ * diagnostics enabled.

+ *

+ * Parameters:

+ *  cmd_parms *cmd       The command structure for this configuration

+ *                       directive.

+ *  void *struct_ptr     Pointer to the current directory configuration.

+ *                       NULL if we are not in a directory configuration.

+ *  const char *arg      The string argument following this configuration

+ *                       directive in the configuraion file.

+ *

+ * Returns:

+ *  NULL on success or an error string on failure.

+ */

+static const char *am_set_module_diag_file_slot(cmd_parms *cmd,

+                                                    void *struct_ptr,

+                                                    const char *arg)

+{

+#ifdef ENABLE_DIAGNOSTICS

+    return ap_set_file_slot(cmd, am_get_diag_cfg(cmd->server), arg);

+#else

+    ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, cmd->server,

+                 "%s has no effect because Mellon was not compiled with"

+                 " diagnostics enabled, use ./configure --enable-diagnostics"

+                 " at build time to turn this feature on.",

+                 cmd->directive->directive);

+    return NULL;

+#endif

+}

+

+/* This function handles configuration directives which sets the

+ * diagnostics flags in the module configuration.

+ *

+ * Parameters:

+ *  cmd_parms *cmd       The command structure for this configuration

+ *                       directive.

+ *  void *struct_ptr     Pointer to the current directory configuration.

+ *                       NULL if we are not in a directory configuration.

+ *  const char *arg      The string argument following this configuration

+ *                       directive in the configuraion file.

+ *

+ * Returns:

+ *  NULL on success or an error string on failure.

+ */

+static const char *am_set_module_diag_flags_slot(cmd_parms *cmd,

+                                                 void *struct_ptr,

+                                                 const char *arg)

+{

+#ifdef ENABLE_DIAGNOSTICS

+    am_diag_cfg_rec *diag_cfg = am_get_diag_cfg(cmd->server);

+

+    if (strcasecmp(arg, "on") == 0) {

+        diag_cfg->flags = AM_DIAG_FLAG_ENABLE_ALL;

+    }

+    else if (strcasecmp(arg, "off") == 0) {

+        diag_cfg->flags = AM_DIAG_FLAG_DISABLE;

+    } else {

+        return apr_psprintf(cmd->pool, "%s: must be one of: 'on', 'off'",

+                            cmd->cmd->name);

+    }

+    return NULL;

+#else

+    ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, cmd->server,

+                 "%s has no effect because Mellon was not compiled with"

+                 " diagnostics enabled, use ./configure --enable-diagnostics"

+                 " at build time to turn this feature on.",

+                 cmd->directive->directive);

+    return NULL;

+#endif

+}

+

 /* This function handles the MellonCookieSameSite configuration directive.

  * This directive can be set to "lax" or "strict"

  *

@@ -684,7 +765,7 @@ static const char *am_set_setenv_no_prefix_slot(cmd_parms *cmd,

 static int am_cond_flags(const char *arg)

 {

     int flags = AM_COND_FLAG_NULL; 

-    static const char const *options[] = { 

+    static const char * const options[] = {

         "OR",  /* AM_EXPIRE_FLAG_OR */

         "NOT", /* AM_EXPIRE_FLAG_NOT */

         "REG", /* AM_EXPIRE_FLAG_REG */

@@ -1123,6 +1204,30 @@ const command_rec auth_mellon_commands[] = {

         "The maximum size of a saved POST, in bytes."

         " Default value is 1048576 (1 MB)."

         ), 

+    AP_INIT_TAKE1(

+        "MellonDiagnosticsFile",

+        am_set_module_diag_file_slot,

+#ifdef ENABLE_DIAGNOSTICS

+        (void *)APR_OFFSETOF(am_diag_cfg_rec, filename),

+#else

+        NULL,

+#endif

+        RSRC_CONF,

+        "Diagnostics log file. [file|pipe] "

+        "If file then file is a filename, relative to the ServerRoot."

+        "If pipe then the filename is a pipe character \"|\", "

+        "followed by the path to a program to receive the log information "

+        "on its standard input. "

+        " Default value is \"logs/mellon_diagnostics\"."

+        ),

+    AP_INIT_ITERATE(

+        "MellonDiagnosticsEnable",

+        am_set_module_diag_flags_slot,

+        NULL,

+        RSRC_CONF,

+        "Diagnostics flags. [on|off] "

+        " Default value is \"off\"."

+        ),

     /* Per-location configuration directives. */

@@ -1855,6 +1960,13 @@ void *auth_mellon_server_config(apr_pool_t *p, server_rec *s)

     srv = apr_palloc(p, sizeof(*srv));

+#ifdef ENABLE_DIAGNOSTICS

+    srv->diag_cfg.filename = default_diag_filename;

+    srv->diag_cfg.fd = NULL;

+    srv->diag_cfg.flags = default_diag_flags;

+    srv->diag_cfg.dir_cfg_emitted = apr_table_make(p, 0);

+#endif

+

     /* we want to keeep our global configuration of shared memory and

      * mutexes, so we try to find it in the userdata before doing anything

      * else */

@@ -1886,6 +1998,47 @@ void *auth_mellon_server_config(apr_pool_t *p, server_rec *s)

     apr_pool_userdata_set(mod, key, apr_pool_cleanup_null, p);

     srv->mc = mod;

+

     return srv;

 }

+/* This function merges two am_srv_cfg_rec structures.

+ * It will try to inherit from the base where possible.

+ *

+ * Parameters:

+ *  apr_pool_t *p        The pool we should allocate memory from.

+ *  void *base           The original structure.

+ *  void *add            The structure we should add to base.

+ *

+ * Returns:

+ *  The merged structure.

+ */

+void *auth_mellon_srv_merge(apr_pool_t *p, void *base, void *add)

+{

+    am_srv_cfg_rec *base_cfg = (am_srv_cfg_rec *)base;

+    am_srv_cfg_rec *new_cfg;

+

+    new_cfg = (am_srv_cfg_rec *)apr_palloc(p, sizeof(*new_cfg));

+

+    new_cfg->mc = base_cfg->mc;

+

+#ifdef ENABLE_DIAGNOSTICS

+    am_srv_cfg_rec *add_cfg = (am_srv_cfg_rec *)add;

+    new_cfg->diag_cfg.filename = (add_cfg->diag_cfg.filename !=

+                                  default_diag_filename ?

+                                  add_cfg->diag_cfg.filename :

+                                  base_cfg->diag_cfg.filename);

+

+    new_cfg->diag_cfg.fd = NULL;

+

+    new_cfg->diag_cfg.flags = (add_cfg->diag_cfg.flags !=

+                               default_diag_flags ?

+                               add_cfg->diag_cfg.flags :

+                               base_cfg->diag_cfg.flags);

+

+    new_cfg->diag_cfg.dir_cfg_emitted = apr_table_make(p, 0);

+

+#endif

+

+    return new_cfg;

+}

diff --git a/auth_mellon_diagnostics.c b/auth_mellon_diagnostics.c

new file mode 100644

index 0000000..519a44f

--- /dev/null

+++ b/auth_mellon_diagnostics.c

@@ -0,0 +1,1017 @@

+#ifdef ENABLE_DIAGNOSTICS

+

+#include "auth_mellon.h"

+

+#if APR_HAVE_UNISTD_H

+#include <unistd.h>

+#endif

+#if APR_HAVE_PROCESS_H

+#include <process.h>            /* for getpid() on Win32 */

+#endif

+

+/*============================= Internal Static ==============================*/

+

+/*------------------ Defines ------------------*/

+

+#define AM_DIAG_ENABLED(diag_cfg)                                       \

+    (diag_cfg && diag_cfg->fd && (diag_cfg->flags & AM_DIAG_FLAG_ENABLED))

+

+/*------------------ Typedefs ------------------*/

+

+typedef struct iter_callback_data {

+    apr_file_t *diag_fd;

+    int level;

+} iter_callback_data;

+

+/*------------------ Prototypes ------------------*/

+

+static const char *

+indent(int level);

+

+static void

+write_indented_text(apr_file_t *diag_fd, int level, const char* text);

+

+static void

+am_diag_format_line(apr_pool_t *pool, apr_file_t *diag_fd, int level,

+                    const char *fmt, va_list ap);

+

+static const char *

+am_diag_cond_flag_str(request_rec *r, am_cond_flag_t flags);

+

+static const char *

+am_diag_enable_str(am_enable_t enable);

+

+static const char *

+am_diag_samesite_str(am_samesite_t samesite);

+

+static const char *

+am_diag_httpd_error_level_str(request_rec *r, int level);

+

+static apr_size_t

+am_diag_time_t_to_8601_buf(char *buf, apr_size_t buf_size, apr_time_t t);

+

+static int

+am_diag_open_log(server_rec *s, apr_pool_t *p);

+

+static int

+am_table_count(void *rec, const char *key, const char *value);

+

+static int

+log_headers(void *rec, const char *key, const char *value);

+

+static int

+log_probe_discovery_idp(void *rec, const char *key, const char *value);

+

+static void

+am_diag_log_dir_cfg(request_rec *r, int level, am_dir_cfg_rec *cfg,

+                    const char *fmt, ...)

+    __attribute__((format(printf,4,5)));

+

+static bool

+am_diag_initialize_req(request_rec *r, am_diag_cfg_rec *diag_cfg,

+                       am_req_cfg_rec *req_cfg);

+

+/*------------------ Functions ------------------*/

+

+static const char *

+indent(int level)

+{

+    static const char * const indents[] = {

+        "",                     /* 0 */

+        "  ",                   /* 1 */

+        "    ",                 /* 2 */

+        "      ",               /* 3 */

+        "        ",             /* 4 */

+        "          ",           /* 5 */

+        "            ",         /* 6 */

+        "              ",       /* 7 */

+        "                ",     /* 8 */

+        "                  ",   /* 9 */

+    };

+    int n_indents = sizeof(indents)/sizeof(indents[0]);

+

+    if (level < 0) {

+        return "";

+    }

+    if (level < n_indents) {

+        return indents[level];

+    }

+    return indents[n_indents-1];

+}

+

+static void

+write_indented_text(apr_file_t *diag_fd, int level, const char* text)

+{

+    const char *start, *end, *prefix;

+    size_t len, prefix_len;

+    bool crlf = false;

+

+    if (!text) return;

+

+    prefix = indent(level);

+    prefix_len = strlen(prefix);

+    start = end = text;

+    while (*end) {

+         /* find end of line */

+        for (; *end && *end != '\n'; end++);