From adc074704603822bed91e2714d93501e926c05a3 Mon Sep 17 00:00:00 2001 From: Raghavendra Bhat Date: Fri, 21 Sep 2018 11:25:08 -0400 Subject: [PATCH 384/385] features/uss: Use xxh64 to generate gfid instead of md5sum * This is to ensure FIPS support * Also changed the signature of svs_uuid_generate to get xlator argument * Added xxh64 wrapper functions in common-utils to generate gfid using xxh64 - Those wrapper functions can be used by other xlators as well to generate gfids using xxh64. But as of now snapview-server is going to be the only consumer. backport of following 2 patches from upstream: https://review.gluster.org/#/c/glusterfs/+/21122/ https://review.gluster.org/#/c/glusterfs/+/20983/ Change-Id: I35047cd1911d7a6ebe036d699d57fbdcd364ec8f BUG: 1459709 Signed-off-by: Raghavendra Bhat Reviewed-on: https://code.engineering.redhat.com/gerrit/150759 Reviewed-by: Rafi Kavungal Chundattu Parambil Tested-by: RHGS Build Bot Reviewed-by: Sunil Kumar Heggodu Gopala Acharya --- libglusterfs/src/common-utils.c | 172 +++++++++++++++++++++ libglusterfs/src/common-utils.h | 1 + libglusterfs/src/libglusterfs-messages.h | 11 +- .../snapview-server/src/snapview-server-helpers.c | 38 +++-- .../features/snapview-server/src/snapview-server.c | 15 +- .../features/snapview-server/src/snapview-server.h | 5 +- 6 files changed, 227 insertions(+), 15 deletions(-) diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c index f632e78..54ef875 100644 --- a/libglusterfs/src/common-utils.c +++ b/libglusterfs/src/common-utils.c @@ -91,6 +91,178 @@ gf_xxh64_wrapper(const unsigned char *data, size_t len, unsigned long long seed, snprintf(xxh64 + i * 2, lim-i*2, "%02x", p[i]); } +/** + * This function takes following arguments + * @this: xlator + * @gfid: The gfid which has to be filled + * @hash: the 8 byte hash which has to be filled inside the gfid + * @index: the array element of the uuid_t structure (which is + * a array of unsigned char) from where the 8 bytes of + * the hash has to be filled. Since uuid_t contains 16 + * char elements in the array, each byte of the hash has + * to be filled in one array element. + * + * This function is called twice for 2 hashes (of 8 byte each) to + * be filled in the gfid. + * + * The for loop in this function actually is doing these 2 things + * for each hash + * + * 1) One of the hashes + * tmp[0] = (hash_2 >> 56) & 0xff; + * tmp[1] = (hash_2 >> 48) & 0xff; + * tmp[2] = (hash_2 >> 40) & 0xff; + * tmp[3] = (hash_2 >> 32) & 0xff; + * tmp[4] = (hash_2 >> 24) & 0xff; + * tmp[5] = (hash_2 >> 16) & 0xff; + * tmp[6] = (hash_2 >> 8) & 0xff; + * tmp[7] = (hash_2) & 0xff; + * + * 2) The other hash: + * tmp[8] = (hash_1 >> 56) & 0xff; + * tmp[9] = (hash_1 >> 48) & 0xff; + * tmp[10] = (hash_1 >> 40) & 0xff; + * tmp[11] = (hash_1 >> 32) & 0xff; + * tmp[12] = (hash_1 >> 24) & 0xff; + * tmp[13] = (hash_1 >> 16) & 0xff; + * tmp[14] = (hash_1 >> 8) & 0xff; + * tmp[15] = (hash_1) & 0xff; + **/ +static int +gf_gfid_from_xxh64 (xlator_t *this, uuid_t gfid, GF_XXH64_hash_t hash, + unsigned short index) +{ + int ret = -1; + int i = -1; + + if ((index != 0) && (index != 8)) { + gf_msg_callingfn ("gfid-from-xxh64", GF_LOG_WARNING, 0, + LG_MSG_INDEX_NOT_FOUND, + "index can only be either 0 or 8, as this" + "function's purpose is to encode a 8 byte " + "hash inside the gfid (index: %d)", + index); + goto out; + } + + for (i = 0; i < sizeof(hash); i++) { + /* + * As of now the below statement is equivalent of this. + * gfid[index+i] = (hash >> (64 - (8 * (i+1)))) & 0xff; + */ + gfid[index + i] = (hash >> ((sizeof(hash) * 8) - (8 * (i + 1)))) & + (0xff); + } + + ret = 0; + +out: + return ret; +} + +/** + * This function does the same thing as gf_xxh64_wrapper. But gf_xxh64_wrapper + * does not return anything and in this xlator there is a need for both the + * actual hash and the canonicalized form of the hash. + * + * To summarize: + * - XXH64_hash_t is needed as return because, those bytes which contain the + * hash can be used for different purposes as needed. One example is + * to have those bytes copied into the uuid_t structure to be used as gfid + * - xxh64 string is needed because, it can be used as the key for generating + * the next hash (and any other purpose which might require canonical form + * of the hash). + **/ +GF_XXH64_hash_t +gf_xxh64_hash_wrapper (const unsigned char *data, size_t const len, + unsigned long long const seed, char *xxh64) +{ + unsigned short i = 0; + const unsigned short lim = GF_XXH64_DIGEST_LENGTH * 2 + 1; + GF_XXH64_hash_t hash = 0; + GF_XXH64_canonical_t c_hash = {{0,},}; + const uint8_t *p = (const uint8_t *)&c_hash; + + hash = GF_XXH64 (data, len, seed); + GF_XXH64_canonicalFromHash (&c_hash, hash); + + for (i = 0; i < GF_XXH64_DIGEST_LENGTH; i++) + snprintf (xxh64 + i * 2, lim - i * 2, "%02x", p[i]); + + return hash; +} + +/** + * This is the algorithm followed for generating new gfid + * 1) generate xxh64 hash using snapname and original gfid of the object + * 2) Using the canonicalized form of above hash as the key, generate + * another hash + * 3) Combine both of the 8 byte hashes to generate a 16 byte uuid_t type + * 4) Use the above uuid as the gfid + * + * Each byte of the hash is stored separately in different elements of the + * character array represented by uuid_t + * Ex: tmp[0] = (hash_2 >> 56) & 0xFF + * This saves the most significant byte of hash_2 in tmp[0] + * tmp[1] = (hash_2 >> 48) & 0xFF + * This saves next most significant byte of hash_2 in tmp[1] + * . + * . + * So on. + * tmp[0] - tmp[7] holds the contents of hash_2 + * tmp[8] - tmp[15] hold the conents of hash_1 + * + * The hash generated (i.e. of type XXH64_hash_t) is 8 bytes long. And for + * gfid 16 byte uuid is needed. Hecne the 2 hashes are combined to form + * one 16 byte entity. + **/ +int +gf_gfid_generate_from_xxh64(uuid_t gfid, char *key) +{ + char xxh64_1[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {0,}; + char xxh64_2[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {0}; + GF_XXH64_hash_t hash_1 = 0; + GF_XXH64_hash_t hash_2 = 0; + int ret = -1; + xlator_t *this = THIS; + + hash_1 = gf_xxh64_hash_wrapper ((unsigned char *)key, strlen (key), + GF_XXHSUM64_DEFAULT_SEED, xxh64_1); + + hash_2 = gf_xxh64_hash_wrapper ((unsigned char *)xxh64_1, strlen (xxh64_1), + GF_XXHSUM64_DEFAULT_SEED, xxh64_2); + + /* hash_2 is saved in 1st 8 elements of uuid_t char array */ + if (gf_gfid_from_xxh64 (this, gfid, hash_2, 0)) { + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, + LG_MSG_XXH64_TO_GFID_FAILED, + "failed to encode the hash %llx into the 1st" + "half of gfid", + hash_2); + goto out; + } + + /* hash_1 is saved in the remaining 8 elements of uuid_t */ + if (gf_gfid_from_xxh64 (this, gfid, hash_1, 8)) { + gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, + LG_MSG_XXH64_TO_GFID_FAILED, + "failed to encode the hash %llx into the 2nd" + "half of gfid", + hash_1); + goto out; + } + + gf_msg_debug (this->name, 0, + "gfid generated is %s (hash1: %llx) " + "hash2: %llx, xxh64_1: %s xxh64_2: %s", + uuid_utoa (gfid), hash_1, hash_2, xxh64_1, xxh64_2); + + ret = 0; + +out: + return ret; +} + /* works similar to mkdir(1) -p. */ int diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h index da943f4..e64dea3 100644 --- a/libglusterfs/src/common-utils.h +++ b/libglusterfs/src/common-utils.h @@ -837,6 +837,7 @@ gf_boolean_t gf_is_local_addr (char *hostname); gf_boolean_t gf_is_same_address (char *host1, char *host2); void gf_xxh64_wrapper(const unsigned char *data, size_t len, unsigned long long seed, char *xxh64); +int gf_gfid_generate_from_xxh64(uuid_t gfid, char *key); int gf_set_timestamp (const char *src, const char* dest); int gf_thread_create (pthread_t *thread, const pthread_attr_t *attr, diff --git a/libglusterfs/src/libglusterfs-messages.h b/libglusterfs/src/libglusterfs-messages.h index dd65701..9e35897 100644 --- a/libglusterfs/src/libglusterfs-messages.h +++ b/libglusterfs/src/libglusterfs-messages.h @@ -37,7 +37,7 @@ #define GLFS_LG_BASE GLFS_MSGID_COMP_LIBGLUSTERFS -#define GLFS_LG_NUM_MESSAGES 211 +#define GLFS_LG_NUM_MESSAGES 212 #define GLFS_LG_MSGID_END (GLFS_LG_BASE + GLFS_LG_NUM_MESSAGES + 1) /* Messaged with message IDs */ @@ -1809,6 +1809,15 @@ * @recommendedaction * */ + +#define LG_MSG_XXH64_TO_GFID_FAILED (GLFS_LG_BASE + 212) + +/*! + * @messageid + * @diagnosis + * @recommendedaction + * + */ /*------------*/ #define glfs_msg_end_lg GLFS_LG_MSGID_END, "Invalid: End of messages" diff --git a/xlators/features/snapview-server/src/snapview-server-helpers.c b/xlators/features/snapview-server/src/snapview-server-helpers.c index 6f305db..2ad74ef 100644 --- a/xlators/features/snapview-server/src/snapview-server-helpers.c +++ b/xlators/features/snapview-server/src/snapview-server-helpers.c @@ -329,18 +329,38 @@ out: return svs_fd; } -void -svs_uuid_generate (uuid_t gfid, char *snapname, uuid_t origin_gfid) +int +svs_uuid_generate (xlator_t *this, uuid_t gfid, char *snapname, + uuid_t origin_gfid) { - unsigned char md5_sum[MD5_DIGEST_LENGTH] = {0}; - char ino_string[NAME_MAX + 32] = ""; + char ino_string[NAME_MAX + 32] = ""; + uuid_t tmp = {0,}; + int ret = -1; + + GF_VALIDATE_OR_GOTO ("snapview-server", this, out); + GF_VALIDATE_OR_GOTO (this->name, snapname, out); + + (void)snprintf (ino_string, sizeof (ino_string), "%s%s", snapname, + uuid_utoa (origin_gfid)); + + if (gf_gfid_generate_from_xxh64 (tmp, ino_string)) { + gf_log (this->name, GF_LOG_WARNING, + "failed to generate " + "gfid for object with actual gfid of %s " + "(snapname: %s, key: %s)", + uuid_utoa (origin_gfid), snapname, ino_string); + goto out; + } + + gf_uuid_copy (gfid, tmp); - GF_ASSERT (snapname); + ret = 0; + + gf_log (this->name, GF_LOG_DEBUG, "gfid generated is %s ", + uuid_utoa(gfid)); - (void) snprintf (ino_string, sizeof (ino_string), "%s%s", - snapname, uuid_utoa(origin_gfid)); - MD5((unsigned char *)ino_string, strlen(ino_string), md5_sum); - gf_uuid_copy (gfid, md5_sum); +out: + return ret; } void diff --git a/xlators/features/snapview-server/src/snapview-server.c b/xlators/features/snapview-server/src/snapview-server.c index 22b0e1c..7068c5a 100644 --- a/xlators/features/snapview-server/src/snapview-server.c +++ b/xlators/features/snapview-server/src/snapview-server.c @@ -330,9 +330,18 @@ svs_lookup_entry (xlator_t *this, loc_t *loc, struct iatt *buf, } if (gf_uuid_is_null (loc->gfid) && - gf_uuid_is_null (loc->inode->gfid)) - svs_uuid_generate (gfid, parent_ctx->snapname, object->gfid); - else { + gf_uuid_is_null (loc->inode->gfid)) { + if (svs_uuid_generate (this, gfid, parent_ctx->snapname, + object->gfid)) { + /* + * should op_errno be something else such as + * EINVAL or ESTALE? + */ + op_ret = -1; + *op_errno = EIO; + goto out; + } + } else { if (!gf_uuid_is_null (loc->inode->gfid)) gf_uuid_copy (gfid, loc->inode->gfid); else diff --git a/xlators/features/snapview-server/src/snapview-server.h b/xlators/features/snapview-server/src/snapview-server.h index a12319f..73524a5 100644 --- a/xlators/features/snapview-server/src/snapview-server.h +++ b/xlators/features/snapview-server/src/snapview-server.h @@ -204,8 +204,9 @@ __svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd); svs_fd_t * svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd); -void -svs_uuid_generate (uuid_t gfid, char *snapname, uuid_t origin_gfid); +int +svs_uuid_generate (xlator_t *this, uuid_t gfid, char *snapname, + uuid_t origin_gfid); void svs_fill_ino_from_gfid (struct iatt *buf); -- 1.8.3.1