From 6122f6c80df874f68ec6d8e2e0098cf703cc531e Mon Sep 17 00:00:00 2001

From: Jason Dillaman <dillaman@redhat.com>

Date: Wed, 8 Apr 2015 19:06:52 -0400

Subject: [PATCH 15/22] librbd: avoid blocking AIO API methods

Enqueue all AIO API methods within the new librbd thread pool to

reduce the possibility of any blocking operations. To maintain

backwards compatibility with the legacy return codes of the API's

AIO methods, it's still possible to block attempting to acquire

the snap_lock.

Fixes: #11056

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

(cherry picked from commit 3a7b5e30efdb21aa1a0aeb68a5d02a1ac2a5faf3)

(cherry picked from commit aa45ee0d7d6f9d8e2cc43ef43f0a9762977ca53f)

---

 src/librbd/librbd.cc | 110 +++++++++++++++++++++++++++++++++++++++++++++------

 1 file changed, 99 insertions(+), 11 deletions(-)

diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc

index 658f24b..7f966be 100644

--- a/src/librbd/librbd.cc

+++ b/src/librbd/librbd.cc

@@ -20,6 +20,7 @@

 #include "common/errno.h"

 #include "common/snap_types.h"

 #include "common/perf_counters.h"

+#include "common/WorkQueue.h"

 #include "include/Context.h"

 #include "include/rbd/librbd.hpp"

 #include "osdc/ObjectCacher.h"

@@ -45,6 +46,82 @@ using ceph::bufferlist;

 using librados::snap_t;

 using librados::IoCtx;

+namespace {

+

+class C_AioReadWQ : public Context {

+public:

+  C_AioReadWQ(librbd::ImageCtx *ictx, uint64_t off, size_t len,

+              char *buf, bufferlist *pbl, librbd::AioCompletion *c)

+    : m_ictx(ictx), m_off(off), m_len(len), m_buf(buf), m_pbl(pbl), m_comp(c) {

+  }

+protected:

+  virtual void finish(int r) {

+    librbd::aio_read(m_ictx, m_off, m_len, m_buf, m_pbl, m_comp);

+  }

+private:

+  librbd::ImageCtx *m_ictx;

+  uint64_t m_off;

+  uint64_t m_len;

+  char *m_buf;

+  bufferlist *m_pbl;

+  librbd::AioCompletion *m_comp;

+};

+

+class C_AioWriteWQ : public Context {

+public:

+  C_AioWriteWQ(librbd::ImageCtx *ictx, uint64_t off, size_t len,

+               const char *buf, librbd::AioCompletion *c)

+    : m_ictx(ictx), m_off(off), m_len(len), m_buf(buf), m_comp(c) {

+  }

+protected:

+  virtual void finish(int r) {

+    librbd::aio_write(m_ictx, m_off, m_len, m_buf, m_comp);

+  }

+private:

+  librbd::ImageCtx *m_ictx;

+  uint64_t m_off;

+  uint64_t m_len;

+  const char *m_buf;

+  librbd::AioCompletion *m_comp;

+};

+

+class C_AioDiscardWQ : public Context {

+public:

+  C_AioDiscardWQ(librbd::ImageCtx *ictx, uint64_t off, uint64_t len,

+                 librbd::AioCompletion *c)

+    : m_ictx(ictx), m_off(off), m_len(len), m_comp(c) {

+  }

+protected:

+  virtual void finish(int r) {

+    librbd::aio_discard(m_ictx, m_off, m_len, m_comp);

+  }

+private:

+  librbd::ImageCtx *m_ictx;

+  uint64_t m_off;

+  uint64_t m_len;

+  librbd::AioCompletion *m_comp;

+};

+

+class C_AioFlushWQ : public Context {

+public:

+  C_AioFlushWQ(librbd::ImageCtx *ictx, librbd::AioCompletion *c)

+    : m_ictx(ictx), m_comp(c) {

+  }

+protected:

+  virtual void finish(int r) {

+    librbd::aio_flush(m_ictx, m_comp);

+  }

+private:

+  librbd::ImageCtx *m_ictx;

+  librbd::AioCompletion *m_comp;

+};

+

+librbd::AioCompletion* get_aio_completion(librbd::RBD::AioCompletion *comp) {

+  return reinterpret_cast<librbd::AioCompletion *>(comp->pc);

+}

+

+} // anonymous namespace

+

 namespace librbd {

   ProgressContext::~ProgressContext()

   {

@@ -483,14 +560,17 @@ namespace librbd {

     ImageCtx *ictx = (ImageCtx *)ctx;

     if (bl.length() < len)

       return -EINVAL;

-    return librbd::aio_write(ictx, off, len, bl.c_str(),

-			     (librbd::AioCompletion *)c->pc);

+    ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, bl.c_str(),

+                                                 get_aio_completion(c)));

+    return 0;

   }

   int Image::aio_discard(uint64_t off, uint64_t len, RBD::AioCompletion *c)

   {

     ImageCtx *ictx = (ImageCtx *)ctx;

-    return librbd::aio_discard(ictx, off, len, (librbd::AioCompletion *)c->pc);

+    ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len,

+                                                   get_aio_completion(c)));

+    return 0;

   }

   int Image::aio_read(uint64_t off, size_t len, bufferlist& bl,

@@ -499,7 +579,9 @@ namespace librbd {

     ImageCtx *ictx = (ImageCtx *)ctx;

     ldout(ictx->cct, 10) << "Image::aio_read() buf=" << (void *)bl.c_str() << "~"

 			 << (void *)(bl.c_str() + len - 1) << dendl;

-    return librbd::aio_read(ictx, off, len, NULL, &bl, (librbd::AioCompletion *)c->pc);

+    ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, NULL, &bl,

+                                                get_aio_completion(c)));

+    return 0;

   }

   int Image::flush()

@@ -511,7 +593,8 @@ namespace librbd {

   int Image::aio_flush(RBD::AioCompletion *c)

   {

     ImageCtx *ictx = (ImageCtx *)ctx;

-    return librbd::aio_flush(ictx, (librbd::AioCompletion *)c->pc);

+    ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, get_aio_completion(c)));

+    return 0;

   }

   int Image::invalidate_cache()

@@ -1102,8 +1185,9 @@ extern "C" int rbd_aio_write(rbd_image_t image, uint64_t off, size_t len,

 {

   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;

   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;

-  return librbd::aio_write(ictx, off, len, buf,

-			   (librbd::AioCompletion *)comp->pc);

+  ictx->aio_work_queue->queue(new C_AioWriteWQ(ictx, off, len, buf,

+                                               get_aio_completion(comp)));

+  return 0;

 }

 extern "C" int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len,

@@ -1111,7 +1195,9 @@ extern "C" int rbd_aio_discard(rbd_image_t image, uint64_t off, uint64_t len,

 {

   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;

   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;

-  return librbd::aio_discard(ictx, off, len, (librbd::AioCompletion *)comp->pc);

+  ictx->aio_work_queue->queue(new C_AioDiscardWQ(ictx, off, len,

+                                                 get_aio_completion(comp)));

+  return 0;

 }

 extern "C" int rbd_aio_read(rbd_image_t image, uint64_t off, size_t len,

@@ -1119,8 +1205,9 @@ extern "C" int rbd_aio_read(rbd_image_t image, uint64_t off, size_t len,

 {

   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;

   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;

-  return librbd::aio_read(ictx, off, len, buf, NULL,

-			  (librbd::AioCompletion *)comp->pc);

+  ictx->aio_work_queue->queue(new C_AioReadWQ(ictx, off, len, buf, NULL,

+                                              get_aio_completion(comp)));

+  return 0;

 }

 extern "C" int rbd_flush(rbd_image_t image)

@@ -1133,7 +1220,8 @@ extern "C" int rbd_aio_flush(rbd_image_t image, rbd_completion_t c)

 {

   librbd::ImageCtx *ictx = (librbd::ImageCtx *)image;

   librbd::RBD::AioCompletion *comp = (librbd::RBD::AioCompletion *)c;

-  return librbd::aio_flush(ictx, (librbd::AioCompletion *)comp->pc);

+  ictx->aio_work_queue->queue(new C_AioFlushWQ(ictx, get_aio_completion(comp)));

+  return 0;

 }

 extern "C" int rbd_invalidate_cache(rbd_image_t image)

-- 

2.1.0