cryptospore / rpms / qemu-kvm

Forked from rpms/qemu-kvm 2 years ago
Clone

Blame SOURCES/kvm-hw-arm-smmu-common-Manage-IOTLB-block-entries.patch

a19a21
From 4770f43dab482e4585d3555933a473cf24e796db Mon Sep 17 00:00:00 2001
a19a21
From: eperezma <eperezma@redhat.com>
a19a21
Date: Tue, 12 Jan 2021 14:36:30 -0500
a19a21
Subject: [PATCH 06/17] hw/arm/smmu-common: Manage IOTLB block entries
a19a21
MIME-Version: 1.0
a19a21
Content-Type: text/plain; charset=UTF-8
a19a21
Content-Transfer-Encoding: 8bit
a19a21
a19a21
RH-Author: eperezma <eperezma@redhat.com>
a19a21
Message-id: <20210112143638.374060-6-eperezma@redhat.com>
a19a21
Patchwork-id: 100598
a19a21
O-Subject: [RHEL-8.4.0 qemu-kvm PATCH v2 05/13] hw/arm/smmu-common: Manage IOTLB block entries
a19a21
Bugzilla: 1843852
a19a21
RH-Acked-by: Xiao Wang <jasowang@redhat.com>
a19a21
RH-Acked-by: Peter Xu <peterx@redhat.com>
a19a21
RH-Acked-by: Auger Eric <eric.auger@redhat.com>
a19a21
a19a21
From: Eric Auger <eric.auger@redhat.com>
a19a21
a19a21
At the moment each entry in the IOTLB corresponds to a page sized
a19a21
mapping (4K, 16K or 64K), even if the page belongs to a mapped
a19a21
block. In case of block mapping this unefficiently consumes IOTLB
a19a21
entries.
a19a21
a19a21
Change the value of the entry so that it reflects the actual
a19a21
mapping it belongs to (block or page start address and size).
a19a21
a19a21
Also the level/tg of the entry is encoded in the key. In subsequent
a19a21
patches we will enable range invalidation. This latter is able
a19a21
to provide the level/tg of the entry.
a19a21
a19a21
Encoding the level/tg directly in the key will allow to invalidate
a19a21
using g_hash_table_remove() when num_pages equals to 1.
a19a21
a19a21
Signed-off-by: Eric Auger <eric.auger@redhat.com>
a19a21
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
a19a21
Message-id: 20200728150815.11446-6-eric.auger@redhat.com
a19a21
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
a19a21
(cherry picked from commit 9e54dee71fcfaae69f87b8e1f51485a832266a39)
a19a21
Signed-off-by: Eugenio PĂ©rez <eperezma@redhat.com>
a19a21
Signed-off-by: Danilo C. L. de Paula <ddepaula@redhat.com>
a19a21
---
a19a21
 hw/arm/smmu-common.c         | 67 ++++++++++++++++++++++++++----------
a19a21
 hw/arm/smmu-internal.h       |  7 ++++
a19a21
 hw/arm/smmuv3.c              |  6 ++--
a19a21
 hw/arm/trace-events          |  2 +-
a19a21
 include/hw/arm/smmu-common.h | 10 ++++--
a19a21
 5 files changed, 67 insertions(+), 25 deletions(-)
a19a21
a19a21
diff --git a/hw/arm/smmu-common.c b/hw/arm/smmu-common.c
a19a21
index 06e9e38b007..8007edeaaa2 100644
a19a21
--- a/hw/arm/smmu-common.c
a19a21
+++ b/hw/arm/smmu-common.c
a19a21
@@ -39,7 +39,7 @@ static guint smmu_iotlb_key_hash(gconstpointer v)
a19a21
 
a19a21
     /* Jenkins hash */
a19a21
     a = b = c = JHASH_INITVAL + sizeof(*key);
a19a21
-    a += key->asid;
a19a21
+    a += key->asid + key->level + key->tg;
a19a21
     b += extract64(key->iova, 0, 32);
a19a21
     c += extract64(key->iova, 32, 32);
a19a21
 
a19a21
@@ -51,24 +51,41 @@ static guint smmu_iotlb_key_hash(gconstpointer v)
a19a21
 
a19a21
 static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2)
a19a21
 {
a19a21
-    const SMMUIOTLBKey *k1 = v1;
a19a21
-    const SMMUIOTLBKey *k2 = v2;
a19a21
+    SMMUIOTLBKey *k1 = (SMMUIOTLBKey *)v1, *k2 = (SMMUIOTLBKey *)v2;
a19a21
 
a19a21
-    return (k1->asid == k2->asid) && (k1->iova == k2->iova);
a19a21
+    return (k1->asid == k2->asid) && (k1->iova == k2->iova) &&
a19a21
+           (k1->level == k2->level) && (k1->tg == k2->tg);
a19a21
 }
a19a21
 
a19a21
-SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova)
a19a21
+SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova,
a19a21
+                                uint8_t tg, uint8_t level)
a19a21
 {
a19a21
-    SMMUIOTLBKey key = {.asid = asid, .iova = iova};
a19a21
+    SMMUIOTLBKey key = {.asid = asid, .iova = iova, .tg = tg, .level = level};
a19a21
 
a19a21
     return key;
a19a21
 }
a19a21
 
a19a21
 SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg,
a19a21
-                                hwaddr iova)
a19a21
+                                SMMUTransTableInfo *tt, hwaddr iova)
a19a21
 {
a19a21
-    SMMUIOTLBKey key = smmu_get_iotlb_key(cfg->asid, iova);
a19a21
-    SMMUTLBEntry *entry = g_hash_table_lookup(bs->iotlb, &key);
a19a21
+    uint8_t tg = (tt->granule_sz - 10) / 2;
a19a21
+    uint8_t inputsize = 64 - tt->tsz;
a19a21
+    uint8_t stride = tt->granule_sz - 3;
a19a21
+    uint8_t level = 4 - (inputsize - 4) / stride;
a19a21
+    SMMUTLBEntry *entry = NULL;
a19a21
+
a19a21
+    while (level <= 3) {
a19a21
+        uint64_t subpage_size = 1ULL << level_shift(level, tt->granule_sz);
a19a21
+        uint64_t mask = subpage_size - 1;
a19a21
+        SMMUIOTLBKey key;
a19a21
+
a19a21
+        key = smmu_get_iotlb_key(cfg->asid, iova & ~mask, tg, level);
a19a21
+        entry = g_hash_table_lookup(bs->iotlb, &key);
a19a21
+        if (entry) {
a19a21
+            break;
a19a21
+        }
a19a21
+        level++;
a19a21
+    }
a19a21
 
a19a21
     if (entry) {
a19a21
         cfg->iotlb_hits++;
a19a21
@@ -89,13 +106,14 @@ SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg,
a19a21
 void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *new)
a19a21
 {
a19a21
     SMMUIOTLBKey *key = g_new0(SMMUIOTLBKey, 1);
a19a21
+    uint8_t tg = (new->granule - 10) / 2;
a19a21
 
a19a21
     if (g_hash_table_size(bs->iotlb) >= SMMU_IOTLB_MAX_SIZE) {
a19a21
         smmu_iotlb_inv_all(bs);
a19a21
     }
a19a21
 
a19a21
-    *key = smmu_get_iotlb_key(cfg->asid, new->entry.iova);
a19a21
-    trace_smmu_iotlb_insert(cfg->asid, new->entry.iova);
a19a21
+    *key = smmu_get_iotlb_key(cfg->asid, new->entry.iova, tg, new->level);
a19a21
+    trace_smmu_iotlb_insert(cfg->asid, new->entry.iova, tg, new->level);
a19a21
     g_hash_table_insert(bs->iotlb, key, new);
a19a21
 }
a19a21
 
a19a21
@@ -114,12 +132,26 @@ static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value,
a19a21
     return SMMU_IOTLB_ASID(*iotlb_key) == asid;
a19a21
 }
a19a21
 
a19a21
-inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova)
a19a21
+static gboolean smmu_hash_remove_by_asid_iova(gpointer key, gpointer value,
a19a21
+                                              gpointer user_data)
a19a21
 {
a19a21
-    SMMUIOTLBKey key = smmu_get_iotlb_key(asid, iova);
a19a21
+    SMMUTLBEntry *iter = (SMMUTLBEntry *)value;
a19a21
+    IOMMUTLBEntry *entry = &iter->entry;
a19a21
+    SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data;
a19a21
+    SMMUIOTLBKey iotlb_key = *(SMMUIOTLBKey *)key;
a19a21
+
a19a21
+    if (info->asid >= 0 && info->asid != SMMU_IOTLB_ASID(iotlb_key)) {
a19a21
+        return false;
a19a21
+    }
a19a21
+    return (info->iova & ~entry->addr_mask) == entry->iova;
a19a21
+}
a19a21
+
a19a21
+inline void smmu_iotlb_inv_iova(SMMUState *s, int asid, dma_addr_t iova)
a19a21
+{
a19a21
+    SMMUIOTLBPageInvInfo info = {.asid = asid, .iova = iova};
a19a21
 
a19a21
     trace_smmu_iotlb_inv_iova(asid, iova);
a19a21
-    g_hash_table_remove(s->iotlb, &key);
a19a21
+    g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid_iova, &info;;
a19a21
 }
a19a21
 
a19a21
 inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid)
a19a21
@@ -247,9 +279,6 @@ static int smmu_ptw_64(SMMUTransCfg *cfg,
a19a21
     baseaddr = extract64(tt->ttb, 0, 48);
a19a21
     baseaddr &= ~indexmask;
a19a21
 
a19a21
-    tlbe->entry.iova = iova;
a19a21
-    tlbe->entry.addr_mask = (1 << granule_sz) - 1;
a19a21
-
a19a21
     while (level <= 3) {
a19a21
         uint64_t subpage_size = 1ULL << level_shift(level, granule_sz);
a19a21
         uint64_t mask = subpage_size - 1;
a19a21
@@ -299,7 +328,9 @@ static int smmu_ptw_64(SMMUTransCfg *cfg,
a19a21
             goto error;
a19a21
         }
a19a21
 
a19a21
-        tlbe->entry.translated_addr = gpa + (iova & mask);
a19a21
+        tlbe->entry.translated_addr = gpa;
a19a21
+        tlbe->entry.iova = iova & ~mask;
a19a21
+        tlbe->entry.addr_mask = mask;
a19a21
         tlbe->entry.perm = PTE_AP_TO_PERM(ap);
a19a21
         tlbe->level = level;
a19a21
         tlbe->granule = granule_sz;
a19a21
diff --git a/hw/arm/smmu-internal.h b/hw/arm/smmu-internal.h
a19a21
index 3104f768cd2..55147f29be4 100644
a19a21
--- a/hw/arm/smmu-internal.h
a19a21
+++ b/hw/arm/smmu-internal.h
a19a21
@@ -97,4 +97,11 @@ uint64_t iova_level_offset(uint64_t iova, int inputsize,
a19a21
 }
a19a21
 
a19a21
 #define SMMU_IOTLB_ASID(key) ((key).asid)
a19a21
+
a19a21
+typedef struct SMMUIOTLBPageInvInfo {
a19a21
+    int asid;
a19a21
+    uint64_t iova;
a19a21
+    uint64_t mask;
a19a21
+} SMMUIOTLBPageInvInfo;
a19a21
+
a19a21
 #endif
a19a21
diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
a19a21
index ad8212779d3..067c9480a03 100644
a19a21
--- a/hw/arm/smmuv3.c
a19a21
+++ b/hw/arm/smmuv3.c
a19a21
@@ -662,7 +662,7 @@ static IOMMUTLBEntry smmuv3_translate(IOMMUMemoryRegion *mr, hwaddr addr,
a19a21
     page_mask = (1ULL << (tt->granule_sz)) - 1;
a19a21
     aligned_addr = addr & ~page_mask;
a19a21
 
a19a21
-    cached_entry = smmu_iotlb_lookup(bs, cfg, aligned_addr);
a19a21
+    cached_entry = smmu_iotlb_lookup(bs, cfg, tt, aligned_addr);
a19a21
     if (cached_entry) {
a19a21
         if ((flag & IOMMU_WO) && !(cached_entry->entry.perm & IOMMU_WO)) {
a19a21
             status = SMMU_TRANS_ERROR;
a19a21
@@ -732,7 +732,7 @@ epilogue:
a19a21
     case SMMU_TRANS_SUCCESS:
a19a21
         entry.perm = flag;
a19a21
         entry.translated_addr = cached_entry->entry.translated_addr +
a19a21
-                                    (addr & page_mask);
a19a21
+                                    (addr & cached_entry->entry.addr_mask);
a19a21
         entry.addr_mask = cached_entry->entry.addr_mask;
a19a21
         trace_smmuv3_translate_success(mr->parent_obj.name, sid, addr,
a19a21
                                        entry.translated_addr, entry.perm);
a19a21
@@ -960,7 +960,7 @@ static int smmuv3_cmdq_consume(SMMUv3State *s)
a19a21
 
a19a21
             trace_smmuv3_cmdq_tlbi_nh_vaa(vmid, addr);
a19a21
             smmuv3_inv_notifiers_iova(bs, -1, addr);
a19a21
-            smmu_iotlb_inv_all(bs);
a19a21
+            smmu_iotlb_inv_iova(bs, -1, addr);
a19a21
             break;
a19a21
         }
a19a21
         case SMMU_CMD_TLBI_NH_VA:
a19a21
diff --git a/hw/arm/trace-events b/hw/arm/trace-events
a19a21
index b808a1bfc19..f74d3e920f1 100644
a19a21
--- a/hw/arm/trace-events
a19a21
+++ b/hw/arm/trace-events
a19a21
@@ -16,7 +16,7 @@ smmu_iotlb_inv_iova(uint16_t asid, uint64_t addr) "IOTLB invalidate asid=%d addr
a19a21
 smmu_inv_notifiers_mr(const char *name) "iommu mr=%s"
a19a21
 smmu_iotlb_lookup_hit(uint16_t asid, uint64_t addr, uint32_t hit, uint32_t miss, uint32_t p) "IOTLB cache HIT asid=%d addr=0x%"PRIx64" hit=%d miss=%d hit rate=%d"
a19a21
 smmu_iotlb_lookup_miss(uint16_t asid, uint64_t addr, uint32_t hit, uint32_t miss, uint32_t p) "IOTLB cache MISS asid=%d addr=0x%"PRIx64" hit=%d miss=%d hit rate=%d"
a19a21
-smmu_iotlb_insert(uint16_t asid, uint64_t addr) "IOTLB ++ asid=%d addr=0x%"PRIx64
a19a21
+smmu_iotlb_insert(uint16_t asid, uint64_t addr, uint8_t tg, uint8_t level) "IOTLB ++ asid=%d addr=0x%"PRIx64" tg=%d level=%d"
a19a21
 
a19a21
 # smmuv3.c
a19a21
 smmuv3_read_mmio(uint64_t addr, uint64_t val, unsigned size, uint32_t r) "addr: 0x%"PRIx64" val:0x%"PRIx64" size: 0x%x(%d)"
a19a21
diff --git a/include/hw/arm/smmu-common.h b/include/hw/arm/smmu-common.h
a19a21
index 277923bdc0a..bbf3abc41fd 100644
a19a21
--- a/include/hw/arm/smmu-common.h
a19a21
+++ b/include/hw/arm/smmu-common.h
a19a21
@@ -97,6 +97,8 @@ typedef struct SMMUPciBus {
a19a21
 typedef struct SMMUIOTLBKey {
a19a21
     uint64_t iova;
a19a21
     uint16_t asid;
a19a21
+    uint8_t tg;
a19a21
+    uint8_t level;
a19a21
 } SMMUIOTLBKey;
a19a21
 
a19a21
 typedef struct SMMUState {
a19a21
@@ -159,12 +161,14 @@ IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid);
a19a21
 
a19a21
 #define SMMU_IOTLB_MAX_SIZE 256
a19a21
 
a19a21
-SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg, hwaddr iova);
a19a21
+SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg,
a19a21
+                                SMMUTransTableInfo *tt, hwaddr iova);
a19a21
 void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *entry);
a19a21
-SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova);
a19a21
+SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova,
a19a21
+                                uint8_t tg, uint8_t level);
a19a21
 void smmu_iotlb_inv_all(SMMUState *s);
a19a21
 void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid);
a19a21
-void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova);
a19a21
+void smmu_iotlb_inv_iova(SMMUState *s, int asid, dma_addr_t iova);
a19a21
 
a19a21
 /* Unmap the range of all the notifiers registered to any IOMMU mr */
a19a21
 void smmu_inv_notifiers_all(SMMUState *s);
a19a21
-- 
a19a21
2.27.0
a19a21