[MTHCA] erge MR and FMR space on 64-bit systems (mlnx: 2960)

(Linux, 13/02/07, Michael S. Tsirkin, commit: c20e20ab0f3af9a44842ea11287c9ecd034a5d33)

For Tavor, we currently reserve separate MPT and MTT space for FMRs to
avoid abusing the vmalloc space on 32 bit kernels. No such problem
exists on 64 bit kernels so let's not do it there.

This way we have a shared pool for MR and FMR resources, used on
demand.  This will also make it possible to write MTTs for regular
regions directly from driver.

git-svn-id: svn://openib.tc.cornell.edu/gen1@1510 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/trunk/hw/mthca/kernel/mt_memory.h b/trunk/hw/mthca/kernel/mt_memory.h
index 66be696dc726d9caca20a79774b738e8ab961155..29cd334d933fe3d42893eefd79c058af92227098 100644 (file)
--- a/trunk/hw/mthca/kernel/mt_memory.h
+++ b/trunk/hw/mthca/kernel/mt_memory.h
@@ -18,6 +18,7 @@
 // ===========================================
 
 #define memcpy_toio            memcpy
+#define dma_get_cache_alignment                (int)KeGetRecommendedSharedDataAlignment
 
 // ===========================================
 // MACROS

diff --git a/trunk/hw/mthca/kernel/mthca_cmd.c b/trunk/hw/mthca/kernel/mthca_cmd.c
index 47e3df412c7c1b4b3834a31428c192b3155e9493..b1d592d580e42e5414435dc70e384674b1b6785d 100644 (file)
--- a/trunk/hw/mthca/kernel/mthca_cmd.c
+++ b/trunk/hw/mthca/kernel/mthca_cmd.c
@@ -1005,7 +1005,7 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev *dev,
        dev_lim->max_eqs = 1 << (field & 0x7);
        MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MTT_OFFSET);
        if (mthca_is_memfree(dev))
-               dev_lim->reserved_mtts = ALIGN((1 << (field >> 4)) * sizeof(u64),
+               dev_lim->reserved_mtts = ALIGN((1Ui64 << (field >> 4)) * sizeof(u64),
                        MTHCA_MTT_SEG_SIZE) / MTHCA_MTT_SEG_SIZE;
        else
                dev_lim->reserved_mtts = 1 << (field >> 4);

diff --git a/trunk/hw/mthca/kernel/mthca_mr.c b/trunk/hw/mthca/kernel/mthca_mr.c
index 5f8231c57d29757a68706814e0c2ae0d7ea1a7d6..be3c836006d9320b03819bd472b923af4a314616 100644 (file)
--- a/trunk/hw/mthca/kernel/mthca_mr.c
+++ b/trunk/hw/mthca/kernel/mthca_mr.c
@@ -840,7 +840,7 @@ void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
 
 int mthca_init_mr_table(struct mthca_dev *dev)
 {
-       int err, i;
+       int mpts, mtts, err, i;
 
        err = mthca_alloc_init(&dev->mr_table.mpt_alloc,
                               (u32)dev->limits.num_mpts,
@@ -875,9 +875,19 @@ int mthca_init_mr_table(struct mthca_dev *dev)
                        goto err_fmr_mpt;
                }
 
+               mpts = mtts = 1 << i;
+
+       } else {
+               mpts = dev->limits.num_mtt_segs;
+               mtts = dev->limits.num_mpts;
+       }
+       
+       if (!mthca_is_memfree(dev) &&
+               (dev->mthca_flags & MTHCA_FLAG_FMR)) {
+
                dev->mr_table.tavor_fmr.mpt_base =
                        ioremap(dev->mr_table.mpt_base,
-                               (u32)(1 << i) * sizeof (struct mthca_mpt_entry), 
+                               mpts * sizeof (struct mthca_mpt_entry), 
                                &dev->mr_table.tavor_fmr.mpt_base_size);
 
                if (!dev->mr_table.tavor_fmr.mpt_base) {
@@ -888,7 +898,7 @@ int mthca_init_mr_table(struct mthca_dev *dev)
 
                dev->mr_table.tavor_fmr.mtt_base =
                        ioremap(dev->mr_table.mtt_base,
-                               (1 << i) * MTHCA_MTT_SEG_SIZE,
+                               mtts * MTHCA_MTT_SEG_SIZE,
                                &dev->mr_table.tavor_fmr.mtt_base_size );
 
                if (!dev->mr_table.tavor_fmr.mtt_base) {
@@ -896,13 +906,15 @@ int mthca_init_mr_table(struct mthca_dev *dev)
                        err = -ENOMEM;
                        goto err_fmr_mtt;
                }
+       }
 
-               err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, i);
+       if (dev->limits.fmr_reserved_mtts) {
+               err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, fls(mtts - 1));
                if (err)
                        goto err_fmr_mtt_buddy;
 
                /* Prevent regular MRs from using FMR keys */
-               err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, i);
+               err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, fls(mtts - 1));
                if (err)
                        goto err_reserve_fmr;
 

diff --git a/trunk/hw/mthca/kernel/mthca_profile.c b/trunk/hw/mthca/kernel/mthca_profile.c
index 873c1e51c840742f20c0a4220c581dd552ea5d02..a2aaca20081fe8b05cfd7e3b33e4ded9f20f0ebe 100644 (file)
--- a/trunk/hw/mthca/kernel/mthca_profile.c
+++ b/trunk/hw/mthca/kernel/mthca_profile.c
@@ -281,6 +281,10 @@ u64 mthca_make_profile(struct mthca_dev *dev,
        else
                dev->limits.fmr_reserved_mtts = request->fmr_reserved_mtts;
 
+#ifdef _WIN64
+       dev->limits.fmr_reserved_mtts = 0;
+#endif
+
        kfree(profile);
        return total_size;
 }