IB/mlx4: Implement FMRs

Implement FMRs for mlx4.  This is an adaptation of code from mthca.

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Michael S. Tsirkin <mst@dev.mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index d9fc822a146866a75061cc0e78edb608bb026271..d8287d9db41e5a8eb0bb3167db75635c266b4960 100644 (file)
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -607,6 +607,11 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
        ibdev->ib_dev.detach_mcast      = mlx4_ib_mcg_detach;
        ibdev->ib_dev.process_mad       = mlx4_ib_process_mad;
 
+       ibdev->ib_dev.alloc_fmr         = mlx4_ib_fmr_alloc;
+       ibdev->ib_dev.map_phys_fmr      = mlx4_ib_map_phys_fmr;
+       ibdev->ib_dev.unmap_fmr         = mlx4_ib_unmap_fmr;
+       ibdev->ib_dev.dealloc_fmr       = mlx4_ib_fmr_dealloc;
+
        if (init_node_data(ibdev))
                goto err_map;
 

diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 705ff2fa237e85e73d3ddf8a6e0d8911f737a9e1..28697653a370f18fac368f3ecaed4c070517601c 100644 (file)
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -93,6 +93,11 @@ struct mlx4_ib_mr {
        struct ib_umem         *umem;
 };
 
+struct mlx4_ib_fmr {
+       struct ib_fmr           ibfmr;
+       struct mlx4_fmr         mfmr;
+};
+
 struct mlx4_ib_wq {
        u64                    *wrid;
        spinlock_t              lock;
@@ -199,6 +204,10 @@ static inline struct mlx4_ib_mr *to_mmr(struct ib_mr *ibmr)
        return container_of(ibmr, struct mlx4_ib_mr, ibmr);
 }
 
+static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr)
+{
+       return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr);
+}
 static inline struct mlx4_ib_qp *to_mqp(struct ib_qp *ibqp)
 {
        return container_of(ibqp, struct mlx4_ib_qp, ibqp);
@@ -284,6 +293,13 @@ int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags,    u8 port_num,
 int mlx4_ib_mad_init(struct mlx4_ib_dev *dev);
 void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev);
 
+struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags,
+                                 struct ib_fmr_attr *fmr_attr);
+int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages,
+                        u64 iova);
+int mlx4_ib_unmap_fmr(struct list_head *fmr_list);
+int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr);
+
 static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah)
 {
        return !!(ah->av.g_slid & 0x80);

diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 734ec2bd15cd58229abe5fb1e59f1503eada5a94..7dc91a3e712ddb812304984ad5b5f24403709e68 100644 (file)
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -181,3 +181,96 @@ int mlx4_ib_dereg_mr(struct ib_mr *ibmr)
 
        return 0;
 }
+
+struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc,
+                                struct ib_fmr_attr *fmr_attr)
+{
+       struct mlx4_ib_dev *dev = to_mdev(pd->device);
+       struct mlx4_ib_fmr *fmr;
+       int err = -ENOMEM;
+
+       fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
+       if (!fmr)
+               return ERR_PTR(-ENOMEM);
+
+       err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc),
+                            fmr_attr->max_pages, fmr_attr->max_maps,
+                            fmr_attr->page_shift, &fmr->mfmr);
+       if (err)
+               goto err_free;
+
+       err = mlx4_mr_enable(to_mdev(pd->device)->dev, &fmr->mfmr.mr);
+       if (err)
+               goto err_mr;
+
+       fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key;
+
+       return &fmr->ibfmr;
+
+err_mr:
+       mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr);
+
+err_free:
+       kfree(fmr);
+
+       return ERR_PTR(err);
+}
+
+int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
+                     int npages, u64 iova)
+{
+       struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
+       struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device);
+
+       return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova,
+                                &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
+}
+
+int mlx4_ib_unmap_fmr(struct list_head *fmr_list)
+{
+       struct ib_fmr *ibfmr;
+       int err;
+       struct mlx4_dev *mdev = NULL;
+
+       list_for_each_entry(ibfmr, fmr_list, list) {
+               if (mdev && to_mdev(ibfmr->device)->dev != mdev)
+                       return -EINVAL;
+               mdev = to_mdev(ibfmr->device)->dev;
+       }
+
+       if (!mdev)
+               return 0;
+
+       list_for_each_entry(ibfmr, fmr_list, list) {
+               struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
+
+               mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
+       }
+
+       /*
+        * Make sure all MPT status updates are visible before issuing
+        * SYNC_TPT firmware command.
+        */
+       wmb();
+
+       err = mlx4_SYNC_TPT(mdev);
+       if (err)
+               printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when "
+                      "unmapping FMRs\n", err);
+
+       return 0;
+}
+
+int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr)
+{
+       struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
+       struct mlx4_ib_dev *dev = to_mdev(ibfmr->device);
+       int err;
+
+       err = mlx4_fmr_free(dev->dev, &ifmr->mfmr);
+
+       if (!err)
+               kfree(ifmr);
+
+       return err;
+}

diff --git a/drivers/net/mlx4/mr.c b/drivers/net/mlx4/mr.c
index 3cc98c699aaf5e82c9fbc466503771650435f959..4bc39e65015a6aab4483dd7bf407bee75170f9a1 100644 (file)
--- a/drivers/net/mlx4/mr.c
+++ b/drivers/net/mlx4/mr.c
@@ -68,6 +68,9 @@ struct mlx4_mpt_entry {
 
 #define MLX4_MTT_FLAG_PRESENT          1
 
+#define MLX4_MPT_STATUS_SW             0xF0
+#define MLX4_MPT_STATUS_HW             0x00
+
 static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order)
 {
        int o;
@@ -469,3 +472,165 @@ void mlx4_cleanup_mr_table(struct mlx4_dev *dev)
        mlx4_buddy_cleanup(&mr_table->mtt_buddy);
        mlx4_bitmap_cleanup(&mr_table->mpt_bitmap);
 }
+
+static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list,
+                                 int npages, u64 iova)
+{
+       int i, page_mask;
+
+       if (npages > fmr->max_pages)
+               return -EINVAL;
+
+       page_mask = (1 << fmr->page_shift) - 1;
+
+       /* We are getting page lists, so va must be page aligned. */
+       if (iova & page_mask)
+               return -EINVAL;
+
+       /* Trust the user not to pass misaligned data in page_list */
+       if (0)
+               for (i = 0; i < npages; ++i) {
+                       if (page_list[i] & ~page_mask)
+                               return -EINVAL;
+               }
+
+       if (fmr->maps >= fmr->max_maps)
+               return -EINVAL;
+
+       return 0;
+}
+
+int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,
+                     int npages, u64 iova, u32 *lkey, u32 *rkey)
+{
+       u32 key;
+       int i, err;
+
+       err = mlx4_check_fmr(fmr, page_list, npages, iova);
+       if (err)
+               return err;
+
+       ++fmr->maps;
+
+       key = key_to_hw_index(fmr->mr.key);
+       key += dev->caps.num_mpts;
+       *lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
+
+       *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
+
+       /* Make sure MPT status is visible before writing MTT entries */
+       wmb();
+
+       for (i = 0; i < npages; ++i)
+               fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT);
+
+       dma_sync_single(&dev->pdev->dev, fmr->dma_handle,
+                       npages * sizeof(u64), DMA_TO_DEVICE);
+
+       fmr->mpt->key    = cpu_to_be32(key);
+       fmr->mpt->lkey   = cpu_to_be32(key);
+       fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift));
+       fmr->mpt->start  = cpu_to_be64(iova);
+
+       /* Make MTT entries are visible before setting MPT status */
+       wmb();
+
+       *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW;
+
+       /* Make sure MPT status is visible before consumer can use FMR */
+       wmb();
+
+       return 0;
+}
+EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr);
+
+int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
+                  int max_maps, u8 page_shift, struct mlx4_fmr *fmr)
+{
+       struct mlx4_priv *priv = mlx4_priv(dev);
+       u64 mtt_seg;
+       int err = -ENOMEM;
+
+       if (page_shift < 12 || page_shift >= 32)
+               return -EINVAL;
+
+       /* All MTTs must fit in the same page */
+       if (max_pages * sizeof *fmr->mtts > PAGE_SIZE)
+               return -EINVAL;
+
+       fmr->page_shift = page_shift;
+       fmr->max_pages  = max_pages;
+       fmr->max_maps   = max_maps;
+       fmr->maps = 0;
+
+       err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages,
+                           page_shift, &fmr->mr);
+       if (err)
+               return err;
+
+       mtt_seg = fmr->mr.mtt.first_seg * dev->caps.mtt_entry_sz;
+
+       fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table,
+                                   fmr->mr.mtt.first_seg,
+                                   &fmr->dma_handle);
+       if (!fmr->mtts) {
+               err = -ENOMEM;
+               goto err_free;
+       }
+
+       fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table,
+                                   key_to_hw_index(fmr->mr.key), NULL);
+       if (!fmr->mpt) {
+               err = -ENOMEM;
+               goto err_free;
+       }
+
+       return 0;
+
+err_free:
+       mlx4_mr_free(dev, &fmr->mr);
+       return err;
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_alloc);
+
+int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
+{
+       return mlx4_mr_enable(dev, &fmr->mr);
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_enable);
+
+void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
+                   u32 *lkey, u32 *rkey)
+{
+       u32 key;
+
+       if (!fmr->maps)
+               return;
+
+       key = key_to_hw_index(fmr->mr.key);
+       key &= dev->caps.num_mpts - 1;
+       *lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
+
+       fmr->maps = 0;
+
+       *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_unmap);
+
+int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
+{
+       if (fmr->maps)
+               return -EBUSY;
+
+       fmr->mr.enabled = 0;
+       mlx4_mr_free(dev, &fmr->mr);
+
+       return 0;
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_free);
+
+int mlx4_SYNC_TPT(struct mlx4_dev *dev)
+{
+       return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000);
+}
+EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT);

diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index a93520c76fde238093644f1366164b5552c7423c..222815d91c40108f87d8c47b3a0f5541d7b0c5ae 100644 (file)
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -214,6 +214,17 @@ struct mlx4_mr {
        int                     enabled;
 };
 
+struct mlx4_fmr {
+       struct mlx4_mr          mr;
+       struct mlx4_mpt_entry  *mpt;
+       __be64                 *mtts;
+       dma_addr_t              dma_handle;
+       int                     max_pages;
+       int                     max_maps;
+       int                     maps;
+       u8                      page_shift;
+};
+
 struct mlx4_uar {
        unsigned long           pfn;
        int                     index;
@@ -337,4 +348,14 @@ int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port);
 int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);
 int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);
 
+int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,
+                     int npages, u64 iova, u32 *lkey, u32 *rkey);
+int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
+                  int max_maps, u8 page_shift, struct mlx4_fmr *fmr);
+int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr);
+void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
+                   u32 *lkey, u32 *rkey);
+int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr);
+int mlx4_SYNC_TPT(struct mlx4_dev *dev);
+
 #endif /* MLX4_DEVICE_H */