1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

$ yum install -y libaio libaio-devel ncurses ncureses-devel CUnit fuse3 fuse3-devel jq
$ pip3 install meson
$ wget https://sourceforge.net/projects/cunit/files/latest/download

$ git clone https://github.com/spdk/spdk
$ cd spdk
$ git submodule update --init

$ sh scripts/pkgdep.sh
$ wget -O /tmp/ninja-linux.zip https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip && unzip /tmp/ninja-linux.zip && mv ninja /usr/local/bin/
$ ./configure --with-fuse
$ wget http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/libaio-devel-0.3.112-1.el8.x86_64.rpm -o /tmp/
$ sudo rpm -ivh /tmp/
$ make -j8

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

#iommu 设置
#/etc/default/grub中，增加
GRUB_CMDLINE_LINUX="xxx default_hugepagesz=1G hugepagesz=1G hugepages=16 hugepagesz=2M hugepages=2048 intel_iommu=on"
$ grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg #uefi
$ grub2-mkconfig -o /boot/grub2/grub.cfg   #no uefi

$ mkdir -p /mnt/huge1G
$ echo "nodev /mnt/huge1G hugetlbfs pagesize=1GB 0 0" >> /etc/fstab

# 设置hugepage
$ echo 16 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
$ mount -t hugetlbfs none /mnt/huge1G -o pagesize=1G

# 加载vfio-pci 驱动模块
$ modprobe vfio-pci
$ modprobe

# 查看nvme 设备
$ lspci -nn | grep "Non-Volatile"
#> 84:00.0 Non-Volatile memory controller [0108]: Intel Corporation NVMe Datacenter SSD [3DNAND, Beta Rock Controller] [8086:0a54]
# [[[[<domain>]:]<bus>]:][<device>][.[<func>]]：0000:84:00.0
# [<vendor>]:[<device>]： 8086:0a54

# 解绑nvme驱动
$ echo "0000:84:00.0" >/sys/bus/pci/devices/0000:84:00.0/driver/unbind

# 绑定到vfio驱动
$ echo 8086 0a54 | tee /sys/bus/pci/drivers/vfio-pci/new_id
#> 绑定成功后，检查
#> - lsblk 看不到nvme盘
#> - vfio下面多了一个设备文件：ls /dev/vfio/4

# 或者绑定到uio
$ echo 8086 0a54 > /sys/bus/pci/drivers/uio_pci_generic/new_id

# 驱动准备完成
$ cd <SPDK_HOME>

# 生成nvme配置文件
$ scripts/gen_nvme.sh --json-with-subsystems > nvme.json

# 查看状态
$ sudo scripts/setup.sh status

# 设置hugepage
$ HUGEMEM=5120 scripts/setup.sh

# 生成配置文件
$ scripts/gen_nvme.sh --json-with-subsystems > nvme.json

# 初始化blobfs
$ ./test/blobfs/mkfs/mkfs ./nvme.json Nvme0n1

# 挂载至/mnt/fuse
$ ./test/blobfs/fuse/fuse ./nvme.json Nvme0n1 /mnt/fuse

1
2
3
4
5
6
7
8
9

// spdk/module/blobfs/bdev/blobfs_bdev.c

// 检测bdev设备是否存在blobfs，若存在，则加载blobfs
void spdk_blobfs_bdev_detect()
//
static void spdk_blobfs_bdev_create()

// 挂载
static void spdk_blobfs_bdev_mount(const char* bdev_name, const char *mountpoint, spdk_blobfs_bdev_op_complete cb_fn, void *cb_arg)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

// include 目录
// spdk/include/blobfs.h

// lib 目录
// spdk/lib/blobfs.c

// module目录
// spdk/module/blobfs/bdev/blobfs_bdev.c
// spdk/module/blobfs/bdev/blobfs_bdev_rpc.c
// spdk/module/blobfs/bdev/blobfs_fuse.c
// spdk/module/blobfs/bdev/blobfs_fuse.h

// test目录
// spdk/test/blobfs/blobfs.sh        //blobfs 功能测试启动脚本
// spdk/test/blobfs/fuse/fuse.c
// spdk/test/blobfs/mkfs/mkfs.c

// spdk/test/blobfs/rocksdb/