本节主要是将存储设备加入ceph进行管理，并测试ceph的三种存储方式之一cephfs。
1、查看设备状态
sudo ceph osd status ID HOST USED AVAIL WR OPS WR DATA RD OPS RD DATA STATE 0 ceph01 1027M 298G 0 0 0 0 exists,up 1 ceph02 1027M 298G 0 0 0 0 exists,up 2 ceph03 1027M 298G 0 0 0 0 exists,up 3 ceph04 1027M 298G 0 0 0 0 exists,up sudo ceph orch device ls HOST PATH TYPE SIZE DEVICE AVAIL REJECT REASONS ceph01 /dev/sdb hdd 300G VBOX_HARDDISK_VB434b1565-528a303a True ceph01 /dev/sda hdd 300G VBOX_HARDDISK_VB3eec2162-9aed4ffc False locked ceph02 /dev/sdb hdd 300G VBOX_HARDDISK_VBa6445865-c497aa8e True ceph02 /dev/sda hdd 300G VBOX_HARDDISK_VB64e04201-60c7209f False locked ceph03 /dev/sdb hdd 300G VBOX_HARDDISK_VB20fd0c04-b14ef3fa True ceph03 /dev/sda hdd 300G VBOX_HARDDISK_VB6f4439ab-85f80c78 False locked ceph04 /dev/sdb hdd 300G VBOX_HARDDISK_VB2c293541-3183e992 True ceph04 /dev/sda hdd 300G VBOX_HARDDISK_VBd81d45d4-a88d6ff3 False locked
2、存储设备加入osd管理
sudo ceph orch apply osd --all-available-devices Scheduled osd update... sudo ceph orch device ls --refresh HOST PATH TYPE SIZE DEVICE AVAIL REJECT REASONS ceph01 /dev/sda hdd 300G VBOX_HARDDISK_VB3eec2162-9aed4ffc False locked ceph01 /dev/sdb hdd 300G VBOX HARDDISK_VB434b1565-528a303a False LVM detected, Insufficient space (<5GB) on vgs, locked ceph02 /dev/sda hdd 300G VBOX_HARDDISK_VB64e04201-60c7209f False locked ceph02 /dev/sdb hdd 300G VBOX HARDDISK_VBa6445865-c497aa8e False LVM detected, locked, Insufficient space (<5GB) on vgs ceph03 /dev/sda hdd 300G VBOX_HARDDISK_VB6f4439ab-85f80c78 False locked ceph03 /dev/sdb hdd 300G VBOX HARDDISK_VB20fd0c04-b14ef3fa False locked, Insufficient space (<5GB) on vgs, LVM detected ceph04 /dev/sda hdd 300G VBOX_HARDDISK_VBd81d45d4-a88d6ff3 False locked ceph04 /dev/sdb hdd 300G VBOX HARDDISK_VB2c293541-3183e992 False LVM detected, locked, Insufficient space (<5GB) on vgs
3、查看osd状态
sudo ceph osd df ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS 0 hdd 0.29300 1.00000 300 GiB 1.0 GiB 3.4 MiB 0 B 1 GiB 299 GiB 0.33 1.00 0 up 1 hdd 0.29300 1.00000 300 GiB 1.0 GiB 3.4 MiB 0 B 1 GiB 299 GiB 0.33 1.00 1 up 2 hdd 0.29300 1.00000 300 GiB 1.0 GiB 3.4 MiB 0 B 1 GiB 299 GiB 0.33 1.00 1 up 3 hdd 0.29300 1.00000 300 GiB 1.0 GiB 3.4 MiB 0 B 1 GiB 299 GiB 0.33 1.00 1 up TOTAL 1.2 TiB 4.0 GiB 14 MiB 0 B 4 GiB 1.2 TiB 0.33 MIN/MAX VAR: 1.00/1.00 STDDEV: 0 sudo ceph osd utilization avg 0.75 stddev 0.433013 (expected baseline 0.75) min osd.0 with 0 pgs (0 * mean) max osd.1 with 1 pgs (1.33333 * mean) neohope@ceph01:~$ sudo ceph osd pool stats pool device_health_metrics id 1 nothing is going on sudo ceph osd tree ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF -1 1.17200 root default -3 0.29300 host ceph01 0 hdd 0.29300 osd.0 up 1.00000 1.00000 -5 0.29300 host ceph02 1 hdd 0.29300 osd.1 up 1.00000 1.00000 -7 0.29300 host ceph03 2 hdd 0.29300 osd.2 up 1.00000 1.00000 -9 0.29300 host ceph04 3 hdd 0.29300 osd.3 up 1.00000 1.00000 sudo ceph pg stat 1 pgs: 1 active+clean; 0 B data, 14 MiB used, 1.2 TiB / 1.2 TiB avail
4、创建cephfs
sudo ceph fs volume ls
[]
sudo ceph fs volume create v1
sudo ceph fs volume ls
[
{
"name": "v1"
}
]
sudo ceph fs subvolumegroup create v1 g1
sudo ceph fs subvolumegroup ls v1
sudo ceph fs subvolume create v1 sv1
sudo ceph fs subvolume ls v1
sudo ceph fs ls
name: v1, metadata pool: cephfs.v1.meta, data pools: [cephfs.v1.data ]
5、挂载cephfs
sudo apt-get install ceph-fuse # 挂载cephfs sudo mkdir -p /mnt/ceph/ceph_fuse sudo ceph-fuse /mnt/ceph/ceph_fuse ceph-fuse[24512]: starting ceph client 2020-05-18 05:57:36.039818 7f7d221a2500 -1 init, newargv = 0x559708e0e2e0 newarg c=9 ceph-fuse[24512]: starting fuse # 查看挂载情况 sudo mount | grep ceph ceph-fuse on /mnt/ceph/ceph_fuse type fuse.ceph-fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other) # 可以当作本地磁盘做一些日常操作 sudo ls /mnt/ceph/ceph_fuse/ volumes sudo ls /mnt/ceph/ceph_fuse/volumes g1 _nogroup sudo ls /mnt/ceph/ceph_fuse/volumes/g1 sudo vi /mnt/ceph/ceph_fuse/volumes/g1/hi.txt sudo cat /mnt/ceph/ceph_fuse/volumes/g1/hi.txt hello ceph fuse