1.部署mds到k-master節(jié)點
[root@k-master ceph]# pwd
/etc/ceph
[root@k-master ceph]# ls
ceph.bootstrap-mds.keyring? ceph.bootstrap-osd.keyring? ceph.client.admin.keyring? ceph-deploy-ceph.log? etc? ? s3test.py? ? ? ? testrbd
ceph.bootstrap-mgr.keyring? ceph.bootstrap-rgw.keyring? ceph.conf? ? ? ? ? ? ? ? ? ceph.mon.keyring? ? ? rbdmap? swift_openrc.sh
[root@k-master ceph]# ceph-deploy? mds create k-master
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO? ] Invoked (2.0.1): /usr/bin/ceph-deploy mds create k-master
[ceph_deploy.cli][INFO? ] ceph-deploy options:
[ceph_deploy.cli][INFO? ]? username? ? ? ? ? ? ? ? ? ? ? : None
[ceph_deploy.cli][INFO? ]? verbose? ? ? ? ? ? ? ? ? ? ? : False
[ceph_deploy.cli][INFO? ]? overwrite_conf? ? ? ? ? ? ? ? : False
[ceph_deploy.cli][INFO? ]? subcommand? ? ? ? ? ? ? ? ? ? : create
[ceph_deploy.cli][INFO? ]? quiet? ? ? ? ? ? ? ? ? ? ? ? : False
[ceph_deploy.cli][INFO? ]? cd_conf? ? ? ? ? ? ? ? ? ? ? : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f47093418c0>
[ceph_deploy.cli][INFO? ]? cluster? ? ? ? ? ? ? ? ? ? ? : ceph
[ceph_deploy.cli][INFO? ]? func? ? ? ? ? ? ? ? ? ? ? ? ? : <function mds at 0x7f470937ced8>
[ceph_deploy.cli][INFO? ]? ceph_conf? ? ? ? ? ? ? ? ? ? : None
[ceph_deploy.cli][INFO? ]? mds? ? ? ? ? ? ? ? ? ? ? ? ? : [('k-master', 'k-master')]
[ceph_deploy.cli][INFO? ]? default_release? ? ? ? ? ? ? : False
[ceph_deploy.mds][DEBUG ] Deploying mds, cluster ceph hosts k-master:k-master
[k-master][DEBUG ] connected to host: k-master
[k-master][DEBUG ] detect platform information from remote host
[k-master][DEBUG ] detect machine type
[ceph_deploy.mds][INFO? ] Distro info: CentOS Linux 7.7.1908 Core
[ceph_deploy.mds][DEBUG ] remote host will use systemd
[ceph_deploy.mds][DEBUG ] deploying mds bootstrap to k-master
[k-master][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[k-master][WARNIN] mds keyring does not exist yet, creating one
[k-master][DEBUG ] create a keyring file
[k-master][DEBUG ] create path if it doesn't exist
[k-master][INFO? ] Running command: ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.k-master osd allow rwx mds allow mon allow profile mds -o /var/lib/ceph/mds/ceph-k-master/keyring
[k-master][INFO? ] Running command: systemctl enable ceph-mds@k-master
[k-master][WARNIN] Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@k-master.service to /usr/lib/systemd/system/ceph-mds@.service.
[k-master][INFO? ] Running command: systemctl start ceph-mds@k-master
[k-master][INFO? ] Running command: systemctl enable ceph.target
[root@k-master ceph]#
[root@k-master ceph]# ps -ef | grep ceph
ceph? ? ? ? 2194? ? ? 1? 0 Jul26 ?? ? ? ? 00:17:56 /usr/bin/ceph-osd -f --cluster ceph --id 0 --setuser ceph --setgroup ceph
ceph? ? ? ? 2638? ? ? 1? 0 Jul26 ?? ? ? ? 00:19:42 /usr/bin/ceph-osd -f --cluster ceph --id 1 --setuser ceph --setgroup ceph
ceph? ? ? ? 3094? ? ? 1? 2 Jul26 ?? ? ? ? 00:50:43 /usr/bin/ceph-mgr -f --cluster ceph --id k-master --setuser ceph --setgroup ceph
root? ? ? 12196? ? ? 2? 0 Jul26 ?? ? ? ? 00:00:00 [ceph-msgr]
root? ? ? 21112? ? ? 2? 0 Jul26 ?? ? ? ? 00:00:00 [ceph-watch-noti]
ceph? ? ? 55743? ? ? 1? 0 Jul27 ?? ? ? ? 00:03:50 /usr/bin/ceph-mon -f --cluster ceph --id k-master --setuser ceph --setgroup ceph
ceph? ? ? 83272? ? ? 1? 0 03:51 ?? ? ? ? 00:01:22 /usr/bin/radosgw -f --cluster ceph --name client.rgw.k-master --setuser ceph --setgroup ceph
ceph? ? ? 93467? ? ? 1? 0 08:16 ?? ? ? ? 00:00:00 /usr/bin/ceph-mds -f --cluster ceph --id k-master --setuser ceph --setgroup ceph
root? ? ? 93551? 64987? 0 08:17 pts/0? ? 00:00:00 grep --color=auto ceph
[root@k-master ceph]#
2.創(chuàng)建兩個pool(元數(shù)據(jù)與數(shù)據(jù))
[root@k-master ceph]# ceph osd pool create cephfs_metadata 16 16
pool 'cephfs_metadata' created
[root@k-master ceph]# ceph osd pool create cephfs_data 16 16
pool 'cephfs_data' created
[root@k-master ceph]# ceph -s
? cluster:
? ? id:? ? 4221c53f-a286-4b7f-b20d-3bd453eb841c
? ? health: HEALTH_OK
? services:
? ? mon: 1 daemons, quorum k-master
? ? mgr: k-master(active)
? ? osd: 3 osds: 3 up, 3 in
? ? rgw: 1 daemon active
? data:
? ? pools:? 14 pools, 292 pgs
? ? objects: 275 objects, 85.5MiB
? ? usage:? 3.19GiB used, 11.8GiB / 15.0GiB avail
? ? pgs:? ? 292 active+clean
? io:
? ? client:? 0B/s rd, 0op/s rd, 0op/s wr
[root@k-master ceph]# ceph osd lspools
1 rbd,3 ceph,4 k8s,6 ceph-demo,7 ceph-tt,8 rbd-test,9 .rgw.root,10 default.rgw.control,11 default.rgw.meta,12 default.rgw.log,13 default.rgw.buckets.index,14 default.rgw.buckets.data,15 cephfs_metadata,16 cephfs_data,
[root@k-master ceph]#
3.創(chuàng)建文件系統(tǒng)
[root@k-master ceph]# ceph fs new cephfs-demo cephfs_metadata cephfs_data
new fs with metadata pool 15 and data pool 16
[root@k-master ceph]# ceph fs ls
name: cephfs-demo, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
[root@k-master ceph]# ceph -s
? cluster:
? ? id:? ? 4221c53f-a286-4b7f-b20d-3bd453eb841c
? ? health: HEALTH_OK
? services:
? ? mon: 1 daemons, quorum k-master
? ? mgr: k-master(active)
? ? mds: cephfs-demo-1/1/1 up? {0=k-worker001=up:active}, 1 up:standby
? ? osd: 3 osds: 3 up, 3 in
? ? rgw: 1 daemon active
? data:
? ? pools:? 14 pools, 292 pgs
? ? objects: 296 objects, 85.5MiB
? ? usage:? 3.20GiB used, 11.8GiB / 15.0GiB avail
? ? pgs:? ? 292 active+clean
? io:
? ? client:? 0B/s rd, 0op/s rd, 0op/s wr
[root@k-master ceph]#
4.掛載測試
[1]內(nèi)核驅(qū)動型(性能高)
參閱：https://ceph.readthedocs.io/en/latest/cephfs/mount-using-kernel-driver/
[root@k-master ceph]# mount -t ceph k-master:/ /mnt/cephfs/
mount error 22 = Invalid argument
[root@k-master ceph]# cat ./ceph.client.admin.keyring
[client.admin]
? ? ? ? key = AQAwmhpf7pEnOBAAXK4KC/J2YbVPDyNgfdFrdg==
[root@k-master ceph]# vim ~/admin.keyring
[root@k-master ceph]# cat ~/admin.keyring
AQAwmhpf7pEnOBAAXK4KC/J2YbVPDyNgfdFrdg==
[root@k-master ceph]# mount -t ceph k-master:6789:/ /mnt/cephfs -o name=admin,secretfile=~/admin.keyring
unable to read secretfile: No such file or directory
error reading secret file
failed to parse ceph_options
[root@k-master ceph]# ll /root/admin.keyring
-rw-r--r-- 1 root root 41 Jul 28 08:47 /root/admin.keyring
[root@k-master ceph]# mount -t ceph k-master:6789:/ /mnt/cephfs -o name=admin,secretfile=/root/admin.keyring
[root@k-master ceph]# mount | grep /mnt/cephfs
192.168.43.60:6789:/ on /mnt/cephfs type ceph (rw,relatime,name=admin,secret=<hidden>,acl,wsize=16777216)
[root@k-master ceph]#
[root@k-master ceph]# df -h | grep /mnt/cephfs
192.168.43.60:6789:/? ? 5.5G? ? 0? 5.5G? 0% /mnt/cephfs
[root@k-master ceph]# cd /mnt/cephfs/
[root@k-master cephfs]# touch {1..10}.log
[root@k-master cephfs]# ls
10.log? 1.log? 2.log? 3.log? 4.log? 5.log? 6.log? 7.log? 8.log? 9.log
[root@k-master cephfs]#
[root@k-master cephfs]# lsmod | grep ceph
ceph? ? ? ? ? ? ? ? ? 358802? 1
libceph? ? ? ? ? ? ? 306742? 2 rbd,ceph
dns_resolver? ? ? ? ? 13140? 1 libceph
libcrc32c? ? ? ? ? ? ? 12644? 2 xfs,libceph
[root@k-master cephfs]#
[2]使用ceph-fuse掛載
[root@k-master cephfs]# yum install ceph-fuse -y
[root@k-master ~]# mkdir /mnt/ceph-fuse
[root@k-master ~]# ceph-fuse -n client.admin -m k-master:6789 /mnt/ceph-fuse/
2020-07-28 09:49:28.465970 7fc655159240 -1 init, newargv = 0x5584e03c9380 newargc=9
ceph-fuse[97512]: starting ceph client
ceph-fuse[97512]: starting fuse
[root@k-master ~]# mount | grep '/mnt/ceph-fuse'
ceph-fuse on /mnt/ceph-fuse type fuse.ceph-fuse (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
[root@k-master ~]# df -h | grep mnt
ceph-fuse? ? ? ? ? ? ? ? 5.5G? ? 0? 5.5G? 0% /mnt/ceph-fuse
[root@k-master ~]#
[root@k-master ~]# cd /mnt/ceph-fuse
[root@k-master ceph-fuse]# ls
10.log? 1.log? 2.log? 3.log? 4.log? 5.log? 6.log? 7.log? 8.log? 9.log
[root@k-master ceph-fuse]# touch {a..z}.log
[root@k-master ceph-fuse]# ls
10.log? 2.log? 4.log? 6.log? 8.log? a.log? c.log? e.log? g.log? i.log? k.log? m.log? o.log? q.log? s.log? u.log? w.log? y.log
1.log? 3.log? 5.log? 7.log? 9.log? b.log? d.log? f.log? h.log? j.log? l.log? n.log? p.log? r.log? t.log? v.log? x.log? z.log
[root@k-master ceph-fuse]#
