菜鸟小站-钱锅锅【BD4RPX】

HDFS 不适合低延时的数据访问方式，不适合大量小文件、不适合多方读写，需要任意的文件修改，不支持多个机器同时写入。

HBase 适合低延时的数据访问方式。

ceph 支持文件存储、块存储、对象存储。 支持cephfs rbd

swift 在swift对象存储中，客户端需要联系swift网关，这会带来一些潜在的单点故障。

持续更新中。。。。。。

###############centos7

mkdir my-cluster

cd my-cluster

yum install ceph-deploy -y

ceph-deploy new mon01

yum install python-minimal -y

ceph-deploy install mon01 node01 node02

MDS=”mon01″

MON=”mon01 node01 node02″

OSDS=”mon01 node01 node02″

INST=”$OSDS $MON”

echo “osd pool default size = 2

osd max object name len = 256

osd max object namespace len = 64

mon_pg_warn_max_per_osd = 2000

mon clock drift allowed = 30

mon clock drift warn backoff = 30

rbd cache writethrough until flush = false” >> ceph.conf

apt-get install -y ceph-base

apt-get install -y ceph-common

apt-get install -y ceph-fs-common

apt-get install -y ceph-fuse

apt-get install -y ceph-mds

apt-get install -y ceph-mon

apt-get install -y ceph-osd

[Ceph]

name=Ceph packages for $basearch

baseurl=http://mirrors.163.com/ceph/rpm-nautilus/el7/$basearch

enabled=1

priority=1

gpgcheck=1

gpgkey=https://download.ceph.com/keys/release.asc

[Ceph-noarch]

name=Ceph noarch packages

baseurl=http://mirrors.163.com/ceph/rpm-nautilus/el7/noarch

enabled=1

priority=1

gpgcheck=1

gpgkey=https://download.ceph.com/keys/release.asc

[ceph-source]

name=Ceph source packages

baseurl=http://mirrors.163.com/ceph/rpm-nautilus/el7/SRPMS

enabled=0

priority=1

gpgcheck=1

gpgkey=https://download.ceph.com/keys/release.asc

yum install python-setuptools

yum install -y deltarpm

yum install -y gdisk

ceph mgr module disable dashboard

ceph-deploy mgr create mon01 node01 node02

ceph dashboard ac-user-create admin passw0rd administrator

systemctl restart ceph-mon.target

ceph-deploy –overwrite-conf mds create mon01 node01 node02 创建mds

ceph osd pool create cephfs_data 128

ceph osd pool create cephfs_metadata 128

ceph fs new myfs cephfs_metadata cephfs_data

ceph osd pool rm cephfs_data cephfs_data –yes-i-really-really-mean-it 删除cephfs_data

ceph fs rm myfs –yes-i-really-mean-it 删除myfs

mount -t ceph 192.168.169.190:6789:/ /fsdata -o name=admin,secretfile=/etc/ceph/admin.secret

##ceph添加新节点

ceph-deploy –overwrite-conf config push admin mon01 node01 node02 node03

ceph-deploy –overwrite-conf mon create node03

ceph-deploy –overwrite-conf mon add node03

ceph-deploy osd create –data /dev/sdb node03

#######rbd链接方式

ceph osd pool create rbd_test 128 128

rbd create rbd_date –size 20480 -p rbd_test

rbd –image rbd_date -p rbd_test info

rbd feature disable rbd_test/rbd_date object-map fast-diff deep-flatten

rbd map rbd_date -p rbd_test

rbd showmapped

ceph osd pool application enable rbd_test rbd_date

mkfs.xfs /dev/rbd0

mkdir /rbddate

mount /dev/rbd0 /rbddate/

dd if=/dev/zero of=/rbddate/10G bs=1M count=10240

#cloudstack挂载rbd存储

ceph auth get-or-create client.cloudstack mon ‘allow r’ osd ‘allow rwx pool=vm-data’

AQD+VQVfELMbJRAA5LspVxtCykwJ3LFzwYLyFQ==

####删除RBD

rbd list -p rbd_test

rbd unmap /dev/rbd0

rbd rm rbd_date -p rbd_test

rbd list -p rbd_test 查看镜像

rbd snap ls rbd_test/dfe36912-ba7f-11ea-a837-000c297bc10e 查看镜像的快照

rbd snap unprotect rbd_test/dfe36912-ba7f-11ea-a837-000c297bc10e@cloudstack-base-snap 解除快照保护

rbd snap purge rbd_test/ede76ccf-f86a-4ab7-afa7-1adc4f1b576b 删除快照

rbd rm rbd_test/ede76ccf-f86a-4ab7-afa7-1adc4f1b576b 删除镜像

rbd children vm-data/2368966f-0ea3-11eb-8538-3448edf6aa08@cloudstack-base-snap 查看子快照

rbd flatten vm-data/79900df4-0b18-42cb-854b-c29778f02aff 还原子快照

问题：This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout.

解决：

rbd status vm-data/6926af02-27c3-47ad-a7ee-86c7d95aa353 查看戳号

ceph osd blacklist add 172.31.156.11:0/4126702798

查看rbd 残留的watch信息

[root@node-2 ~]# rbd status compute/2d05517a-8670-4cce-b39d-709e055381d6_disk Watchers: watcher=192.168.55.2:0/2900899764 client.14844 cookie=139644428642944

将该残留的watch信息添加到osd的黑名单，再查看watch是否存在。

[root@node-2 ~]# ceph osd blacklist add 192.168.55.2:0/2900899764 blacklisting 192.168.55.2:0/2900899764 until 2018-06-11 14:25:31.027420 (3600 sec) [root@node-2 ~]# rbd status compute/2d05517a-8670-4cce-b39d-709e055381d6_disk Watchers: none

删除rbd

[root@node-2 ~]# rbd rm compute/2d05517a-8670-4cce-b39d-709e055381d6_disk Removing image: 100% complete…done.

docker run -d –name minio \

–restart=always –net=host \

-e MINIO_ACCESS_KEY=admin \

-e MINIO_SECRET_KEY=passw0rd \

-v /data:/data \

minio/minio server \

http://oss-{01…04}/data/

########客户端实时同步脚本如下：########

#修改成你需要实时同步备份的文件夹

backup=”/backup/filebackup”

#修改成你要备份到的存储桶

bucket=”minio”

#将以下代码一起复制到SSH运行

cat > /etc/systemd/system/miniocfile.service <

[Unit]

Description=miniocfile

After=network.target

[Service]

Type=simple

ExecStart=$(command -v mc) mirror -w –overwrite ${backup} ${bucket}/${backup}

Restart=on-failure

[Install]

WantedBy=multi-user.target

EOF

#######################################

1、systemctl stop ceph-osd@21

2、ceph osd out osd.21

3、ceph osd crush remove osd.21

4、ceph auth del osd.21

5、ceph osd rm 21

6、cd /var/lib/ceph/osd/ceph-xx #查找相应的OSD对应的/dev是什么。

7、umount /dev/sdj

8、换硬盘

smartctl –all /dev/sde ########识别硬盘SN号

9、登录到c01

10、cd /root/my-cluster

11、ceph-deploy osd create –data /dev/sdj c02