块存储服务提供对 volume 从创建到删除整个生命周期的管理。从 instance 的角度看,挂载的每一个 Volume 都是一块硬盘。

    OpenStack 提供 Block Storage Service 的是 Cinder,其具体功能是:

        1. 提供 REST API 使用户能够查询和管理 volume、volume snapshot 以及 volume type。

        2. 提供 scheduler 调度 volume 创建请求,合理优化存储资源的分配

        3. 通过 driver 架构支持多种 back-end(后端)存储方式,包括 LVM,NFS,Ceph 和其他诸如 EMC、IBM 等商业存储产品和方案

    这里给出如何对接lvm和nfs存储,后面有文章专门讲如何对接ceph

1.安装块存储服务cinder(控制节点,node1)

---------------------------------------------#初始化数据库,neutron连接mysql用户密码neutron/neutron
mysql -uroot -pmysql << EOF
CREATE DATABASE cinder;
GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'localhost' \
  IDENTIFIED BY 'cinder';
GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'%' \
  IDENTIFIED BY 'cinder';
EOF

---------------------------------------------#创建用户,
. admin-openrc
[root@node1 ~]# openstack user create --domain default --password-prompt cinder # 输入密码cinder
User Password:
Repeat User Password:
+---------------------+----------------------------------+
| Field               | Value                            |
+---------------------+----------------------------------+
| domain_id           | default                          |
| enabled             | True                             |
| id                  | f4fee59fa3d64c23b90b9e269d9ff5e1 |
| name                | cinder                           |
| options             | {}                               |
| password_expires_at | None                             |
+---------------------+----------------------------------+
[root@node1 ~]# openstack role add --project service --user cinder admin
[root@node1 ~]# openstack service create --name cinderv2 --description "OpenStack Block Storage" volumev2
+-------------+----------------------------------+
| Field       | Value                            |
+-------------+----------------------------------+
| description | OpenStack Block Storage          |
| enabled     | True                             |
| id          | c6328632f50d40538ff472b7a2d9184b |
| name        | cinderv2                         |
| type        | volumev2                         |
+-------------+----------------------------------+
[root@node1 ~]# openstack service create --name cinderv3 --description "OpenStack Block Storage" volumev3
+-------------+----------------------------------+
| Field       | Value                            |
+-------------+----------------------------------+
| description | OpenStack Block Storage          |
| enabled     | True                             |
| id          | 2410de3a3dbe45f881f27ac6ba23178e |
| name        | cinderv3                         |
| type        | volumev3                         |
+-------------+----------------------------------+
[root@node1 ~]# openstack endpoint create --region RegionOne volumev2 public http://node1:8776/v2/%\(project_id\)s
+--------------+-------------------------------------+
| Field        | Value                               |
+--------------+-------------------------------------+
| enabled      | True                                |
| id           | b0c56c23b2ec436185d7f0cb25ec415a    |
| interface    | public                              |
| region       | RegionOne                           |
| region_id    | RegionOne                           |
| service_id   | c6328632f50d40538ff472b7a2d9184b    |
| service_name | cinderv2                            |
| service_type | volumev2                            |
| url          | http://node1:8776/v2/%(project_id)s |
+--------------+-------------------------------------+
[root@node1 ~]# openstack endpoint create --region RegionOne volumev2 internal http://node1:8776/v2/%\(project_id\)s
+--------------+-------------------------------------+
| Field        | Value                               |
+--------------+-------------------------------------+
| enabled      | True                                |
| id           | b1c664185c10486b8bb51f640a7ce2cc    |
| interface    | internal                            |
| region       | RegionOne                           |
| region_id    | RegionOne                           |
| service_id   | c6328632f50d40538ff472b7a2d9184b    |
| service_name | cinderv2                            |
| service_type | volumev2                            |
| url          | http://node1:8776/v2/%(project_id)s |
+--------------+-------------------------------------+
[root@node1 ~]# openstack endpoint create --region RegionOne volumev2 admin http://node1:8776/v2/%\(project_id\)s
+--------------+-------------------------------------+
| Field        | Value                               |
+--------------+-------------------------------------+
| enabled      | True                                |
| id           | 09d006e3df00404a9f70b78c7643a24a    |
| interface    | admin                               |
| region       | RegionOne                           |
| region_id    | RegionOne                           |
| service_id   | c6328632f50d40538ff472b7a2d9184b    |
| service_name | cinderv2                            |
| service_type | volumev2                            |
| url          | http://node1:8776/v2/%(project_id)s |
+--------------+-------------------------------------+
[root@node1 ~]# openstack endpoint create --region RegionOne volumev3 public http://node1:8776/v3/%\(project_id\)s
+--------------+-------------------------------------+
| Field        | Value                               |
+--------------+-------------------------------------+
| enabled      | True                                |
| id           | 60778080ebca45ab977472f4c1e01339    |
| interface    | public                              |
| region       | RegionOne                           |
| region_id    | RegionOne                           |
| service_id   | 2410de3a3dbe45f881f27ac6ba23178e    |
| service_name | cinderv3                            |
| service_type | volumev3                            |
| url          | http://node1:8776/v3/%(project_id)s |
+--------------+-------------------------------------+
[root@node1 ~]# openstack endpoint create --region RegionOne volumev3 internal http://node1:8776/v3/%\(project_id\)s
+--------------+-------------------------------------+
| Field        | Value                               |
+--------------+-------------------------------------+
| enabled      | True                                |
| id           | 14ad3059eb8849c6b5c531e8379a660d    |
| interface    | internal                            |
| region       | RegionOne                           |
| region_id    | RegionOne                           |
| service_id   | 2410de3a3dbe45f881f27ac6ba23178e    |
| service_name | cinderv3                            |
| service_type | volumev3                            |
| url          | http://node1:8776/v3/%(project_id)s |
+--------------+-------------------------------------+
[root@node1 ~]# openstack endpoint create --region RegionOne volumev3 admin http://node1:8776/v3/%\(project_id\)s
+--------------+-------------------------------------+
| Field        | Value                               |
+--------------+-------------------------------------+
| enabled      | True                                |
| id           | aa24ffb743124012b3a662ed5a57245e    |
| interface    | admin                               |
| region       | RegionOne                           |
| region_id    | RegionOne                           |
| service_id   | 2410de3a3dbe45f881f27ac6ba23178e    |
| service_name | cinderv3                            |
| service_type | volumev3                            |
| url          | http://node1:8776/v3/%(project_id)s |
+--------------+-------------------------------------+

---------------------------------------------#安装配置组件
yum install -y openstack-cinder
crudini --set /etc/cinder/cinder.conf database connection mysql+pymysql://cinder:cinder@node1/cinder
crudini --set /etc/cinder/cinder.conf DEFAULT transport_url rabbit://openstack:openstack@node1
crudini --set /etc/cinder/cinder.conf DEFAULT auth_strategy keystone
crudini --set /etc/cinder/cinder.conf DEFAULT my_ip 192.168.31.101
crudini --set /etc/cinder/cinder.conf keystone_authtoken www_authenticate_uri http://node1:5000
crudini --set /etc/cinder/cinder.conf keystone_authtoken auth_url http://node1:5000
crudini --set /etc/cinder/cinder.conf keystone_authtoken memcached_servers node1:11211
crudini --set /etc/cinder/cinder.conf keystone_authtoken auth_type password
crudini --set /etc/cinder/cinder.conf keystone_authtoken project_domain_name default
crudini --set /etc/cinder/cinder.conf keystone_authtoken user_domain_name default
crudini --set /etc/cinder/cinder.conf keystone_authtoken project_name service
crudini --set /etc/cinder/cinder.conf keystone_authtoken username cinder
crudini --set /etc/cinder/cinder.conf keystone_authtoken password cinder
crudini --set /etc/cinder/cinder.conf oslo_concurrency lock_path /var/lib/cinder/tmp

---------------------------------------------# 初始化数据
su -s /bin/sh -c "cinder-manage db sync" cinder

---------------------------------------------# 
crudini --set /etc/nova/nova.conf cinder os_region_name RegionOne

---------------------------------------------# 启动服务
systemctl restart openstack-nova-api.service
systemctl enable openstack-cinder-api.service openstack-cinder-scheduler.service
systemctl start openstack-cinder-api.service openstack-cinder-scheduler.service

2.安装块存储服务cinder(存储节点,node4/node5)

---------------------------------------------# 安装配置存储组件,在node4和node5上操作,用于提供lvm后端存储
yum -y install lvm2 device-mapper-persistent-data
# 将两个200G的硬盘创建pv,vg
pvcreate /dev/sdb
pvcreate /dev/sdc
vgcreate cinder-volumes /dev/sdb /dev/sdc
# 修改配置文件
vim /etc/lvm/lvm.conf
 devices {
...
filter = [ "a/sdb/", "a/sdc/", "r/.*/"]

 ---------------------------------------------#安装配置组件,在node4和node5上操作
yum install -y openstack-cinder targetcli python3-keystone
crudini --set /etc/cinder/cinder.conf database connection mysql+pymysql://cinder:cinder@node1/cinder
crudini --set /etc/cinder/cinder.conf DEFAULT transport_url rabbit://openstack:openstack@node1
crudini --set /etc/cinder/cinder.conf DEFAULT auth_strategy keystone
crudini --set /etc/cinder/cinder.conf DEFAULT my_ip 192.168.31.104     # 主机IP,192.168.31.104,105
crudini --set /etc/cinder/cinder.conf DEFAULT enabled_backends lvm,nfs  # 这里对接lvm和nfs后端存储,后面有文章专门讲如何对接ceph
crudini --set /etc/cinder/cinder.conf DEFAULT glance_api_servers http://node1:9292
crudini --set /etc/cinder/cinder.conf www_authenticate_uri http://node1:5000
crudini --set /etc/cinder/cinder.conf auth_url http://node1:5000
crudini --set /etc/cinder/cinder.conf memcached_servers node1:11211
crudini --set /etc/cinder/cinder.conf auth_type password
crudini --set /etc/cinder/cinder.conf project_domain_name default
crudini --set /etc/cinder/cinder.conf user_domain_name default
crudini --set /etc/cinder/cinder.conf project_name service
crudini --set /etc/cinder/cinder.conf username cinder
crudini --set /etc/cinder/cinder.conf password cinder
crudini --set /etc/cinder/cinder.conf lvm volume_driver cinder.volume.drivers.lvm.LVMVolumeDriver
crudini --set /etc/cinder/cinder.conf lvm volume_group cinder-volumes
crudini --set /etc/cinder/cinder.conf lvm target_protocol iscsi
crudini --set /etc/cinder/cinder.conf lvm target_helper lioadm
crudini --set /etc/cinder/cinder.conf lvm volume_backend_name lvm
crudini --set /etc/cinder/cinder.conf nfs volume_driver cinder.volume.drivers.nfs.NfsDriver
crudini --set /etc/cinder/cinder.conf nfs nfs_shares_config /etc/cinder/nfs_shares
crudini --set /etc/cinder/cinder.conf nfs volume_backend_name nfs
crudini --set /etc/cinder/cinder.conf nfs nfs_mount_point_base \$state_path/mnt
crudini --set /etc/cinder/cinder.conf nfs nfs_snapshot_support True   # 好像不好使,nfs类型的卷还是无法创建快照
crudini --set /etc/cinder/cinder.conf nfs nas_secure_file_operations false # 好像不好使
crudini --set /etc/cinder/cinder.conf nfs nas_secure_file_permissions false  # 好像不好使
crudini --set /etc/cinder/cinder.conf oslo_concurrency lock_path /var/lib/cinder/tmp

---设置nfs作为后端存储的配置
echo "192.168.31.101:/vmdata" >>  /etc/cinder/nfs_shares
chown root.cinder /etc/cinder/nfs_shares
chmod 640 /etc/cinder/nfs_shares

--- 控制节点node1执行,创建卷类型lvm和nfs,默认类型为__DEFAULT__会自动选择lvm
cinder type-create lvm
cinder type-create nfs 
cinder type-key lvm set volume_backend_name=lvm
cinder type-key nfs set volume_backend_name=nfs

---------------------------------------------# 启动服务,在node4和node5上操作
systemctl enable openstack-cinder-volume.service target.service
systemctl start openstack-cinder-volume.service target.service

---------------------------------------------# 验证,在node1上操作
[root@node1 ~]# . admin-openrc
[root@node1 ~]# openstack volume service list
+------------------+-----------+------+---------+-------+----------------------------+
| Binary           | Host      | Zone | Status  | State | Updated At                 |
+------------------+-----------+------+---------+-------+----------------------------+
| cinder-scheduler | node1     | nova | enabled | up    | 2022-03-01T09:18:46.000000 |
| cinder-volume    | node5@lvm | nova | enabled | up    | 2022-03-01T09:18:44.000000 |
| cinder-volume    | node5@nfs | nova | enabled | up    | 2022-03-01T09:18:48.000000 |
| cinder-volume    | node4@lvm | nova | enabled | up    | 2022-03-01T09:18:48.000000 |
| cinder-volume    | node4@nfs | nova | enabled | up    | 2022-03-01T09:18:43.000000 |
+------------------+-----------+------+---------+-------+----------------------------+
[root@node1 ~]# openstack volume create --size 1 volume1
+---------------------+--------------------------------------+
| Field               | Value                                |
+---------------------+--------------------------------------+
| attachments         | []                                   |
| availability_zone   | nova                                 |
| bootable            | false                                |
| consistencygroup_id | None                                 |
| created_at          | 2022-03-01T09:19:02.000000           |
| description         | None                                 |
| encrypted           | False                                |
| id                  | faf6fdba-d607-4765-8e42-5306ad49bbcc |
| migration_status    | None                                 |
| multiattach         | False                                |
| name                | volume1                              |
| properties          |                                      |
| replication_status  | None                                 |
| size                | 1                                    |
| snapshot_id         | None                                 |
| source_volid        | None                                 |
| status              | creating                             |
| type                | __DEFAULT__                          |
| updated_at          | None                                 |
| user_id             | 5a44718261844cbd8a65621b9e3cea8d     |
+---------------------+--------------------------------------+
[root@node1 ~]# openstack volume list
+--------------------------------------+---------+-----------+------+-------------+
| ID                                   | Name    | Status    | Size | Attached to |
+--------------------------------------+---------+-----------+------+-------------+
| faf6fdba-d607-4765-8e42-5306ad49bbcc | volume1 | available |    1 |             |
+--------------------------------------+---------+-----------+------+-------------+