环境:Centos 5.8 64bit ,iptables 关闭,selinux关闭,开发包组库与工具已经安装,gcc版本4.1.2
  1. ns1 192.168.10.11

  2. ns2 192.168.10.12

  3. ds1 192.168.10.13

  4. ds2 192.168.10.14

  5. vip 192.168.19.10

ds1,ds2为dataserver,ns1,ns2为namserver,ns1为master,ns2为backup,其中ds1与ds2共4块盘 sda sdb sdc sdd,sda为系统所在盘,sdb,sdc,sdd为待用盘,它们会被格式化为ext4挂载
ns1操作
一.编译安装tfs
1.1 安装淘宝底层公共库tbsys和tbnet
  1. # svn checkout -r 15 http://code.taobao.org/svn/tb-common-utils/trunk/ tb-common-utils  ##最新版本有问题

  2. # echo 'export TBLIB_ROOT="/usr/local/tb_lib"' >> /etc/profile ##生明环境变量,安装库会使用该变量,将来tfs会根椐该变量来调用库

  3. # . /etc/profile  ##生效配置


  4. # cd tb-common-utils/

  5. # chmod +x build.sh

  6. # ./build.sh  编译安装  


1.2 安装gtest库,tfs依赖
  1. # wget http://googletest.googlecode.com/files/gtest-1.5.0.tar.bz2

  2. # tar xvf gtest-1.5.0.tar.bz2

  3. # cd gtest-1.5.0

  4. # ./configure

  5. # make && make install

1.3 安装mysql-devel等包组,要求>5.1.28,我的云盘下载如下,也可以用编计的
http://pan.baidu.com/share/link?shareid=444580&uk=2501966565#dir/path=%2FLinux%2Fmysql
  1. # rpm -ivh MySQL-*  

  2. # service mysql stop   ##安装完毕后mysql会自动启动

  3. # chkconfig mysql off

  4. # cd /usr/lib64/mysql/

  5. # ln -s libmysqlclient.a libmysqlclient.so  ##否则编译tfs会报错

  6. # ln -s libmysqlclient_r.a libmysqlclient_r.so

1.4 编译安装tfs
  1. # svn checkout http://code.taobao.org/svn/tfs/tags/release-2.2.11 tfs-2.2.11   ##获取2.2.11版本的

  2. # cd tfs-2.2.11/

  3. # ./build.sh init   ##初始化

  4. # ./configure --prefix=/usr/local/tfs --without-tcmalloc    

  5. ## 如果mysql是编译的添加--with-mysql 选项后面是mysql安装目录bin下的mysql_config

  6. ##安装到/usr/local/tfs下,取消tcmalloc这个特性否则会报错

  7. # cp conf/ns.conf /usr/local/tfs/conf/   ##ns1,ns2拷贝ns.conf

  8. # cp conf/{ds.conf,ads.conf} /usr/local/tfs/conf  ## ds1,ds2拷贝这两个


如果不出错,tfs安装成功,ns2,ds1,ds2同样操作
ds1,ds2上操作
二.将sdb,sdc,sdd格式化为ext4
  1. # modprobe ext4  ##加载ext4的模块

  2. # yum -y install e4fsprogs e4fsprogs-devel   ##安装格式化软件  

  3. # mkfs.ext4 /dev/sdb;mkfs.ext4 /dev/sdc;mkfs.ext4 /dev/sdd  ##格式化

  4. # mkdir -p /data/{tfs1,tfs2,tfs3}  

  5. # mount /dev/sdb /data/tfs1;mount /dev/sdc /data/tfs2;mount /dev/sdd /data/tfs3  ##挂载

三.配置ns1与ns2
  1. # cd /usr/local/tfs

  2. # vim conf/ns.conf

  1. [public]

  2. log_size=1073741824

  3. log_num = 64

  4. log_level=debug

  5. task_max_queue_size = 10240

  6. port = 8108                  ##监听端口号,重要!dataserver配置需要

  7. work_dir=/usr/local/tfs      ##工作目录,也就是tfs的安装目录

  8. dev_name= eth0               ##网卡设备,我的是eth0

  9. thread_count = 4             ##工作线程

  10. ip_addr = 192.168.10.11      ##如果是ha写vip,如果是主备或单namserver,写主nameserver的ip

  11. [nameserver]

  12. safe_mode_time = 300

  13. ip_addr_list = 192.168.10.11|192.168.10.12  ##前面是主nameserver的ip后是备nameserver的

  14. group_mask = 255.255.255.255

  15. block_max_size = 83886080

  16. max_replication = 2          ##最大复制份数,如果你就一个ds,则为1

  17. min_replication = 2          ##最小复制份数

  18. use_capacity_ratio = 98

  19. block_max_use_ratio = 98

  20. heart_interval = 2

  21. object_dead_max_time = 3600

  22. cluster_id = 1

  23. replicate_ratio_ = 50

  24. max_write_filecount = 16

  25. heart_thread_count = 2

  26. heart_max_queue_size = 10

  27. repl_max_time = 60

  28. compact_delete_ratio =  15

  29. compact_max_load = 200

  30. object_dead_max_time = 86400

  31. object_clear_max_time = 300

  32. max_wait_write_lease = 15

  33. lease_expired_time = 3

  34. max_lease_timeout = 3000

  35. cleanup_lease_threshold = 102400

  36. build_plan_interval = 10

  37. run_plan_expire_interval = 120

  38. build_plan_ratio = 25

  39. dump_stat_info_interval = 60000000

  40. build_plan_default_wait_time = 2

  41. balance_max_diff_block_num = 5

  42. add_primary_block_count = 3

  43. block_chunk_num = 32

  44. task_percent_sec_size = 200

  45. task_max_queue_size = 10000

  46. oplog_sync_max_slots_num = 1024

  47. oplog_sync_thread_num = 1

##详细解释见http://code.taobao.org/p/tfs/wiki/deploy/ns.conf/或http://250688049.blog.51cto.com/643101/849903
主备的配置文件先一致,我们一会儿会配置ha,先测试主备模式namaserver是否正常
四.ds1,ds2设置
# cd /usr/local/tfs/
4.1 修改ds1的配置文件ds.conf
# vim conf/ds.conf
  1. [public]

  2. log_size=1073741824

  3. log_num = 64

  4. log_level=error

  5. task_max_queue_size = 10240

  6. port = 9998                            #设置dataserver监听端口号

  7. work_dir = /usr/local/tfs              #设置tfs的实际安装目录

  8. dev_name= eth0                         #设置网卡设备

  9. thread_count = 4                       #设置dataserver工作线程池,根据CPU核数量*2定

  10. ip_addr = 192.168.10.13                #设置当前dataserver的IP地址,只有这与ds2不同

  11. [dataserver]

  12. ip_addr = 192.168.10.11                #设置nameserver的地址,如果是ha为vip否则是主ns IP

  13. ip_addr_list = 192.168.10.11|192.168.10.12 #设置需要通讯的主备nameserver的Real IP列表,与ns.conf配置项一致

  14. port = 8108                            #设置nameserver的监听端口,与ns.conf配置项一致

  15. heart_interval = 2

  16. check_interval = 2

  17. replicate_threadcount = 2

  18. block_max_size = 75497472              #块最大尺寸,单位(字节),根据实际情况修改,这里使用默认值:75497472,即64M。或83886080(80m)

  19. dump_visit_stat_interval = 60

  20. backup_type = 0                        #备件类型, 1: tfs, 2: nfs, 0:不备份

  21. backup_path = /data/tfs                #备件路径

  22. max_data_file_nums = 100               #最大datafile值, default: 50,这里改为100;

  23. mount_name = /data/tfs                 #设置DataServer  

  24. mount_maxsize = 1782579200             #mount 时磁盘的大小

  25. base_filesystem_type = 1               #文件系统类型: 0: no initialize, 1: ext4, 2: ext3 posix fallocate, 3: ext3 ftruncate

  26. superblock_reserve = 0

  27. avg_file_size = 40960

  28. mainblock_size = 75497472               #主块的大小, 单位(字节),不能超过block_max_size设置,这里改为:75497472;或83886080(80m)

  29. extblock_size = 419430

  30. block_ratio = 0.5

  31. hash_slot_ratio = 0.5

  32. ds_thread_count = 4


4.2 修改ds1的配置文件ads.conf
  1. vim ./conf/ads.conf

  2. [public]

  3. #log file size default 1GB

  4. log_size=1073741824

  5. #log file num default 64

  6. log_num = 64

  7. #log file level default debug

  8. log_level=debug

  9. #main queue size default 10240

  10. task_max_queue_size = 10240

  11. #listen port

  12. port = 12000

  13. #work directoy

  14. work_dir=/usr/local/tfs

  15. #device name

  16. ev_name= eth0

  17. #work thread count default 4

  18. thread_count = 4

  19. ip_addr = 192.168.10.13   ##只有这与ds2不同

  20. [adminserver]

  21. # check interval

  22. check_interval = 5

  23. # check count

  24. check_count = 5

  25. # warn threshhold

  26. warn_dead_count = 3

  27. # wait time before kill ds

  28. ds_fkill_waittime = 15

  29. # ds start base cmd

  30. ds_script = /usr/local/tfs/bin/dataserver -f /usr/local/tfs/conf/ds.conf -d    

  31. ds_index_list = 1,2,3

  32. [nameserver]

  33. ip_addr = 192.168.10.11

  34. port = 8108

  35. [dataserver]

  36. port = 9998

  37. lock_file = /usr/local/tfs/logs/dataserver

  38. mount_name = /usr/local/tfs/    

##ds1,与ds2差别只有那几个ip
五.启动tfs并测试
ns1上启动nameserver
  1. # /usr/local/tfs/scripts/tfs start_ns    ##关闭是stop_ns

  2. ds1上存储区预分配

  3. # /usr/local/tfs/scripts/stfs format 1-3    ##如果有successful字样代表成功

  4. ds2上存储区预分配

  5. # /usr/local/tfs/scripts/stfs format 1-3

  6. ds1,ds2上启动dataserver

  7. # /usr/local/tfs/scripts/tfs admin_ds    ##启动了ds

  8. # /usr/local/tfs/scripts/tfs check_ds    ##查看ds运行情况,如下代表正常

  9.  dataserver [ 1 2 3 ] is running  

  10. # ./scripts/tfs stop_admin               ##关闭ds

  11. # ./scripts/tfs stop_ds_all              ##关闭ds

  1. # /usr/local/tfs/bin/ds_client -d 192.168.10.13:9998  ##链接dsserver

  2. DataServer> list_block 1      ##是数字1,查看block,如果没类似如下的显示,可能是ds启动不正常

  3. get message type: 32

  4. Logic Block Nums :9

  5. 496 497 498 499 500 502 506 508 515  

  6. Logic Block Nums :9

  7. DataServer> quit

  8. # /usr/local/tfs/bin/tfstool -s 192.168.10.11:8108 ##连接到namserver,上传文件测试

  9. TFS> put /etc/passwd      ##上传/etc/passwd文件

  10. put /etc/passwd => T19RETByJT1RCvBVdK success.   ##出现这个代表成功

  11. TFS> get T19RETByJT1RCvBVdK passwd.1

  12. fetch T19RETByJT1RCvBVdK => passwd.1 success     ##下载成功

  13. TFS>quit   ##退出,打开查看该文件

  14. # /usr/local/tfs/bin/tfstool -s 192.168.10.12:8108 ##连接到备用namserver,下载测试

  15. TFS> get T19RETByJT1RCvBVdK passwd.bak

  16. fetch T19RETByJT1RCvBVdK => passwd.bak success

成功部署了主备模式的tfs

六.部署namserver Ha

未完待续。。。。。。。。。。。。