1.- 配置机器环境
Ubuntu Server 22.04 x64版本,EC2,内存32G,8CPU
a.- 安装Java:sudo apt install default-jre
检查:java --version
b.- 下载dolphin scheduler: wget https://dlcdn.apache.org/dolphinscheduler/3.2.0/apache-dolphinscheduler-3.2.0-bin.tar.gz
解压:tar -xvzf apache-dolphinscheduler-3.2.0-bin.tar.gz
c.- 下载zookeeper:wget https://dlcdn.apache.org/zookeeper/zookeeper-3.8.3/apache-zookeeper-3.8.3-bin.tar.gz
解压:tar -xvzf apache-zookeeper-3.8.3-bin.tar.gz
d.- 安装mysql:
sudo apt install mysql-server
sudo systemctl start mysql.service
sudo service mysql status
2.- 配置用户环境
EC2默认用户名 ubuntu
a.- 创建用户dolphinscheduler
sudo useradd dolphinscheduler
sudo passwd dolphinscheduler
查看:cat /etc/passwd
创建dolphinscheduler用户home目录:
cd /home
sudo mkdir dolphinscheduler
sudo chown dolphinscheduler.dolphinscheduler dolphinscheduler/
b.- 配置sudo免密
sudo sed -i '$adolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
sudo sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
查看:sudo cat /etc/sudoers
c.- 配置机器免密登陆
因为后面启动dolphin scheduler服务需要通过 sudo命令,因此需要root用户具备ssh回自己账户的权限
切换到root用户:sudo su
创建ssh key:ssh-keygen (都按回车默认即可)
切换到key目录:cd .ssh/
然后将id_rsa.pub添加到authorized_keys中
同样dolphinscheduler用户也一样设置:
切换到dolphinscheduler: su dolphinscheduler (照提示输入密码:dolphinscheduler)
创建ssh key:ssh-keygen
切换到key目录:cd .ssh/
然后将id_rsa.pub添加到authorized_keys中
验证:sudo ssh -i /root/.ssh/id_rsa localhost (如果无需密码进入root则ok)
3.- 启动zookeeper
cd apache-zookeeper-3.8.3-bin/conf/
cp zoo_sample.cfg zoo.cfg
修改dataDir目录dataDir=./tmp/zookeeper
~/apache-zookeeper-3.8.3-bin/bin/zkServer.sh start
如下完成启动
4.- 修改dolphin scheduler相关配置
a.- 赋予用户dolphinscheduler文件夹权限
sudo chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-3.2.0-bin
sudo chmod -R 755 apache-dolphinscheduler-3.2.0-bin
切换到用户dolphinscheduler:su dolphinscheduler
b.- 修改install_env.sh文件
cd apache-dolphinscheduler-3.2.0-bin/bin/env/
cp install_env.sh install_env.sh-bak (先备份一下,再将其删除)
rm install_env.sh
vim install_env.sh
文件内容如下:
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# Due to the master, worker, and API server being deployed on a single node, the IP of the server is the machine IP or localhost
ips="localhost"
sshPort="22"
masters="localhost"
workers="localhost:default"
alertServer="localhost"
apiServers="localhost"
# DolphinScheduler installation path, it will auto-create if not exists
installPath=~/dolphinscheduler-3.2
# Deploy user, use the user you create in section **Configure machine SSH password-free login**
deployUser="ubuntu"
# ---------------------------------------------------------
chmod +x install_env.sh (改回为shell可执行文件)
c.- 修改dolphinscheduler_env.sh文件
cp dolphinscheduler_env.sh dolphinscheduler_env.sh-bak
rm dolphinscheduler_env.sh
vim dolphinscheduler_env.sh
文件内容如下:
# ---------------------------------------------------------
# JAVA_HOME, will use it to start DolphinScheduler server
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
# Database related configuration, set database type, username and password
# for mysql
export DATABASE=${DATABASE:-mysql}
export SPRING_PROFILES_ACTIVE=${DATABASE}
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true"
export SPRING_DATASOURCE_USERNAME=dolphinscheduler
export SPRING_DATASOURCE_PASSWORD=dolphinscheduler
# DolphinScheduler server related configuration
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
# Registry center configuration, determines the type and link of the registry center
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}
# Tasks related configurations, need to change the configuration if you use the related tasks.
#export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
#export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
#export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
#export PYTHON_LAUNCHER=${PYTHON_LAUNCHER:-/opt/soft/python}
#export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
#export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
#export DATAX_LAUNCHER=${DATAX_LAUNCHER:-/opt/soft/datax/bin/python3}
#export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_LAUNCHER:$PATH
# ---------------------------------------------------------
chmod +x dolphinscheduler_env.sh
5.- 配置MySQL数据库
a.- 首先需要下载mysql连接驱动
wget https://cdn.mysql.com/archives/mysql-connector-java-8.1/mysql-connector-j_8.1.0-1ubuntu22.04_all.deb
dpkg -x mysql-connector-j_8.1.0-1ubuntu22.04_all.deb ./
cd /home/ubuntu/usr/share/java/ (解压后在这个地方)
切换到dolphinscheduler,因为需要将mysql连接驱动copy到dolphinscheduler文件夹里,而dolphinscheduler是这些文件夹到owner, 如果不用dolphinscheduler自己copy则后续会有权限问题
su dolphinscheduler
cp mysql-connector-j* /home/ubuntu/apache-dolphinscheduler-3.2.0-bin/api-server/libs/
cp mysql-connector-j* /home/ubuntu/apache-dolphinscheduler-3.2.0-bin/alert-server/libs/
cp mysql-connector-j* /home/ubuntu/apache-dolphinscheduler-3.2.0-bin/master-server/libs/
cp mysql-connector-j* /home/ubuntu/apache-dolphinscheduler-3.2.0-bin/worker-server/libs/
如上,需要将驱动复制到所有文件夹到libs目录中
b.-配置mysql
切换回ubuntu用户
sudo mysql -uroot -p (输入后会提示输入密码,直接按回车即可,无需密码),执行命令如下:
mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
mysql> CREATE USER 'dolphinscheduler'@'%' IDENTIFIED BY 'dolphinscheduler';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%';
mysql> CREATE USER 'dolphinscheduler'@'localhost' IDENTIFIED BY 'dolphinscheduler';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'localhost';
mysql> FLUSH PRIVILEGES;
mysql> exit
c.- 初始化数据库
sudo bash /home/ubuntu/apache-dolphinscheduler-3.2.0-bin/tools/bin/upgrade-schema.sh
6.- 启动Dolphin Scheduler
sudo bash ~/apache-dolphinscheduler-3.2.0-bin/bin/install.sh
然后即可通过浏览器打开:http://localhost:12345/dolphinscheduler/ui
默认的用户名和密码是:admin/dolphinscheduler123
按照这个方式安装完成后,Dolphin默认在/root/下
启动后的log在这个dolphinscheduler-3.2里
设置重启时自动启动DS的脚本,加到cron job里
start_dolphin.sh
/home/ubuntu/apache-zookeeper-3.8.3-bin/bin/zkServer.sh start
sudo bash /home/ubuntu/apache-dolphinscheduler-3.2.0-bin/bin/start-all.sh
重启集群
停止命令:
bin/stop-all.sh
停止master服务:
bash bin/dolphinscheduler-daemon.sh stop master-server
停止worker服务:
bash bin/dolphinscheduler-daemon.sh stop worker-server
停止api服务:
bash bin/dolphinscheduler-daemon.sh stop api-server
停止alert服务:
bash bin/dolphinscheduler-daemon.sh stop alert-server
启动命令:
bin/start-all.sh
启动master服务:
bash bin/dolphinscheduler-daemon.sh start master-server
启动worker服务:
bash bin/dolphinscheduler-daemon.sh start worker-server
启动api服务:
bash bin/dolphinscheduler-daemon.sh start api-server
启动alert服务:
bash bin/dolphinscheduler-daemon.sh start alert-server