大数据集群软件启动脚本

大家也可以访问我的gitee仓库直接克隆源码大数据软件启动脚本

包括：hadoop，zookeeper，kafka，flume，hive，maxwell，dolphinscheduler，superset，集群一键启动，集群文件分发，集群执行命令，集群同步时间脚本

hadoop

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#!/bin/bash

# hadoop集群启动

if [ $# -eq 0 ]; then
	echo "Usage $0 start|stop"
fi
case $1 in

    "start"){
        echo " -------- 启动 集群 -------"
        echo " -------- 启动 hadoop集群 -------"
        ssh niit01 "source /etc/profile;start-dfs.sh"
        ssh niit03 "source /etc/profile;start-yarn.sh"
        ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh start historyserver"
    };;
    "stop"){
        echo " -------- 停止 集群 -------"
        echo " -------- 停止 hadoop集群 -------"
        ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh stop historyserver"
        ssh niit03 "source /etc/profile;stop-yarn.sh"
        ssh niit01 "source /etc/profile;stop-dfs.sh"
    };;
esac

zookeeper

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
#!/bin/bash

# zookeeper集群启动脚本

echo "Usage: $0 start|stop|status (zookeeper)"
case $1 in
"start"){
    for i in niit01 niit02 niit03
    do
        echo ---------- zookeeper $i 启动 ------------
        ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh start"
    done
};;
"stop"){
for i in niit01 niit02 niit03
    do
        echo ---------- zookeeper $i 停止 ------------
        ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh stop"
    done
};;
"status"){
    for i in niit01 niit02 niit03
    do
        echo ---------- zookeeper $i 状态 ------------
        ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh status"
    done
};;
esac

kafka

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#!/bin/bash

# kafka集群启动

case $1 in
"start"){
    for i in niit01 niit02 niit03
    do
    	echo " --------启动 $i Kafka-------"
        ssh $i "/opt/onlineedu/kafka/bin/kafka-server-start.sh -daemon /opt/onlineedu/kafka/config/server.properties"
    done
};;

"stop"){
    for i in niit01 niit02 niit03
    do
        echo " --------停止 $i Kafka-------"
        ssh $i "/opt/onlineedu/kafka/bin/kafka-server-stop.sh stop"
    done
};;
esac

flume

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#! /bin/bash

# flume采集通道1启动脚本

case $1 in
"start"){
    for i in niit01 niit02
    do
        echo " --------启动 $i 采集flume-------"
        ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/file-flume-kafka.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log1.txt 2>&1  &"
    done
};;
"stop"){
    for i in niit01 niit02
    do
        echo " --------停止 $i 采集flume-------"
        ssh $i "ps -ef | grep file-flume-kafka | grep -v grep |awk  '{print \$2}' | xargs -n1 kill -9 "
    done
};;
esac

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#! /bin/bash

# flume采集通道2启动脚本

case $1 in
"start"){
    for i in niit03
    do
        echo " --------启动 $i 消费flume-------"
        ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/kafka-flume-hdfs.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log2.txt 2>&1  &"
    done
};;
"stop"){
    for i in niit03
    do
        echo " --------停止 $i 消费flume-------"
        ssh $i "ps -ef | grep kafka-flume-hdfs | grep -v grep | awk '{print \$2}' | xargs -n1 kill -9 "
    done
};;
esac

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/bash

# flume采集通道3启动脚本

case $1 in
"start")
        echo " --------启动 niit03 业务数据flume-------"
        ssh niit03 "nohup /opt/onlineedu/flume/bin/flume-ng agent -n a1 -c /opt/onlineedu/flume/conf -f /opt/onlineedu/flume/job/kafka_to_hdfs_db.conf >/opt/onlineedu/flume/log3.txt >/dev/null 2>&1 &"
;;
"stop")

        echo " --------停止 niit03 业务数据flume-------"
        ssh niit03 "ps -ef | grep kafka_to_hdfs_db | grep -v grep |awk '{print \$2}' | xargs -n1 kill"
;;
esac

hive

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#!/bin/bash

# hive启动

HIVE_LOG_DIR=$HIVE_HOME/logs
mkdir -p $HIVE_LOG_DIR
#检查进程是否运行正常，参数1为进程名，参数2为进程端口

function check_process()
{
  pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
  ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
  echo $pid
  [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
}

function hive_start()
{
  metapid=$(check_process HiveMetastore 9083)
  cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
  cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1"
  [ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"
  server2pid=$(check_process HiveServer2 10000)
  cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
  [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
}

function hive_stop()
{
  metapid=$(check_process HiveMetastore 9083)
  [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
  server2pid=$(check_process HiveServer2 10000)
  [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
}

case $1 in
"start")
  hive_start
  ;;
"stop")
  hive_stop
  ;;
"restart")
  hive_stop
  sleep 2
  hive_start
  ;;
"status")
  check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
  check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
  ;;
*)
  echo Invalid Args!
  echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)'
  ;;
esac

maxwell

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#!/bin/bash

# hive启动

HIVE_LOG_DIR=$HIVE_HOME/logs
mkdir -p $HIVE_LOG_DIR
#检查进程是否运行正常，参数1为进程名，参数2为进程端口

function check_process()
{
  pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
  ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
  echo $pid
  [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
}

function hive_start()
{
  metapid=$(check_process HiveMetastore 9083)
  cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
  cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1"
  [ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动"
  server2pid=$(check_process HiveServer2 10000)
  cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
  [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
}

function hive_stop()
{
  metapid=$(check_process HiveMetastore 9083)
  [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
  server2pid=$(check_process HiveServer2 10000)
  [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
}

case $1 in
"start")
  hive_start
  ;;
"stop")
  hive_stop
  ;;
"restart")
  hive_stop
  sleep 2
  hive_start
  ;;
"status")
  check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
  check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
  ;;
*)
  echo Invalid Args!
  echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)'
  ;;
esac

dolphinscheduler

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#!/bin/bash

# dolphinscheduler集群启动

function node_start()
{
  /opt/onlineedu/dolphinscheduler/bin/start-all.sh  
  /home/niit/bin/xRun.sh jps
}

function node_stop()
{
  /opt/onlineedu/dolphinscheduler/bin/stop-all.sh
  /home/niit/bin/xRun.sh jps
}
case $1 in
"start")
  node_start
  ;;
"stop")
  node_stop
  ;;
"restart")
  node_stop
  sleep 10
  node_start
  ;;
*)
  
esac

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#!/bin/bash

# dolphinscheduler单机版启动

function node_start()
{
  /opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh start standalone-server   
  /home/niit/bin/xRun.sh jps
}

function node_stop()
{
  /opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop standalone-server
  /home/niit/bin/xRun.sh jps
}
case $1 in
"start")
  node_start
  ;;
"stop")
  node_stop
  ;;
"restart")
  node_stop
  sleep 10
  node_start
  ;;
*)
  
esac

superset

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
#!/bin/bash

# superset可视化启动

superset_status(){
    result=`ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | wc -l`
    if [[ $result -eq 0 ]]; then
        return 0
    else
        return 1
    fi
}
superset_start(){
        source ~/.bashrc
        superset_status >/dev/null 2>&1
        if [[ $? -eq 0 ]]; then
            conda activate superset ; gunicorn --workers 5 --timeout 120 --bind niit01:8787 --daemon 'superset.app:create_app()'
        else
            echo "superset正在运行"
        fi

}

superset_stop(){
    superset_status >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
        echo "superset未在运行"
    else
        ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | xargs kill -9
    fi
}


case $1 in
    start )
        echo "启动Superset"
        superset_start
    ;;
    stop )
        echo "停止Superset"
        superset_stop
    ;;
    restart )
        echo "重启Superset"
        superset_stop
        superset_start
    ;;
    status )
        superset_status >/dev/null 2>&1
        if [[ $? -eq 0 ]]; then
            echo "superset未在运行"
        else
            echo "superset正在运行"
        fi
esac

集群一键启动

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
#!/bin/bash

# 所有集群所有软件启动脚本

function node_start()
{
  /home/niit/bin/xCluster.sh start
  sleep 15
  /home/niit/bin/zk.sh start
  sleep 10
  /home/niit/bin/kf.sh start
  sleep 10
  /home/niit/bin/hv.sh start
  sleep 5
  /home/niit/bin/mxw.sh start
  sleep 5
  /home/niit/bin/f1.sh start
  sleep 5
  /home/niit/bin/f2.sh start
  sleep 5
  /home/niit/bin/f3.sh start
  sleep 5
  /home/niit/bin/dolphinscheduler-standalone.sh stop 
  sleep 5
  /home/niit/bin/superset.sh start
  sleep 5
  /home/niit/bin/xRun.sh jps
}

function node_stop()
{
  /home/niit/bin/mxw.sh stop
  sleep 5
  /home/niit/bin/f1.sh stop
  sleep 5
  /home/niit/bin/f2.sh stop
  sleep 5
  /home/niit/bin/f3.sh stop
  sleep 5
  /home/niit/bin/kf.sh stop
  sleep 10
  /home/niit/bin/kf.sh stop
  sleep 10
  /home/niit/bin/hv.sh stop
  sleep 5
  /home/niit/bin/xCluster.sh stop
  sleep 15
  /home/niit/bin/zk.sh stop
  sleep 10
  /home/niit/bin/superset.sh stop
  sleep 5
  /home/niit/bin/dolphinscheduler-standalone.sh stop 
  sleep 5
  /home/niit/bin/xRun.sh jps
}
case $1 in
"start")
  node_start
  ;;
"stop")
  node_stop
  ;;
"restart")
  node_stop
  sleep 10
  node_start
  ;;
*)
  
esac

集群文件分发

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#!/bin/bash

# 集群分发文件脚本，普通用户权限

#1. 判断参数个数
if [ $# -lt 1 ]
then
    echo Not Enough Arguement!
    echo Usage: xSync.sh path1 path2..
    exit;
fi

#2. 遍历集群所有机器
for host in niit01 niit02 niit03
do
echo ==================== $host ====================
#3. 遍历所有目录，挨个发送

    for file in $@
    do
        #4 判断文件是否存在
        if [ -e $file ]
        then
            #5. 获取父目录
            pdir=$(cd -P $(dirname $file); pwd)

            #6. 获取当前文件的名称
            fname=$(basename $file)
            ssh $host "mkdir -p $pdir"
            rsync -av $pdir/$fname $host:$pdir
        else
            echo $file does not exists!
        fi
    done
done

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#!/bin/bash

# 集群分发文件脚本，root权限

#1. 判断参数个数
if [ $# -lt 1 ]
then
    echo Not Enough Arguement!
    echo Usage: xSync.sh path1 path2 ...
    exit;
fi

#2. 遍历集群所有机器
for host in niit01 niit02 niit03
do
echo ==================== $host ====================
#3. 遍历所有目录，挨个发送

    for file in $@
    do
        #4 判断文件是否存在
        if [ -e $file ]
        then
            #5. 获取父目录
            pdir=$(cd -P $(dirname $file); pwd)

            #6. 获取当前文件的名称
            fname=$(basename $file)
            sudo ssh $host "mkdir -p $pdir"
            sudo rsync -av $pdir/$fname $host:$pdir
        else
            echo $file does not exists!
        fi
    done
done

集群执行命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#/bin/bash

# 集群执行命令脚本

if (( $# == 0 ));then
    echo "Usage xRun.sh \"<COMMAND>\""
    exit 0
fi
for node in niit01 niit02 niit03
do
    echo "======== $node ========"
    echo "ssh $node $1"
    ssh $node "source /etc/profile;$1"
done

集群同步时间

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
#!/bin/bash

# 集群时间同步脚本，需要安装chrony到集群中

# 检查参数是否为空
if [ -z "$1" ]; then
        echo "Usage:  `basename $0` yyyy-MM-dd HH:mm:ss"
  exit 1
fi

# 使用date命令将时间字符串转换为日期和时间
# 如果转换失败，则说明时间字符串不合法
if ! date -d "$*" >/dev/null 2>&1; then
        echo "Wrong argument for $*"
        echo "Usage:  `basename $0` yyyy-MM-dd HH:mm:ss"
  exit 1
fi

echo ">>>>>>>>>>>> SYNC TIME START >>>>>>>>>>>>"
sum=-1

while [ $sum -ne 0 ]; do
  echo set time for niit01 to $1 '>>>'
  ssh niit01 "(sudo timedatectl set-ntp false && sudo date -s \"$*\" && sudo timedatectl set-ntp true)"
  ok1=$?
  echo sync time from niie02 to niit01 '>>>'
  ssh niit02 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)"
  ok2=$?
  echo sync time from niit03 to niit01 '>>>'
  ssh niit03 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)"
  ok3=$?
  sum=`expr $ok1 + $ok2 + $ok3`

  if [ $sum -eq 0 ]; then
    echo "<<<<<<<<<<<<< SYNC TIME END <<<<<<<<<<<<<"
    sleep 5
    xRun.sh date
  else
    echo "sync time failed, will try 10 senconds later"
    sleep 10
  fi
done