CentOS7安装CDH6.3.2

1、CDH安装包地址 通过网盘分享的文件：CDH6.3.2.zip 链接: https://pan.baidu.com/s/1QVcPclxU4dB1TfnlkRoDxw?pwd=3env 2、安装脚本 #!/bin/bash # CDH 6.3.2 自动安装脚本 - 单机伪分布式 # 安装组件: HDFS + YARN + Hive + ZooKeeper set -e HOST_IP="192.168.2.113" HOSTNAME="root" MYSQL_ROOT_PASSWORD="xxx" echo "==========================================" echo "CDH 6.3.2 自动化安装" echo "==========================================" # 1. 配置主机名和hosts echo "[1/10] 配置网络..." hostnamectl set-hostname $HOSTNAME cat > /etc/hosts << HOSTEOF 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 $HOST_IP $HOSTNAME HOSTEOF # 2. 关闭防火墙和SELinux echo "[2/10] 关闭防火墙和SELinux..." systemctl stop firewalld 2>/dev/null || true systemctl disable firewalld 2>/dev/null || true setenforce 0 2>/dev/null || true sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config sed -i 's/SELINUX=permissive/SELINUX=disabled/g' /etc/selinux/config # 3. 配置系统参数 echo "[3/10] 配置系统参数..." echo never > /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true echo never > /sys/kernel/mm/transparent_hugepage/defrag 2>/dev/null || true echo "vm.swappiness=10" >> /etc/sysctl.conf sysctl -p echo '* soft nproc 65535' >> /etc/security/limits.conf echo '* hard nproc 65535' >> /etc/security/limits.conf echo '* soft nofile 65535' >> /etc/security/limits.conf echo '* hard nofile 65535' >> /etc/security/limits.conf # 4. 安装基础依赖 echo "[4/10] 安装基础依赖..." yum install -y -q wget curl net-tools vim unzip zip \ bind-utils psmisc cyrus-sasl-plain cyrus-sasl-gssapi \ fuse portmap fuse-libs httpd mod_ssl openssl-devel \ libxslt chrony ntpdate 2>/dev/null || yum install -y wget curl net-tools vim unzip # 5. 配置时间同步 echo "[5/10] 配置时间同步..." systemctl enable chronyd 2>/dev/null || true systemctl start chronyd 2>/dev/null || true timedatectl set-timezone Asia/Shanghai 2>/dev/null || true # 6. 安装JDK echo "[6/10] 安装JDK..." # 卸载系统自带的OpenJDK rpm -qa | grep -E "java|jdk" | xargs -I {} rpm -e --nodeps {} 2>/dev/null || true # 从CDH包中查找JDK JDK_RPM=$(find /opt/software -name "oracle-j2sdk*.rpm" 2>/dev/null | head -1) if [[ -n "$JDK_RPM" ]]; then rpm -ivh $JDK_RPM 2>/dev/null || true fi # 配置环境变量 if [[ -d "/usr/java/jdk1.8.0_181-cloudera" ]]; then echo 'export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera' > /etc/profile.d/jdk.sh echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile.d/jdk.sh source /etc/profile.d/jdk.sh fi # 创建CDH数据库 echo "[8/10] 创建CDH数据库..." # 9. 安装Cloudera Manager echo "[10/10] 安装Cloudera Manager..." mkdir -p /opt/cloudera-manager # 查找并复制RPM包 find /opt/software/CDH6.3.2 -name "*.rpm" -exec cp {} /opt/cloudera-manager/ \; 2>/dev/null || true cd /opt/cloudera-manager # 安装RPM包 for rpm in cloudera-manager-daemons-*.rpm; do [[ -f "$rpm" ]] && rpm -ivh --force --nodeps $rpm 2>/dev/null || true done for rpm in cloudera-manager-agent-*.rpm; do [[ -f "$rpm" ]] && rpm -ivh --force --nodeps $rpm 2>/dev/null || true done for rpm in cloudera-manager-server-*.rpm; do [[ -f "$rpm" ]] && rpm -ivh --force --nodeps $rpm 2>/dev/null || true done # 配置MySQL JDBC驱动 mkdir -p /usr/share/java CONNECTOR=$(find /opt/software -name "mysql-connector-java*.jar" 2>/dev/null | head -1) [[ -n "$CONNECTOR" ]] && cp "$CONNECTOR" /usr/share/java/mysql-connector-java.jar # 配置CM cat > /etc/cloudera-scm-agent/config.ini << CMEOF [General] server_host=cdh-node server_port=7182 CMEOF cat > /etc/cloudera-scm-server/db.properties << DBEOF com.cloudera.cmf.db.type=mysql com.cloudera.cmf.db.host=192.168.2.113:3306 com.cloudera.cmf.db.name=scm com.cloudera.cmf.db.user=root com.cloudera.cmf.db.password=xxx com.cloudera.cmf.db.setupType=EXTERNAL DBEOF # 配置Parcel mkdir -p /opt/cloudera/parcel-repo find /opt/software/cdh_extracted -name "*.parcel" -exec cp {} /opt/cloudera/parcel-repo/ \; 2>/dev/null || true find /opt/software/cdh_extracted -name "*.sha*" -exec cp {} /opt/cloudera/parcel-repo/ \; 2>/dev/null || true find /opt/software/cdh_extracted -name "manifest.json" -exec cp {} /opt/cloudera/parcel-repo/ \; 2>/dev/null || true cd /opt/cloudera/parcel-repo for f in *.sha1; do [[ -f "$f" ]] && mv "$f" "${f%.sha1}.sha" 2>/dev/null || true; done chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo 2>/dev/null || true # 初始化数据库 /opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h192.168.2.113 -uroot -p"$MYSQL_ROOT_PASSWORD" scm scm scm 2>/dev/null || echo "数据库初始化可能已存在" # 启动服务 echo "启动Cloudera Manager服务..." systemctl start cloudera-scm-server 2>/dev/null || /opt/cloudera/cm/etc/init.d/cloudera-scm-server start 2>/dev/null || true systemctl start cloudera-scm-agent 2>/dev/null || /opt/cloudera/cm/etc/init.d/cloudera-scm-agent start 2>/dev/null || true systemctl enable cloudera-scm-server 2>/dev/null || true systemctl enable cloudera-scm-agent 2>/dev/null || true echo "" echo "==========================================" echo "CDH 6.3.2 基础安装完成！" echo "==========================================" echo "" echo "请等待2-3分钟后访问: http://$HOST_IP:7180" echo "用户名: admin" echo "密码: admin" echo "" echo "Web UI配置步骤:" echo "1. 选择版本: Cloudera Express (免费版)" echo "2. 集群名称: 自定义" echo "3. 选择主机: 选择 cdh-node" echo "4. 选择Parcel: CDH-6.3.2" echo "5. 检查主机: 修复警告(交换空间、透明大页)" echo "6. 选择服务: 自定义 - HDFS + YARN + Hive + ZooKeeper" echo "7. 配置数据库: 使用之前创建的hive用户" echo "=========================================="