cloudera CDH 5.11.1 在线安装 最新版

cloudera CDH 5.11.1 在线安装 最新版

摘要

cloudera CDH 5.11.1 在线安装 最新版

基础配置 ubuntu14.04

参考链接

https://www.cloudera.com/documentation/enterprise/latest/topics/cm_ig_install_path_b.html

修改每个机器上的名字

/etc/hostname

修改域名解析

/etc/hosts

127.0.0.1 localhost
192.168.2.200 master
192.168.2.201 slave01
192.168.2.202 slave02

修改ip

/etc/network/interfaces

# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
#auto eth0
#iface eth0 inet dhcp
auto eth0 
iface eth0 inet static
address 192.168.2.200
gateway 192.168.2.1
netmask 255.255.255.0

修改dns
sudo vi /etc/resolvconf/resolv.conf.d/base

nameserver 192.168.2.1
nameserver 223.5.5.5
nameserver 223.6.6.6

ssh免密码登录,master上执行

ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
scp ~/.ssh/authorized_keys root@slave01:~/.ssh/
scp ~/.ssh/authorized_keys root@slave02:~/.ssh/

验证 ssh slave01 ; ssh slave02 ;

ubuntu下修改文件最大打开数

查看 ulimit -n
临时修改 ulimit -SHn 51200

sudo vi /etc/security/limits.conf
* soft nofile 1000000
* hard nofile 1000000

重启生效

更新系统

sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 327574EE02A818DD

wget -O cloudera-manager.list https://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/cloudera.list?_ga=2.224820442.1934962751.1498223273-1517185705.1498059178 
mv cloudera-manager.list /etc/apt/sources.list.d/
apt-get update

ntp

sudo apt-get update && sudo apt-get install ntp
sudo service ntp start

jdk

sudo apt-get install oracle-j2sdk1.7
sudo apt-get install libmysql-java

mysql

sudo apt-get install mysql-server

mysql -h127.0.0.1 -uroot -p
grant all privileges on *.* to root@'%' identified by 'root' with grant option;
flush privileges;
service mysql restart

CREATE DATABASE cmf DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;

各个数据库解释
https://www.cloudera.com/documentation/enterprise/latest/topics/cm_ig_mysql.html#concept_dsg_3mq_bl

系统配置

cat /proc/sys/vm/swappiness
sudo sysctl vm.swappiness=10
sudo vi /etc/sysctl.conf
vm.swappiness=10

master

sudo apt-get install cloudera-manager-daemons cloudera-manager-server
sudo apt-get install oracle-j2sdk1.7

mkdir -p /var/lib/cloudera-host-monitor
mkdir -p /var/lib/cloudera-service-monitor
rm -rf /var/lib/cloudera-scm-server/search/*
cd /usr/share/cmf/lib && wget http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.42/mysql-connector-java-5.1.42.jar

sudo service cloudera-scm-server restart
tail -F /var/log/cloudera-scm-server/cloudera-scm-server.log
http://master:7180

配置cmf数据库链接信息
vi /etc/cloudera-scm-server/db.properties

slave

http://master:7180
搜索slave的时候,比如格式可以是 slave[01-02]

更新系统

cat /proc/sys/vm/swappiness
sudo sysctl vm.swappiness=10
sudo vi /etc/sysctl.conf
vm.swappiness=10

sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 327574EE02A818DD
wget -O cloudera-manager.list https://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm/cloudera.list?_ga=2.224820442.1934962751.1498223273-1517185705.1498059178 
mv cloudera-manager.list /etc/apt/sources.list.d/
apt-get update

sudo apt-get install oracle-j2sdk1.7
sudo apt-get install cloudera-manager-agent cloudera-manager-daemons

指定master
sudo vi /etc/cloudera-scm-agent/config.ini
server_host=192.168.2.200

sudo service cloudera-scm-agent restart
tail -F /var/log/cloudera-scm-agent/cloudera-scm-agent.log

备注

清空docker相关的内容

sudo apt-get purge -y docker-engine
sudo apt-get autoremove -y —purge docker-engine
sudo apt-get autoclean
sudo rm -rf /var/lib/docker
sudo rm /etc/apparmor.d/docker
sudo groupdel docker

sudo apt-get install bridge-utils
ifconfig docker0 down
brctl delbr docker0