軟件包位置

https://pan.baidu.com/s/1pLFVWkr

文檔地址

http://www.mashibing.com/hadoop_install.html

http://www.mashibing.com/hadoop_test.html

http://www.mashibing.com/hdfs_java.html

http://www.mashibing.com/yarn_test.html

http://www.mashibing.com/map_reduce.html

IP 地址設(shè)置

master:? 192.168.56.101slave1:? 192.168.56.102slave2:? 192.168.56.103slave3:? 192.168.56.104

上傳文件。

解壓jdk

#rpm -ivh jdk-8u91-linux-x64.rpm#cd /usr#ls#java#cd

解壓hadoop

#tar -xvf hadoop-2.7.3.tar.gz#mv hadoop-2.7.3 hadoop #mv hadoop /usr/local

配置

#vi /usr/local/hadoop/etc/hadoop/hadoop-env.sh

把export JAVA_HOME=${JAVA_HOME}改成export JAVA_HOME=/usr/java/default,然后：wq保存退出。

#vi /etc/profile//在最下面加入：exportPATH=$PATH:/usr/local/hadoop/bin:/usr/local/hadoop/sbin

source/etc/profile

關(guān)機(jī)復(fù)制三臺(tái)。

關(guān)閉防火墻。

#systemctl stop firewalld#systemctl disable firewalld

啟用

#vi /usr/local/hadoop/etc/hadoop/core-site.xmlfs.defaultFShdfs://master:9000

#vi /etc/hosts192.168.56.101 master192.168.56.102 slave1192.168.56.103 slave2192.168.56.104 slave3#hdfs namenode -format

master:

#hadoop-daemon.shstartnamenode#jps

slaves:

#hadoop-daemon.shstartdatanode#jps

查看集群情況

#hdfs dfsadmin -report | more

用瀏覽器瀏覽

192.168.56.101:50070

集中式管理集群。

關(guān)閉集群。

master:

#hadoop-daemon.shstopnamenode#jps

slaves:

#hadoop-daemon.shstopdatanode#jps

master

#vi /usr/local/hadoop/etc/hadoop/slavesslave1slave2slave3

#start-dfs.sh

slaves:

#jps

設(shè)置免密登錄(master)

#cd.ssh#ssh-keygen-trsa#ssh-copy-idslave1#ssh-copy-idslave2#ssh-copy-idslave3#ssh-copy-idmaster#stop-dfs.sh#start-dfs.sh

對(duì)文件進(jìn)行增刪改查操作

#hadoop fs -ls /#cd /usr/local#ll#hadoop fs -put ./hadoop-2.7.3.tar.gz /#hadoop fs -ls /

修改默認(rèn)屬性

#cd hadoop/etc/hadoop#vi hdfs-site.xmldfs.replication2

#cd /usr/local#hadoop fs -put ./jdk-8u91-Linux-x64.rpm /#hadoop fs -ls /

修改檢查時(shí)間

#cd hadoop/etc/hadoop#vi hdfs-site.xmldfs.namenode.heartbeat.recheck-interval10000

#stop-dfs.sh#start-dfs.sh

修改配置文件，所有機(jī)器都改

/usr/local/hadoop/etc/hadoop#vi /usr/local/hadoop/etc/hadoop/core-site.xmlhadoop.tmp.dir/var/hadoop#hdfs namenode -format

#cdvi hello.txt#hadoop fs -put ./hello.txt /#hadoop fs -ls /

用eclipse打開，file,new ,project,java project,HelloHDFS

-DHADOOP_USER_NAME=root

image.png

導(dǎo)入用到的包

image.png

lib下的所有包

image.png

publicstaticvoidmain(String[] args)throwsException{

URL url =newURL("http://www.baidu.com");? ? ? ? InputStreamin= url.openStream();? ? ? ? IOUtils.copyBytes(in,System.out,4096,true);

URL url =newURL("hdfs://192.168.56.101:9000/hello.txt");? ? ? ? InputStreamin= url.openStream();? ? ? ? IOUtils.copyBytes(in,System.out,4096,true);

URL.setURLStreamHandlerFactory(newFsUrlStreamHandlerFactory());? ? ? ? URL url =newURL("hdfs://192.168.56.101:9000/hello.txt");? ? ? ? InputStreamin= url.openStream();? ? ? ? IOUtils.copyBytes(in,System.out,4096,true);

Configuration conf =newConfiguration();? ? ? ? conf.set("fs.defaultFS","hdfs://192.168.56.101:9000");? ? ? ? FileSystem fileSystem = FileSystem.get(conf);

新建一個(gè)目錄

booleansuccess = fileSystem.mkdirs(newPath("/xuehuai"));? ? ? ? System.out.println(success);

master

#hadoop fs -ls /

判斷存不存在

success = fileSystem.exists(newPath("/hello.txt"));? ? ? ? System.out.println(success);

刪除一個(gè)文件

success = fileSystem.delete(newPath("/msb"),true);? ? ? ? System.out.println(success);

master:

#cd /usr/local/hadoop/etc/hadoop#vi hdfs-site.xmldfs.permissions.enabledfalse#hadoop-daemon.sh stop namenode#hadoop-daemon.sh start namenode

FSDataOutputStream out = fileSystem.create(newPath("/test.data"),true);? ? ? ? FileInputStream fis =newFileInputStream("d:/test1/hive-env.sh.template");? ? ? ? IOUtils.copyBytes(fis, out,4096,true);

master:

#hadoop fs -ls /#hadoop fs -text /test.data#hadoop fs -rm /test.data

FSDataOutputStreamout= fileSystem.create(new Path("/test.data"),true);? ? ? ? FileInputStreamin= new FileInputStream("d:/test1/hive-env.sh.template");? ? ? ? byte[] buf = new byte[4096];intlen =in.read(buf);while(len !=-1) {out.write(buf,0,len);? ? ? ? ? ? len =in.read(buf);? ? ? ? }in.close();out.close();

}

}

配置計(jì)算調(diào)度系統(tǒng)yarn和計(jì)算引擎mr

配置yarn(所有）

#vi /usr/local/hadoop/etc/hadoop/yarn-site.xmlyarn.resourcemanager.hostnamemasteryarn.nodemanager.aux-servicesmapreduce_shuffleyarn.nodemanager.auxservices.mapreduce.shuffle.classorg.apache.hadoop.mapred.ShuffleHandler

配置mapreduce.(所有)

#cp /usr/local/hadoop/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/etc/hadoop/mapred-site.xml#vi /usr/local/hadoop/etc/hadoop/mapred-site.xmlmapreduce.framework.nameyarn

#start-yarn.sh//master

瀏覽器：192.168.56.101:8088

3、上傳文件

#vi input.txt#hadoop fs -mkdir /input#hadoop fs -put input.txt /input#hadoop fs -ls /input#find /usr/local/hadoop -name *example*.jar#hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar word