{"id":6499,"date":"2022-04-08T21:31:11","date_gmt":"2022-04-08T13:31:11","guid":{"rendered":"https:\/\/egonlin.com\/?p=6499"},"modified":"2022-04-27T15:38:05","modified_gmt":"2022-04-27T07:38:05","slug":"hadoop%e9%9b%86%e7%be%a4spark%e9%9b%86%e7%be%a4%e9%83%a8%e7%bd%b2","status":"publish","type":"post","link":"https:\/\/egonlin.com\/?p=6499","title":{"rendered":"hadoop\u96c6\u7fa4+spark\u96c6\u7fa4\u90e8\u7f72"},"content":{"rendered":"

\u4e00\u3001\u51c6\u5907<\/h1>\n

1.1 \u8f6f\u4ef6\u7248\u672c<\/h2>\n
Ubuntu 16.04.6 (ubuntu-16.04.6-server-amd64.iso)\nJDK 1.8 (jdk-8u201-linux-x64.tar.gz)\nHadoop 2.7.7 (hadoop-2.7.7.tar.gz)\nSpark 2.1.0 (spark-2.1.0-bin-hadoop2.7.tgz)<\/code><\/pre>\n
1.2 \u7f51\u7edc\u89c4\u5212<\/h2>\n
\u672c\u6587\u89c4\u5212\u642d\u5efa3\u53f0\u673a\u5668\u7ec4\u6210\u96c6\u7fa4\u6a21\u5f0f\uff0cIP\u4e0e\u8ba1\u7b97\u673a\u540d\u5206\u522b\u4e3a, \u5982\u679c\u662f\u5355\u53f0\u642d\u5efa\uff0c\u53ea\u9700\u586b\u5199\u4e00\u4e2a\u5373\u53ef<\/p>\n
192.168.241.132 master\n192.168.241.133 slave1\n192.168.241.134 slave2<\/code><\/pre>\n
1.3 \u8f6f\u4ef6\u5305\u62f7\u8d1d<\/h2>\n
\u53ef\u5c06\u4e0a\u8ff0\u8f6f\u4ef6\u5305\u62f7\u8d1d\u52303\u53f0\u673a\u5668\u7684opt\u76ee\u5f55\u4e0b<\/p>\n
JDK 1.8\nHadoop 2.7.7\nSpark 2.1.0<\/code><\/pre>\n
1.4 SSH\u8bbe\u7f6e<\/h2>\n
\u4fee\u6539\/etc\/ssh\/sshd_config\u6587\u4ef6\uff0c\u5c06\u4ee5\u4e0b\u4e09\u9879\u5f00\u542fyes\u72b6\u6001<\/p>\n
PermitRootLogin yes\nPermitEmptyPasswords yes\nPasswordAuthentication yes<\/code><\/pre>\n
\u91cd\u542fssh\u670d\u52a1<\/p>\n
service ssh restart<\/code><\/pre>\n
\u8fd9\u6837root\u7528\u6237\u53ef\u76f4\u63a5\u767b\u9646\uff0c\u4ee5\u53ca\u4e3a\u540e\u7eedssh\u65e0\u5bc6\u7801\u767b\u5f55\u505a\u51c6\u5907\u3002<\/p>\n
1.5 \u7ed1\u5b9aIP\u548c\u4fee\u6539\u8ba1\u7b97\u673a\u540d<\/h2>\n
1.5.1 \u4fee\u6539\/etc\/hosts,\u6dfb\u52a0IP\u7ed1\u5b9a\uff0c\u5e76\u6ce8\u91ca127.0.1.1\uff08\u4e0d\u6ce8\u91ca\u4f1a\u5f71\u54cdhadoop\u96c6\u7fa4\uff09<\/h3>\n
root@master:\/opt# cat \/etc\/hosts\n127.0.0.1   localhost\n#127.0.1.1  ubuntu\n# The following lines are desirable for IPv6 capable hosts\n::1     localhost ip6-localhost ip6-loopback\nff02::1 ip6-allnodes\nff02::2 ip6-allrouters\n\n192.168.241.132 master\n192.168.241.133 slave1\n192.168.241.134 slave2<\/code><\/pre>\n
1.5.2 \u4fee\u6539\/etc\/hostname<\/h3>\n
\u4e3a\u7ed1\u5b9a\u8ba1\u7b97\u673a\u540d\u3002\uff08\u8ba1\u7b97\u673a\u540d\u548c\u4e0a\u9762hosts\u7ed1\u5b9a\u540d\u5fc5\u987b\u4e00\u81f4\uff09<\/p>\n
1.6 SSH\u65e0\u5bc6\u7801\u767b\u9646(\u9700\u63d0\u524d\u5b89\u88c5ssh)<\/h2>\n
1.\u7528rsa\u751f\u6210\u5bc6\u94a5\uff0c\u4e00\u8def\u56de\u8f66\u3002<\/h3>\n
ssh-keygen -t rsa<\/code><\/pre>\n
2.\u8fdb\u5230\u5f53\u524d\u7528\u6237\u7684\u9690\u85cf\u76ee\u5f55\uff08.ssh\uff09<\/h3>\n
cd ~\/.ssh<\/code><\/pre>\n
3.\u628a\u516c\u94a5\u590d\u5236\u4e00\u4efd\uff0c\u5e76\u6539\u540d\u4e3aauthorized_keys<\/h3>\n
cp id_rsa.pub authorized_keys<\/code><\/pre>\n
\u8fd9\u6b65\u6267\u884c\u5b8c\u540e\uff0c\u5728\u5f53\u524d\u673a\u5668\u6267\u884cssh localhost\u53ef\u4ee5\u65e0\u5bc6\u7801\u767b\u5f55\u672c\u673a\u4e86\u3002
\n\u5982\u672c\u673a\u88c5\u6709ssh-copy-id\u547d\u4ee4\uff0c\u53ef\u4ee5\u901a\u8fc7<\/p>\n
ssh-copy-id root@\u7b2c\u4e8c\u53f0\u673a\u5668\u540d<\/code><\/pre>\n
\u7136\u540e\u8f93\u5165\u5bc6\u7801\uff0c\u5728\u6b64\u4e4b\u540e\u5728\u767b\u9646\u7b2c\u4e8c\u53f0\u673a\u5668\uff0c\u53ef\u4ee5\u76f4\u63a5<\/p>\n
ssh[\u7a7a\u683c]\u7b2c\u4e8c\u53f0\u673a\u5668\u540d<\/code><\/pre>\n
\u8fdb\u884c\u767b\u5f55\u3002\u521d\u6b21\u6267\u884c\u4f1a\u63d0\u793a\u786e\u8ba4\uff0c\u8f93\u5165yes\u548c\u767b\u9646\u5bc6\u7801\uff0c\u4e4b\u540e\u5c31\u6ca1\u63d0\u793a\u4e86\u3002<\/p>\n
1.7 JDK\u5b89\u88c5\uff08\u4e09\u53f0\u673a\u5668\u53ef\u540c\u6b65\u8fdb\u884c\uff09<\/h2>\n
\u4e0b\u8f7d\uff1ajdk-8u201-linux-x64.tar.gz \u5305\uff0c\u653e\u5230\/opt\u4e0b\u89e3\u538b<\/p>\n
1.7.1 \u5c06\u89e3\u538b\u540e\u7684\u6587\u4ef6\u5939\u91cd\u547d\u540d<\/h3>\n
mv jdk1.8.0_201 jdk<\/code><\/pre>\n
1.7.2 \u5c06JDK\u73af\u5883\u53d8\u91cf\u914d\u7f6e\u5230\/etc\/profile\u4e2d<\/h3>\n
export JAVA_HOME=\/opt\/jdk\nexport JRE_HOME=\/opt\/jdk\/jre\nexport CLASSPATH=$JAVA_HOME\/lib:$JRE_HOME\/lib\nexport PATH=$JAVA_HOME\/bin:$PATH<\/code><\/pre>\n
1.7.3 \u68c0\u67e5JDK\u662f\u5426\u914d\u7f6e\u597d<\/h3>\n
source \/etc\/profile\njava -version<\/code><\/pre>\n
\u63d0\u793a\u4ee5\u4e0b\u4fe1\u606f\u4ee3\u8868JDK\u5b89\u88c5\u5b8c\u6210\uff1a<\/p>\n
java version "1.8.0_201"\nJava(TM) SE Runtime Environment (build 1.8.0_201-b09)\nJava HotSpot(TM) 64-Bit Server VM (build 25.201-b09, mixed mode)<\/code><\/pre>\n
1.8 \u5176\u4ed6\u914d\u7f6e<\/h2>\n
1.8.1 \u7f51\u7edc\u914d\u7f6e<\/h3>\n
\u4fee\u6539\u4e3a\u56fa\u5b9aIP \uff0c\/etc\/network\/interfaces<\/p>\n
# The loopback network interface\nauto lo\niface lo inet loopback\n# The primary network interface\nauto eth0\n#iface eth0 inet dhcp\niface eth0 inet static\naddress 192.168.241.132\nnetmask 255.255.255.0\ngateway 192.168.20.1<\/code><\/pre>\n
\u91cd\u542f\u7f51\u7edc<\/p>\n
service networking restart<\/code><\/pre>\n
1.8.2 DNS\u914d\u7f6e<\/h3>\n
\u7b2c\u4e00\u79cd\u65b9\u6cd5\uff0c\u6c38\u4e45\u6539<\/h4>\n
\u4fee\u6539\/etc\/resolvconf\/resolv.conf.d\/base\uff08\u8fd9\u4e2a\u6587\u4ef6\u9ed8\u8ba4\u662f\u7a7a\u7684\uff09<\/p>\n
nameserver 119.6.6.6<\/code><\/pre>\n
\u4fdd\u5b58\u540e\u6267\u884c<\/p>\n
resolvconf -u<\/code><\/pre>\n
\u67e5\u770bresolv.conf \u6587\u4ef6\u5c31\u53ef\u4ee5\u770b\u5230\u6211\u4eec\u7684\u8bbe\u7f6e\u5df2\u7ecf\u52a0\u4e0a<\/p>\n
cat \/etc\/resolv.conf<\/code><\/pre>\n
\u91cd\u542fresolv<\/p>\n
\/etc\/init.d\/resolvconf restart<\/code><\/pre>\n
\u7b2c\u4e8c\u79cd\u65b9\u6cd5\uff0c\u4e34\u65f6\u6539<\/h4>\n
\u4fee\u6539 \/etc\/resolv.conf\u6587\u4ef6\uff0c\u589e\u52a0<\/p>\n
nameserver 119.6.6.6<\/code><\/pre>\n
\u91cd\u542fresolv<\/p>\n
\/etc\/init.d\/resolvconf restart<\/code><\/pre>\n
\u4e8c\u3001Hadoop\u90e8\u7f72<\/h1>\n
2.1 Hadoop\u5b89\u88c5\uff08\u4e09\u53f0\u673a\u5668\u53ef\u540c\u6b65\u8fdb\u884c\uff09<\/h2>\n
\u4e0b\u8f7dhadoop2.7.7\uff08hadoop-2.7.7.tar.gz\uff09
\n\u89e3\u538b tar -zxvf hadoop-2.7.7.tar.gz \uff0c\u5e76\u5728\u4e3b\u76ee\u5f55\u4e0b\u521b\u5efatmp\u3001dfs\u3001dfs\/name\u3001dfs\/node\u3001dfs\/data<\/p>\n
cd \/opt\/hadoop-2.7.7\nmkdir tmp\nmkdir dfs\nmkdir dfs\/name\nmkdir dfs\/node\nmkdir dfs\/data<\/code><\/pre>\n
2.2 Hadoop\u914d\u7f6e<\/h2>\n
\u4ee5\u4e0b\u64cd\u4f5c\u90fd\u5728hadoop-2.7.7\/etc\/hadoop\u4e0b\u8fdb\u884c<\/p>\n
2.2.1 \u7f16\u8f91hadoop-env.sh\u6587\u4ef6\uff0c\u4fee\u6539JAVA_HOME\u914d\u7f6e\u9879\u4e3aJDK\u5b89\u88c5\u76ee\u5f55<\/h3>\n
export JAVA_HOME=\/opt\/jdk<\/code><\/pre>\n
2.2.2 \u7f16\u8f91core-site.xml\u6587\u4ef6\uff0c\u6dfb\u52a0\u4ee5\u4e0b\u5185\u5bb9<\/h3>\n
\u5176\u4e2dmaster\u4e3a\u8ba1\u7b97\u673a\u540d\uff0c\/opt\/hadoop-2.7.7\/tmp\u4e3a\u624b\u52a8\u521b\u5efa\u7684\u76ee\u5f55<\/p>\n
<configuration>\n <property>  \n  <name>fs.defaultFS<\/name>  \n  <value>hdfs:\/\/master:9000<\/value>  \n <\/property>  \n <property>  \n  <name>io.file.buffer.size<\/name>  \n  <value>131072<\/value>  \n <\/property>  \n <property>  \n  <name>hadoop.tmp.dir<\/name>  \n  <value>file:\/opt\/hadoop-2.7.7\/tmp<\/value>  \n  <description>Abasefor other temporary directories.<\/description>  \n <\/property>  \n <property>  \n  <name>hadoop.proxyuser.spark.hosts<\/name>  \n  <value>*<\/value>  \n <\/property>  \n<property>  \n  <name>hadoop.proxyuser.spark.groups<\/name>  \n  <value>*<\/value>  \n <\/property> \n<\/configuration><\/code><\/pre>\n
2.2.3 \u7f16\u8f91hdfs-site.xml\u6587\u4ef6\uff0c\u6dfb\u52a0\u4ee5\u4e0b\u5185\u5bb9<\/h3>\n
\u5176\u4e2dmaster\u4e3a\u8ba1\u7b97\u673a\u540d\uff0c
\nfile:\/opt\/hadoop-2.7.7\/dfs\/name\u548cfile:\/opt\/hadoop-2.7.7\/dfs\/data\u4e3a\u624b\u52a8\u521b\u5efa\u76ee\u5f55<\/p>\n
<configuration>\n<property>  \n  <name>dfs.namenode.secondary.http-address<\/name>  \n  <value>master:9001<\/value>  \n <\/property>  \n  <property>  \n   <name>dfs.namenode.name.dir<\/name>  \n   <value>file:\/opt\/hadoop-2.7.7\/dfs\/name<\/value>  \n <\/property>  \n <property>  \n  <name>dfs.datanode.data.dir<\/name>  \n  <value>file:\/opt\/hadoop-2.7.7\/dfs\/data<\/value>  \n  <\/property>  \n <property>  \n  <name>dfs.replication<\/name>  \n  <value>3<\/value>  \n <\/property>  \n <property>  \n  <name>dfs.webhdfs.enabled<\/name>  \n  <value>true<\/value>  \n <\/property>  \n<\/configuration><\/code><\/pre>\n
\u590d\u5236mapred-site.xml.template\u5e76\u91cd\u547d\u540d\u4e3amapred-site.xml<\/p>\n
cp mapred-site.xml.template mapred-site.xml<\/code><\/pre>\n
2.2.4 \u7f16\u8f91mapred-site.xml\u6587\u4ef6\uff0c\u6dfb\u52a0\u4ee5\u4e0b\u5185\u5bb9<\/h3>\n
\u5176\u4e2dmaster\u4e3a\u8ba1\u7b97\u673a\u540d<\/p>\n
<configuration>\n<property>  \n   <name>mapreduce.framework.name<\/name>  \n   <value>yarn<\/value>  \n <\/property>  \n <property>  \n  <name>mapreduce.jobhistory.address<\/name>  \n  <value>master:10020<\/value>  \n <\/property>  \n <property>  \n  <name>mapreduce.jobhistory.webapp.address<\/name>  \n  <value>master:19888<\/value>  \n <\/property>  \n<\/configuration><\/code><\/pre>\n
2.2.5 \u7f16\u8f91yarn-site.xml\u6587\u4ef6\uff0c\u6dfb\u52a0\u4ee5\u4e0b\u5185\u5bb9<\/h3>\n
\u5176\u4e2dmaster\u4e3a\u8ba1\u7b97\u673a\u540d<\/p>\n
<configuration>\n<!-- Site specific YARN configuration properties -->\n<property>  \n   <name>yarn.nodemanager.aux-services<\/name>  \n   <value>mapreduce_shuffle<\/value>  \n  <\/property>  \n  <property>  \n   <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class<\/name>  \n   <value>org.apache.hadoop.mapred.ShuffleHandler<\/value>  \n  <\/property>  \n  <property>  \n   <name>yarn.resourcemanager.address<\/name>  \n   <value>master:8032<\/value>  \n  <\/property>  \n  <property>  \n   <name>yarn.resourcemanager.scheduler.address<\/name>  \n   <value>master:8030<\/value>  \n  <\/property>  \n  <property>  \n   <name>yarn.resourcemanager.resource-tracker.address<\/name>  \n   <value>master:8035<\/value>  \n  <\/property>  \n  <property>  \n   <name>yarn.resourcemanager.admin.address<\/name>  \n   <value>master:8033<\/value>  \n  <\/property>  \n  <property>  \n   <name>yarn.resourcemanager.webapp.address<\/name>  \n   <value>master:8088<\/value>  \n  <\/property>  \n<\/configuration><\/code><\/pre>\n
2.2.6 \u4fee\u6539slaves\u6587\u4ef6\uff0c\u6dfb\u52a0\u96c6\u7fa4\u8282\u70b9\uff08\u591a\u673a\u6dfb\u52a0\u591a\u4e2a\uff09<\/h3>\n
\u6dfb\u52a0\u4ee5\u4e0b<\/p>\n
master\nslave1\nslave2<\/code><\/pre>\n
2.2.7 Hadoop\u96c6\u7fa4\u642d\u5efa<\/h3>\n
hadoop\u914d\u7f6e\u96c6\u7fa4\uff0c\u53ef\u4ee5\u5c06\u914d\u7f6e\u6587\u4ef6etc\/hadoop\u4e0b\u5185\u5bb9\u540c\u6b65\u5230\u5176\u4ed6\u673a\u5668\u4e0a\uff0c\u65e22.2.1-2.2.6\u65e0\u9700\u5728\u4e00\u4e2a\u4e2a\u914d\u7f6e\u3002<\/p>\n
cd \/opt\/hadoop-2.7.7\/etc\nscp -r hadoop root@\u53e6\u4e00\u53f0\u673a\u5668\u540d:\/opt\/hadoop-2.7.7\/etc<\/code><\/pre>\n
2.3 Hadoop\u542f\u52a8<\/h2>\n
1.\u683c\u5f0f\u5316\u4e00\u4e2a\u65b0\u7684\u6587\u4ef6\u7cfb\u7edf\uff0c\u8fdb\u5165\u5230hadoop-2.7.7\/bin\u4e0b\u6267\u884c\uff1a<\/h3>\n
.\/hadoop namenode -format<\/code><\/pre>\n
2.\u542f\u52a8hadoop\uff0c\u8fdb\u5165\u5230hadoop-2.7.7\/sbin\u4e0b\u6267\u884c\uff1a<\/h3>\n
.\/start-all.sh<\/code><\/pre>\n
\u770b\u5230\u5982\u4e0b\u5185\u5bb9\u8bf4\u660e\u542f\u52a8\u6210\u529f<\/p>\n
root@master:\/opt\/hadoop-2.7.7\/sbin# .\/start-all.sh\nThis script is Deprecated. Instead use start-dfs.sh and start-yarn.sh\nStarting namenodes on [master]\nmaster: starting namenode, logging to \/opt\/hadoop-2.7.7\/logs\/hadoop-root-namenode-master.out\nslave2: starting datanode, logging to \/opt\/hadoop-2.7.7\/logs\/hadoop-root-datanode-slave2.out\nmaster: starting datanode, logging to \/opt\/hadoop-2.7.7\/logs\/hadoop-root-datanode-master.out\nslave1: starting datanode, logging to \/opt\/hadoop-2.7.7\/logs\/hadoop-root-datanode-slave1.out\nStarting secondary namenodes [master]\nmaster: starting secondarynamenode, logging to \/opt\/hadoop-2.7.7\/logs\/hadoop-root-secondarynamenode-master.out\nstarting yarn daemons\nstarting resourcemanager, logging to \/opt\/hadoop-2.7.7\/logs\/yarn-root-resourcemanager-master.out\nslave2: starting nodemanager, logging to \/opt\/hadoop-2.7.7\/logs\/yarn-root-nodemanager-slave2.out\nslave1: starting nodemanager, logging to \/opt\/hadoop-2.7.7\/logs\/yarn-root-nodemanager-slave1.out\nmaster: starting nodemanager, logging to \/opt\/hadoop-2.7.7\/logs\/yarn-root-nodemanager-master.out<\/code><\/pre>\n
2.4 Hadoop\u96c6\u7fa4\u68c0\u67e5<\/h2>\n
\u65b9\u6cd51\uff1a\u68c0\u67e5hadoop\u96c6\u7fa4\uff0c\u8fdb\u5165hadoop-2.7.7\/bin\u4e0b\u6267\u884c<\/h3>\n
.\/hdfs dfsadmin -report<\/code><\/pre>\n
\u67e5\u770bLive datanodes \u8282\u70b9\u4e2a\u6570\uff0c\u4f8b\u5982\uff1aLive datanodes (3)\uff0c\u5219\u8868\u793a3\u53f0\u90fd\u542f\u52a8\u6210\u529f<\/p>\n
root@master:\/opt\/hadoop-2.7.7\/bin# .\/hdfs dfsadmin -report\nConfigured Capacity: 621051420672 (578.40 GB)\nPresent Capacity: 577317355520 (537.67 GB)\nDFS Remaining: 577317281792 (537.67 GB)\nDFS Used: 73728 (72 KB)\nDFS Used%: 0.00%\nUnder replicated blocks: 0\nBlocks with corrupt replicas: 0\nMissing blocks: 0\nMissing blocks (with replication factor 1): 0\n-------------------------------------------------\nLive datanodes (3):<\/code><\/pre>\n
\u65b9\u6cd52\uff1a\u8bbf\u95ee8088\u7aef\u53e3\uff0chttp:\/\/192.168.241.132:8088\/cluster\/nodes<\/a><\/h3>\n
\"\"<\/div><\/p>\n
\u65b9\u6cd53\uff1a\u8bbf\u95ee50070\u7aef\u53e3http:\/\/192.168.241.132:50070\/<\/h3>\n
\"\"<\/div><\/p>\n
\u4e09\u3001Spark\u90e8\u7f72<\/h1>\n
3.1 Spark\u5b89\u88c5\uff08\u4e09\u53f0\u673a\u5668\u53ef\u540c\u6b65\u8fdb\u884c\uff09<\/h2>\n
\u4e0b\u8f7dspark-2.1.0-bin-hadoop2.7.tgz\uff0c\u653e\u5230opt\u4e0b\u89e3\u538b\u3002\n\u5c06spark\u73af\u5883\u53d8\u91cf\u914d\u7f6e\u5230\/etc\/profile\u4e2d<\/code><\/pre>\n
export SPARK_HOME=\/opt\/spark-2.1.0-bin-hadoop2.7\nexport PATH=$JAVA_HOME\/bin:$SPARK_HOME\/bin:$PATH<\/code><\/pre>\n
3.2 Spark\u914d\u7f6e<\/h2>\n
1.\u8fdb\u5165spark-2.1.0-bin-hadoop2.7\/conf\u590d\u5236spark-env.sh.template\u5e76\u91cd\u547d\u540d\u4e3aspark-env.sh<\/h4>\n
cp spark-env.sh.template spark-env.sh<\/code><\/pre>\n
\u7f16\u8f91spark-env.sh\u6587\u4ef6\uff0c\u6dfb\u52a0\u4ee5\u4e0b\u5185\u5bb9<\/p>\n
export JAVA_HOME=\/opt\/jdk\nexport SPARK_MASTER_IP=192.168.241.132\nexport SPARK_WORKER_MEMORY=8g\nexport SPARK_WORKER_CORES=4\nexport SPARK_EXECUTOR_MEMORY=4g\nexport HADOOP_HOME=\/opt\/hadoop-2.7.7\/\nexport HADOOP_CONF_DIR=\/opt\/hadoop-2.7.7\/etc\/hadoop\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:\/opt\/jdk\/jre\/lib\/amd64<\/code><\/pre>\n
2.\u628aslaves.template\u62f7\u8d1d\u4e3aslaves,\u5e76\u7f16\u8f91 slaves\u6587\u4ef6<\/h4>\n
cp slaves.template slaves<\/code><\/pre>\n
\u7f16\u8f91slaves\u6587\u4ef6\uff0c\u6dfb\u52a0\u4ee5\u4e0b\u5185\u5bb9\uff08\u591a\u673a\u6dfb\u52a0\u591a\u4e2a\uff09<\/p>\n
master\nslave1\nslave2<\/code><\/pre>\n
3.3 \u914d\u7f6eSpark\u96c6\u7fa4<\/h2>\n
\u53ef\u4ee5\u5c06\u914d\u7f6e\u6587\u4ef6spark-2.1.0-bin-hadoop2.7\/conf\u4e0b\u5185\u5bb9\u540c\u6b65\u5230\u5176\u4ed6\u673a\u5668\u4e0a\uff0c\u65e23.2\u65e0\u9700\u5728\u4e00\u4e2a\u4e2a\u914d\u7f6e\u3002<\/p>\n
scp -r conf root@\u53e6\u4e00\u53f0\u673a\u5668\u540d:\/opt\/spark-2.1.0-bin-hadoop2.7<\/code><\/pre>\n
3.4 Spark\u542f\u52a8<\/h2>\n
\u542f\u52a8spark\uff0c\u8fdb\u5165spark-2.1.0-bin-hadoop2.7\/sbin\u4e0b\u6267\u884c<\/p>\n
.\/start-all.sh<\/code><\/pre>\n
\"\"<\/div>3.5 Spark\u96c6\u7fa4\u68c0\u67e5<\/h2>\n
\u8bbf\u95eehttp:\/\/192.168.241.134:8080\/<\/p>\n
\u6ce8\u610f\uff1a\u914d\u7f6eSpark\u96c6\u7fa4\uff0c\u9700\u8981\u4fdd\u8bc1\u5b50\u8282\u70b9\u5185\u5bb9\u548c\u4e3b\u8282\u70b9\u5185\u5bb9\u4e00\u81f4\u3002<\/p>\n
\u8fd9\u6837Hadoop\u96c6\u7fa4\u548cSpark\u96c6\u7fa4\u5c31\u90fd\u642d\u5efa\u597d\u4e86\u3002
\n\u8f6c\u8f7d\uff1ahttps:\/\/www.cnblogs.com\/zhangyongli2011\/p\/10572152.html<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"
\u4e00\u3001\u51c6\u5907 1.1 \u8f6f\u4ef6\u7248\u672c Ubuntu 16.04.6 (ubuntu-16.04.6-server-amd […]<\/p>\n","protected":false},"author":3,"featured_media":6500,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":[],"categories":[411],"tags":[],"_links":{"self":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts\/6499"}],"collection":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/users\/3"}],"replies":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=6499"}],"version-history":[{"count":0,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts\/6499\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/media\/6500"}],"wp:attachment":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=6499"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=6499"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=6499"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}