{"id":6463,"date":"2022-04-08T20:36:38","date_gmt":"2022-04-08T12:36:38","guid":{"rendered":"https:\/\/egonlin.com\/?p=6463"},"modified":"2022-04-08T20:38:09","modified_gmt":"2022-04-08T12:38:09","slug":"hadoop%e5%8e%9f%e7%90%86%e5%88%86%e6%9e%90","status":"publish","type":"post","link":"https:\/\/egonlin.com\/?p=6463","title":{"rendered":"hadoop\u539f\u7406\u5206\u6790"},"content":{"rendered":"

hadoop\u539f\u7406\u5206\u6790\uff1a\u5f3a\u70c8\u63a8\u8350\u9605\u8bfb\uff1ahttps:\/\/www.saoniuhuo.com\/article\/detail-1124.html<\/a><\/p>\n

hadoop\u96c6\u7fa4<\/h1>\n

\"\"<\/div><\/p>\n

\u5982\u4e0a\u56fe\u6240\u793a\uff0cHDFS\u4e5f\u662f\u6309\u7167Master\u548cSlave\u7684\u7ed3\u6784\u3002\u5206NameNode\u3001SecondaryNameNode\u3001DataNode\u8fd9\u51e0\u4e2a\u89d2\u8272\u3002<\/p>\n

1\u3001NameNode\uff1a<\/h2>\n

\u662fMaster\u8282\u70b9\uff0c\u662f\u5927\u9886\u5bfc\u3002\u7ba1\u7406\u6570\u636e\u5757\u6620\u5c04\uff1b\u5904\u7406\u5ba2\u6237\u7aef\u7684\u8bfb\u5199\u8bf7\u6c42\uff1b\u914d\u7f6e\u526f\u672c\u7b56\u7565\uff1b\u7ba1\u7406HDFS\u7684\u540d\u79f0\u7a7a\u95f4\uff1b<\/p>\n

nameNode\u6982\u8ff0\n    (1):nameNode\u662fhdfs\u7684\u6838\u5fc3.\n    (2):nameNode\u4e5f\u79f0\u4e3aMaster\n    (3):nameNode\u5b58\u50a8hdfs\u7684\u6570\u636e,\u6587\u4ef6\u7cfb\u7edf\u4e2d\u6240\u6709\u7684\u6587\u4ef6\u76ee\u5f55\u6811,\u5e76\u8ddf\u8e2a\u6574\u4e2a\u96c6\u7fa4\u4e2d\u7684\u6587\u4ef6\n    (4):nameNode\u4e0d\u5b58\u50a8\u5b9e\u9645\u6570\u636e\u6216\u6570\u636e\u96c6.\u6570\u636e\u672c\u8eab\u6570\u636e\u5b58\u50a8\u5728dataNode\u4e2d.\n    (5):nameNode\u77e5\u9053Hdfs\u4e2d\u4efb\u4f55\u7ed9\u5b9a\u6587\u4ef6\u7684\u5757\u5217\u8868\u53ca\u5176\u4f4d\u7f6e.\u4f7f\u7528\u6b64\u4fe1\u606fnameNode\u77e5\u9053\u5982\u4f55\u4ece\u5757\u4e2d\u6784\u5efa\u6587\u4ef6\n    (6):nameNode\u5e76\u4e0d\u6301\u4e45\u5316\u5b58\u50a8\u6bcf\u4e2a\u6587\u4ef6\u4e2d\u5404\u4e2a\u5757\u6240\u5728\u7684dataNode\u7684\u4f4d\u7f6e\u4fe1\u606f,\u8fd9\u4e9b\u4fe1\u606f\u4f1a\u5728\u7cfb\u7edf\u542f\u52a8\u65f6\u4ece\u5728\u6570\u636e\u8282\u70b9\u91cd\u5efa.\n    (7):nameNode\u5bf9\u4e8ehdfs\u5f88\u91cd\u8981,\u5f53nameNode\u5173\u95ed\u65f6,hdfs\/hadoop\u96c6\u7fa4\u65e0\u6cd5\u8bbf\u95ee.\n    (8):nameNode\u662fhadoop\u96c6\u7fa4\u4e2d\u7684\u5355\u70b9\u6545\u969c\n    (9):nameNode\u6240\u5728\u673a\u5668\u901a\u5e38\u4f1a\u914d\u7f6e\u6709\u5927\u91cf\u5185\u5b58(RAM).<\/code><\/pre>\n
2\u3001SecondaryNameNode\uff1a<\/h2>\n
\u662f\u4e00\u4e2a\u5c0f\u5f1f\uff0c\u5206\u62c5\u5927\u54e5namenode\u7684\u5de5\u4f5c\u91cf\uff1b\u662fNameNode\u7684\u51b7\u5907\u4efd\uff1b\u5408\u5e76fsimage\u548cfsedits\u7136\u540e\u518d\u53d1\u7ed9namenode\u3002<\/p>\n
\u3000\u3000SecondaryNameNode\u8d1f\u8d23\u5b9a\u65f6\u9ed8\u8ba41\u5c0f\u65f6\uff0c\u4ecenamenode\u4e0a\uff0c\u83b7\u53d6fsimage\u548cedits\u6765\u8fdb\u884c\u5408\u5e76\uff0c\u7136\u540e\u518d\u53d1\u9001\u7ed9namenode\u3002\u51cf\u5c11namenode\u7684\u5de5\u4f5c\u91cf\u3002<\/p>\n
\u3000\u3000fsimage:\u5143\u6570\u636e\u955c\u50cf\u6587\u4ef6\uff08\u6587\u4ef6\u7cfb\u7edf\u7684\u76ee\u5f55\u6811\u3002\uff09<\/p>\n
\u3000\u3000edits\uff1a\u5143\u6570\u636e\u7684\u64cd\u4f5c\u65e5\u5fd7\uff08\u9488\u5bf9\u6587\u4ef6\u7cfb\u7edf\u505a\u7684\u4fee\u6539\u64cd\u4f5c\u8bb0\u5f55\uff09<\/p>\n
\u3000\u3000namenode\u5185\u5b58\u4e2d\u5b58\u50a8\u7684\u662f=fsimage+edits\u3002<\/p>\n
3\u3001DataNode\uff1a<\/h2>\n
Slave\u8282\u70b9\uff0c\u5974\u96b6\uff0c\u5e72\u6d3b\u7684\u3002\u8d1f\u8d23\u5b58\u50a8client\u53d1\u6765\u7684\u6570\u636e\u5757block\uff1b\u6267\u884c\u6570\u636e\u5757\u7684\u8bfb\u5199\u64cd\u4f5c\u3002<\/p>\n
dataNode\u6982\u8ff0\n    (1):dataNode\u8d1f\u8d23\u5c06\u6570\u636e\u5b58\u50a8\u5728Hdfs\u4e2d\n    (2):dataNode\u4e5f\u79f0\u4e3aslave.\n    (3):nameNode\u548cdataNode\u4f1a\u4fdd\u6301\u4e0d\u65ad\u901a\u4fe1\n    (4):dataNode\u542f\u52a8\u65f6,\u4ed6\u5c06\u81ea\u5df1\u53d1\u5e03\u5230nameNode\u5e76\u6c47\u62a5\u81ea\u5df1\u8d1f\u8d23\u6301\u6709\u7684\u5757\u5217\u8868\n    (5):\u5f53\u67d0\u4e2anameNode\u5173\u95ed\u65f6,\u4ed6\u4e0d\u4f1a\u5f71\u54cd\u6570\u636e\u6216\u8005\u96c6\u7fa4\u7684\u53ef\u7528\u6027.namnode\u5c06\u5b89\u6392\u7531\u5176\u4ed6DataNode\u7ba1\u7406\u7684\u5757\u526f\u672c\u590d\u5236\n    (6):DataNode\u6240\u5728\u673a\u5668\u901a\u5e38\u914d\u7f6e\u6709\u5927\u91cf\u7684\u786c\u76d8\u7a7a\u95f4.\u56e0\u4e3a\u5b9e\u9645\u6570\u636e\u5b58\u50a8\u5728dataNode\u4e2d.\n    (7):dataNode\u4f1a\u5b9a\u671f(dfs.heartbeat.interval\u914d\u7f6e\u9879\u914d\u7f6e,\u9ed8\u8ba4\u662f3\u79d2)\u5411nameNode\u53d1\u9001\u5fc3\u8df3.\u5982\u679cnameNode\u957f\u65f6\u95f4\u6ca1\u6709\u63a5\u6536\u5230dataNode\u53d1\u9001\u7684\u5fc3\u8df3,nameNode\u5c31\u4f1a\u8ba4\u4e3a\u8be5DataNode\u5931\u6548.\n    (8):block\u6c47\u62a5\u65f6\u95f4\u95f4\u9694\u53d6\u53c2\u6570 dfs.blockreport.intersec,\u53c2\u6570\u672a\u914d\u7f6e\u9ed8\u8ba4\u4e3a6\u5c0f\u65f6<\/code><\/pre>\n
4\u3001\u70ed\u5907\u4efd\uff1a<\/h2>\n
b\u662fa\u7684\u70ed\u5907\u4efd\uff0c\u5982\u679ca\u574f\u6389\u3002\u90a3\u4e48b\u9a6c\u4e0a\u8fd0\u884c\u4ee3\u66ffa\u7684\u5de5\u4f5c\u3002<\/p>\n
5\u3001\u51b7\u5907\u4efd\uff1a<\/h2>\n
b\u662fa\u7684\u51b7\u5907\u4efd\uff0c\u5982\u679ca\u574f\u6389\u3002\u90a3\u4e48b\u4e0d\u80fd\u9a6c\u4e0a\u4ee3\u66ffa\u5de5\u4f5c\u3002\u4f46\u662fb\u4e0a\u5b58\u50a8a\u7684\u4e00\u4e9b\u4fe1\u606f\uff0c\u51cf\u5c11a\u574f\u6389\u4e4b\u540e\u7684\u635f\u5931\u3002<\/p>\n
6\u3001HDFS<\/h2>\n
\uff08Hadoop Distributed File System \uff09Hadoop\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf\u3002\u662f\u6839\u636egoogle\u53d1\u8868\u7684\u8bba\u6587\u7ffb\u7248\u7684\u3002\u8bba\u6587\u4e3aGFS\uff08Google File System\uff09Google \u6587\u4ef6\u7cfb\u7edf\uff08\u4e2d\u6587\uff0c\u82f1\u6587\uff09\u3002<\/p>\n
HDFS\u6709\u5f88\u591a\u7279\u70b9\uff1a<\/h5>\n
\u2460 \u4fdd\u5b58\u591a\u4e2a\u526f\u672c\uff0c\u4e14\u63d0\u4f9b\u5bb9\u9519\u673a\u5236\uff0c\u526f\u672c\u4e22\u5931\u6216\u5b95\u673a\u81ea\u52a8\u6062\u590d\u3002\u9ed8\u8ba4\u5b583\u4efd\u3002\n\n\u2461 \u8fd0\u884c\u5728\u5ec9\u4ef7\u7684\u673a\u5668\u4e0a\u3002\n\n\u2462 \u9002\u5408\u5927\u6570\u636e\u7684\u5904\u7406\u3002\u591a\u5927\uff1f\u591a\u5c0f\uff1fHDFS\u9ed8\u8ba4\u4f1a\u5c06\u6587\u4ef6\u5206\u5272\u6210block\uff0c64M\u4e3a1\u4e2ablock\u3002\u7136\u540e\u5c06block\u6309\u952e\u503c\u5bf9\u5b58\u50a8\u5728HDFS\u4e0a\uff0c\u5e76\u5c06\u952e\u503c\u5bf9\u7684\u6620\u5c04\u5b58\u5230\u5185\u5b58\u4e2d\u3002\u5982\u679c\u5c0f\u6587\u4ef6\u592a\u591a\uff0c\u90a3\u5185\u5b58\u7684\u8d1f\u62c5\u4f1a\u5f88\u91cd\nhdfs\u8bfb\u53d6\u6587\u4ef6\u6d41\u7a0b\n    (1):Client\u5411nameNode\u53d1\u8d77rpc\u8bf7\u6c42,\u6765\u786e\u5b9a\u8bf7\u6c42\u6587\u4ef6block\u6240\u5728\u7684\u4f4d\u7f6e;\n    (2):NameNode\u4f1a\u89c6\u60c5\u51b5\u8fd4\u56de\u6587\u4ef6\u7684\u90e8\u5206\u6216\u8005\u5168\u90e8block\u5217\u8868,\u5bf9\u4e8e\u6bcf\u4e2ablock,nameNode\u90fd\u4f1a\u8fd4\u56de\u542b\u6709\u8be5block\u526f\u672c\u7684dataNode\u5730\u5740\n    (3):\u8fd9\u4e9b\u8fd4\u56de\u7684DN\u5730\u5740,\u4f1a\u6309\u7167\u96c6\u7fa4\u62d3\u6251\u7ed3\u6784\u5f97\u51fadataNode\u4e0e\u5ba2\u6237\u7aef\u7684\u8ddd\u79bb,\u7136\u540e\u8fdb\u884c\u6392\u5e8f,\u6392\u5e8f\u4e24\u4e2a\u89c4\u6d4b,\u7f51\u7edc\u62d3\u6251\u7ed3\u6784\u4e2d\u8ddd\u79bbclient\u8fd1\u7684\u6392\u9760\u524d;\u5fc3\u8df3\u673a\u5236\u4e2d\u8d85\u65f6\u6c47\u62a5\u7684DN\u72b6\u6001\u4e3aSTALE,\u8fd9\u6837\u7684\u6392\u9760\u540e;\n    (4):client\u9009\u53d6\u6392\u5e8f\u9760\u524d\u7684dataNode\u6765\u8bfb\u53d6block,\u5982\u679c\u5ba2\u6237\u7aef\u672c\u8eab\u5c31\u662fdataNode,\u90a3\u4e48\u5c06\u4ece\u672c\u5730\u76f4\u63a5\u83b7\u53d6\u6570\u636e.\n    (5):\u5e95\u5c42\u4e0a\u672c\u8d28\u662f\u5efa\u7acbsocket stream (FSDataInputStream),\u91cd\u590d\u7684\u8c03\u7528\u7236\u7c7b\u7684dataInputStream\u7684read\u65b9\u6cd5,\u76f4\u5230\u8fd9\u4e2a\u5feb\u4e0a\u5f97\u6570\u636e\u8bfb\u53d6\u5b8c\u6bd5;\n    (6):\u5f53\u8bfb\u5b8c\u5217\u8868\u7684block\u540e,\u82e5\u6587\u4ef6\u8bfb\u53d6\u8fd8\u6ca1\u6709\u7ed3\u675f,\u5ba2\u6237\u7aef\u4f1a\u7ee7\u7eed\u5411nameNode\u83b7\u53d6\u4e0b\u4e00\u6279block\u5217\u8868;\n    (7):\u8bfb\u53d6\u5b8c\u4e00\u4e2ablock\u90fd\u4f1a\u8fdb\u884cchecksum\u9a8c\u8bc1,\u5982\u679c\u8bfb\u53d6dataNode\u65f6\u51fa\u73b0\u9519\u8bef,\u5ba2\u6237\u7aef\u4f1a\u901a\u77e5nameNode,\u7136\u540e\u5728\u4ece\u4e0b\u4e00\u4e2a\u62e5\u6709\u8be5block\u526f\u672c\u7684\u5f00\u59cb\nhdsf\u4e0a\u4f20\u6587\u4ef6<\/code><\/pre>\n
\u8be6\u7ec6\u539f\u7406\u89c1
\nhttps:\/\/www.saoniuhuo.com\/article\/detail-1124.html<\/a><\/p>\n
https:\/\/www.cnblogs.com\/liuwei6\/p\/6680188.html<\/a><\/p>\n
https:\/\/www.cnblogs.com\/laov\/p\/3434917.html<\/a><\/p>\n
hdfs\u539f\u7406\u751f\u52a8\u4ecb\u7ecd\uff1ahttps:\/\/bbs.huaweicloud.com\/blogs\/218062<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"
hadoop\u539f\u7406\u5206\u6790\uff1a\u5f3a\u70c8\u63a8\u8350\u9605\u8bfb\uff1ahttps:\/\/www.saoniuhuo.com\/article\/det […]<\/p>\n","protected":false},"author":3,"featured_media":6464,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":[],"categories":[411],"tags":[],"_links":{"self":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts\/6463"}],"collection":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/users\/3"}],"replies":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=6463"}],"version-history":[{"count":0,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts\/6463\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/media\/6464"}],"wp:attachment":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=6463"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=6463"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=6463"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}