{"id":4562,"date":"2022-03-27T11:09:46","date_gmt":"2022-03-27T03:09:46","guid":{"rendered":"https:\/\/egonlin.com\/?p=4562"},"modified":"2023-02-02T17:50:20","modified_gmt":"2023-02-02T09:50:20","slug":"%e7%ac%ac%e4%b8%80%e7%ab%a0%ef%bc%9a%e7%88%ac%e8%99%ab%e5%9f%ba%e6%9c%ac%e5%8e%9f%e7%90%86","status":"publish","type":"post","link":"https:\/\/egonlin.com\/?p=4562","title":{"rendered":"\u7b2c\u4e00\u7ae0\uff1a\u722c\u866b\u57fa\u672c\u539f\u7406"},"content":{"rendered":"

\u4e00 \u722c\u866b\u662f\u4ec0\u4e48<\/h1>\n

\"file\"<\/div><\/p>\n

1\u3001\u4ec0\u4e48\u662f\u4e92\u8054\u7f51\uff1f<\/h2>\n
\u4e92\u8054\u7f51\u662f\u7531\u7f51\u7edc\u8bbe\u5907\uff08\u7f51\u7ebf\uff0c\u8def\u7531\u5668\uff0c\u4ea4\u6362\u673a\uff0c\u9632\u706b\u5899\u7b49\u7b49\uff09\u548c\u4e00\u53f0\u53f0\u8ba1\u7b97\u673a\u8fde\u63a5\u800c\u6210\uff0c\u50cf\u4e00\u5f20\u7f51\u4e00\u6837\u3002<\/code><\/pre>\n
2\u3001\u4e92\u8054\u7f51\u5efa\u7acb\u7684\u76ee\u7684\uff1f<\/h2>\n
\u4e92\u8054\u7f51\u7684\u6838\u5fc3\u4ef7\u503c\u5728\u4e8e\u6570\u636e\u7684\u5171\u4eab\/\u4f20\u9012\uff1a\u6570\u636e\u662f\u5b58\u653e\u4e8e\u4e00\u53f0\u53f0\u8ba1\u7b97\u673a\u4e0a\u7684\uff0c\u800c\u5c06\u8ba1\u7b97\u673a\u4e92\u8054\u5230\u4e00\u8d77\u7684\u76ee\u7684\u5c31\u662f\u4e3a\u4e86\u80fd\u591f\u65b9\u4fbf\u5f7c\u6b64\u4e4b\u95f4\u7684\u6570\u636e\u5171\u4eab\/\u4f20\u9012\uff0c\u5426\u5219\u4f60\u53ea\u80fd\u62ffU\u76d8\u53bb\u522b\u4eba\u7684\u8ba1\u7b97\u673a\u4e0a\u62f7\u8d1d\u6570\u636e\u4e86\u3002<\/code><\/pre>\n
3\u3001\u4ec0\u4e48\u662f\u4e0a\u7f51\uff1f\u722c\u866b\u8981\u505a\u7684\u662f\u4ec0\u4e48\uff1f<\/h2>\n
3.1 \u6211\u4eec\u6240\u8c13\u7684\u4e0a\u7f51\u4fbf\u662f\u7531\u7528\u6237\u7aef\u8ba1\u7b97\u673a\u53d1\u9001\u8bf7\u6c42\u7ed9\u76ee\u6807\u8ba1\u7b97\u673a\uff0c\u5c06\u76ee\u6807\u8ba1\u7b97\u673a\u7684\u6570\u636e\u4e0b\u8f7d\u5230\u672c\u5730\u7684\u8fc7\u7a0b\u3002\u53ea\u4e0d\u8fc7\uff0c\u7528\u6237\u83b7\u53d6\u7f51\u7edc\u6570\u636e\u7684\u65b9\u5f0f\u662f\uff1a\n      \u6d4f\u89c8\u5668\u63d0\u4ea4\u8bf7\u6c42->\u4e0b\u8f7d\u7f51\u9875\u4ee3\u7801->\u89e3\u6790\/\u6e32\u67d3\u6210\u9875\u9762\u3002\n\n3.2 \u800c\u722c\u866b\u7a0b\u5e8f\u8981\u505a\u7684\u5c31\u662f\uff1a\n  \u6a21\u62df\u6d4f\u89c8\u5668\u53d1\u9001\u8bf7\u6c42->\u4e0b\u8f7d\u7f51\u9875\u4ee3\u7801->\u53ea\u63d0\u53d6\u6709\u7528\u7684\u6570\u636e->\u5b58\u653e\u4e8e\u6570\u636e\u5e93\u6216\u6587\u4ef6\u4e2d\n\u3000\n3.1\u4e0e3.2\u7684\u533a\u522b\u5728\u4e8e:\n\n      \u6211\u4eec\u7684\u722c\u866b\u7a0b\u5e8f\u53ea\u63d0\u53d6\u7f51\u9875\u4ee3\u7801\u4e2d\u5bf9\u6211\u4eec\u6709\u7528\u7684\u6570\u636e\n\n4\u3001\u603b\u7ed3\u722c\u866b\n   \u722c\u866b\u7684\u6bd4\u55bb\uff1a\u5982\u679c\u6211\u4eec\u628a\u4e92\u8054\u7f51\u6bd4\u4f5c\u4e00\u5f20\u5927\u7684\u8718\u86db\u7f51\uff0c\u90a3\u4e00\u53f0\u8ba1\u7b97\u673a\u4e0a\u7684\u6570\u636e\u4fbf\u662f\u8718\u86db\u7f51\u4e0a\u7684\u4e00\u4e2a\u730e\u7269\uff0c\u800c\u722c\u866b\u7a0b\u5e8f\u5c31\u662f\u4e00\u53ea\u5c0f\u8718\u86db\uff0c\u6cbf\u7740\u8718\u86db\u7f51\u6293\u53d6\u81ea\u5df1\u60f3\u8981\u7684\u730e\u7269\/\u6570\u636e\n<\/code><\/pre>\n
4.2 \u722c\u866b\u7684\u5b9a\u4e49\uff1a<\/h2>\n
      \u5411\u7f51\u7ad9\u53d1\u8d77\u8bf7\u6c42\uff0c\u83b7\u53d6\u8d44\u6e90\u540e\u5206\u6790\u5e76\u63d0\u53d6\u6709\u7528\u6570\u636e\u7684\u7a0b\u5e8f <\/code><\/pre>\n
4.3 \u722c\u866b\u7684\u4ef7\u503c\uff1a<\/h2>\n
\u4e92\u8054\u7f51\u4e2d\u6700\u6709\u4ef7\u503c\u7684\u4fbf\u662f\u6570\u636e\uff0c\u6bd4\u5982\u5929\u732b\u5546\u57ce\u7684\u5546\u54c1\u4fe1\u606f\uff0c\u94fe\u5bb6\u7f51\u7684\u79df\u623f\u4fe1\u606f\uff0c\u96ea\u7403\u7f51\u7684\u8bc1\u5238\u6295\u8d44\u4fe1\u606f\u7b49\u7b49\uff0c\u8fd9\u4e9b\u6570\u636e\u90fd\u4ee3\u8868\u4e86\u5404\u4e2a\u884c\u4e1a\u7684\u771f\u91d1\u767d\u94f6\uff0c\u53ef\u4ee5\u8bf4\uff0c\u8c01\u638c\u63e1\u4e86\u884c\u4e1a\u5185\u7684\u7b2c\u4e00\u624b\u6570\u636e\uff0c\u8c01\u5c31\u6210\u4e86\u6574\u4e2a\u884c\u4e1a\u7684\u4e3b\u5bb0\uff0c\u5982\u679c\u628a\u6574\u4e2a\u4e92\u8054\u7f51\u7684\u6570\u636e\u6bd4\u55bb\u4e3a\u4e00\u5ea7\u5b9d\u85cf\uff0c\u90a3\u6211\u4eec\u7684\u722c\u866b\u8bfe\u7a0b\u5c31\u662f\u6765\u6559\u5927\u5bb6\u5982\u4f55\u6765\u9ad8\u6548\u5730\u6316\u6398\u8fd9\u4e9b\u5b9d\u85cf\uff0c\u638c\u63e1\u4e86\u722c\u866b\u6280\u80fd\uff0c\u4f60\u5c31\u6210\u4e86\u6240\u6709\u4e92\u8054\u7f51\u4fe1\u606f\u516c\u53f8\u5e55\u540e\u7684\u8001\u677f\uff0c\u6362\u8a00\u4e4b\uff0c\u5b83\u4eec\u90fd\u5728\u514d\u8d39\u4e3a\u4f60\u63d0\u4f9b\u6709\u4ef7\u503c\u7684\u6570\u636e\u3002<\/p>\n
\u4e8c \u722c\u866b\u7684\u57fa\u672c\u6d41\u7a0b<\/h1>\n
\"file\"<\/div><\/p>\n
1\u3001\u53d1\u8d77\u8bf7\u6c42<\/h2>\n
\u4f7f\u7528http\u5e93\u5411\u76ee\u6807\u7ad9\u70b9\u53d1\u8d77\u8bf7\u6c42\uff0c\u5373\u53d1\u9001\u4e00\u4e2aRequest\nRequest\u5305\u542b\uff1a\u8bf7\u6c42\u5934\u3001\u8bf7\u6c42\u4f53\u7b49<\/code><\/pre>\n
2 \u3001\u83b7\u53d6\u54cd\u5e94\u5185\u5bb9<\/h2>\n
\u5982\u679c\u670d\u52a1\u5668\u80fd\u6b63\u5e38\u54cd\u5e94\uff0c\u5219\u4f1a\u5f97\u5230\u4e00\u4e2aResponse\nResponse\u5305\u542b\uff1ahtml\uff0cjson\uff0c\u56fe\u7247\uff0c\u89c6\u9891\u7b49<\/code><\/pre>\n
3\u3001\u89e3\u6790\u5185\u5bb9<\/h2>\n
\u89e3\u6790html\u6570\u636e\uff1a\u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u7b2c\u4e09\u65b9\u89e3\u6790\u5e93\u5982Beautifulsoup\uff0cpyquery\u7b49\n\u89e3\u6790json\u6570\u636e\uff1ajson\u6a21\u5757\n\u89e3\u6790\u4e8c\u8fdb\u5236\u6570\u636e:\u4ee5b\u7684\u65b9\u5f0f\u5199\u5165\u6587\u4ef6<\/code><\/pre>\n
4 \u3001\u4fdd\u5b58\u6570\u636e<\/h2>\n
\u6570\u636e\u5e93\n\u6587\u4ef6<\/code><\/pre>\n
\u4e09 \u8bf7\u6c42\u4e0e\u54cd\u5e94<\/h1>\n
\"file\"<\/div><\/p>\n
#http\u534f\u8bae\uff1ahttp:\/\/www.cnblogs.com\/linhaifeng\/articles\/8243379.html\n\n#Request\uff1a\u7528\u6237\u5c06\u81ea\u5df1\u7684\u4fe1\u606f\u901a\u8fc7\u6d4f\u89c8\u5668\uff08socket client\uff09\u53d1\u9001\u7ed9\u670d\u52a1\u5668\uff08socket server\uff09\n\n#Response\uff1a\u670d\u52a1\u5668\u63a5\u6536\u8bf7\u6c42\uff0c\u5206\u6790\u7528\u6237\u53d1\u6765\u7684\u8bf7\u6c42\u4fe1\u606f\uff0c\u7136\u540e\u8fd4\u56de\u6570\u636e\uff08\u8fd4\u56de\u7684\u6570\u636e\u4e2d\u53ef\u80fd\u5305\u542b\u5176\u4ed6\u94fe\u63a5\uff0c\u5982\uff1a\u56fe\u7247\uff0cjs\uff0ccss\u7b49\uff09\n\n#ps\uff1a\u6d4f\u89c8\u5668\u5728\u63a5\u6536Response\u540e\uff0c\u4f1a\u89e3\u6790\u5176\u5185\u5bb9\u6765\u663e\u793a\u7ed9\u7528\u6237\uff0c\u800c\u722c\u866b\u7a0b\u5e8f\u5728\u6a21\u62df\u6d4f\u89c8\u5668\u53d1\u9001\u8bf7\u6c42\u7136\u540e\u63a5\u6536Response\u540e\uff0c\u662f\u8981\u63d0\u53d6\u5176\u4e2d\u7684\u6709\u7528\u6570\u636e\u3002<\/code><\/pre>\n
\u56db Request<\/h1>\n
1\u3001\u8bf7\u6c42\u65b9\u5f0f\uff1a<\/h2>\n
\u5e38\u7528\u7684\u8bf7\u6c42\u65b9\u5f0f\uff1aGET\uff0cPOST\n\u5176\u4ed6\u8bf7\u6c42\u65b9\u5f0f\uff1aHEAD\uff0cPUT\uff0cDELETE\uff0cOPTHONS<\/code><\/pre>\n
ps\uff1a\u7528\u6d4f\u89c8\u5668\u6f14\u793aget\u4e0epost\u7684\u533a\u522b\uff0c\uff08\u7528\u767b\u5f55\u6f14\u793apost\uff09\n\npost\u4e0eget\u8bf7\u6c42\u6700\u7ec8\u90fd\u4f1a\u62fc\u63a5\u6210\u8fd9\u79cd\u5f62\u5f0f\uff1ak1=xxx&k2=yyy&k3=zzz\npost\u8bf7\u6c42\u7684\u53c2\u6570\u653e\u5728\u8bf7\u6c42\u4f53\u5185\uff1a\n    \u53ef\u7528\u6d4f\u89c8\u5668\u67e5\u770b\uff0c\u5b58\u653e\u4e8eform data\u5185\nget\u8bf7\u6c42\u7684\u53c2\u6570\u76f4\u63a5\u653e\u5728url\u540e<\/code><\/pre>\n
2\u3001\u8bf7\u6c42url<\/h2>\n
url\u5168\u79f0\u7edf\u4e00\u8d44\u6e90\u5b9a\u4f4d\u7b26\uff0c\u5982\u4e00\u4e2a\u7f51\u9875\u6587\u6863\uff0c\u4e00\u5f20\u56fe\u7247\n\u4e00\u4e2a\u89c6\u9891\u7b49\u90fd\u53ef\u4ee5\u7528url\u552f\u4e00\u6765\u786e\u5b9a<\/code><\/pre>\n
url\u7f16\u7801\nhttps:\/\/www.baidu.com\/s?wd=\u56fe\u7247\n\u56fe\u7247\u4f1a\u88ab\u7f16\u7801\uff08\u770b\u793a\u4f8b\u4ee3\u7801\uff09\n\u7f51\u9875\u7684\u52a0\u8f7d\u8fc7\u7a0b\u662f\uff1a\n\u52a0\u8f7d\u4e00\u4e2a\u7f51\u9875\uff0c\u901a\u5e38\u90fd\u662f\u5148\u52a0\u8f7ddocument\u6587\u6863\uff0c\n\u5728\u89e3\u6790document\u6587\u6863\u7684\u65f6\u5019\uff0c\u9047\u5230\u94fe\u63a5\uff0c\u5219\u9488\u5bf9\u8d85\u94fe\u63a5\u53d1\u8d77\u4e0b\u8f7d\u56fe\u7247\u7684\u8bf7\u6c42<\/code><\/pre>\n
3\u3001\u8bf7\u6c42\u5934<\/h2>\n
User-agent\uff1a\u8bf7\u6c42\u5934\u4e2d\u5982\u679c\u6ca1\u6709user-agent\u5ba2\u6237\u7aef\u914d\u7f6e\uff0c\n\u670d\u52a1\u7aef\u53ef\u80fd\u5c06\u4f60\u5f53\u505a\u4e00\u4e2a\u975e\u6cd5\u7528\u6237\nhost\ncookies\uff1acookie\u7528\u6765\u4fdd\u5b58\u767b\u5f55\u4fe1\u606f<\/code><\/pre>\n
\u4e00\u822c\u505a\u722c\u866b\u90fd\u4f1a\u52a0\u4e0a\u8bf7\u6c42\u5934<\/code><\/pre>\n
4\u3001\u8bf7\u6c42\u4f53<\/h2>\n
\u5982\u679c\u662fget\u65b9\u5f0f\uff0c\u8bf7\u6c42\u4f53\u6ca1\u6709\u5185\u5bb9\n\u5982\u679c\u662fpost\u65b9\u5f0f\uff0c\u8bf7\u6c42\u4f53\u662fformat data<\/code><\/pre>\n
ps\uff1a\n1\u3001\u767b\u5f55\u7a97\u53e3\uff0c\u6587\u4ef6\u4e0a\u4f20\u7b49\uff0c\u4fe1\u606f\u90fd\u4f1a\u88ab\u9644\u52a0\u5230\u8bf7\u6c42\u4f53\u5185\n2\u3001\u767b\u5f55\uff0c\u8f93\u5165\u9519\u8bef\u7684\u7528\u6237\u540d\u5bc6\u7801\uff0c\u7136\u540e\u63d0\u4ea4\uff0c\u5c31\u53ef\u4ee5\u770b\u5230post\uff0c\u6b63\u786e\u767b\u5f55\u540e\u9875\u9762\u901a\u5e38\u4f1a\u8df3\u8f6c\uff0c\u65e0\u6cd5\u6355\u6349\u5230post <\/code><\/pre>\n
from urllib.parse import urlencode\nimport requests\n\nheaders={\n'Accept':'text\/html,application\/xhtml+xml,application\/xml;q=0.9,image\/webp,image\/apng,*\/*;q=0.8',\n'Cookie':'H_WISE_SIDS=101556_115474_115442_114743_108373_100099_115725_106201_107320_115339_114797_115553_116093_115546_115625_115446_114329_115350_114275_116040_110085; PSTM=1494300712; BAIDUID=42FE2934E37AF7AD1FA31D8CC7006D45:FG=1; BIDUPSID=2996557DB2710279BD865C50F9A68615; MCITY=-%3A; __cfduid=da9f97dea6458ca26aa4278280752ebb01508939712; BDSFRCVID=PGLsJeCCxG3wt_3ZUrBLDfv2D_qBZSjAgcEe3J; H_BDCLCKID_SF=tJAOoCLytI03qn5zq4Oh-4oHhxoJq5QxbT7Z0l8KtfcNVJQs-lCMhbtp-l3GJPoLWK6hBKQmWIQHDnbsbq0M2tcQXR5-WROCte74KKJx-4PWeIJo5tKh04JbhUJiB5OLBan7Lq7xfDDbbDtmej_3-PC3ql6354Rj2C_X3b7EfKjIOtO_bfbT2MbyeqrNQlTkLIvXoITJQD_bEP3Fbfj2DPQ3KabZqjDjJbue_I05f-oqebT4btbMqRtthf5KeJ3KaKrKW5rJabC3hPJeKU6qLT5Xjh6B5qDfyDoAbKOt-IOjhb5hMpnx-p0njxQyaR3RL2Kj0p_EWpcxsCQqLUonDh8L3H7MJUntKjnRonTO5hvvhb6O3M7-XpOhDG0fJjtJJbksQJ5e24oqHP-kKPrV-4oH5MQy5toyHD7yWCvjWlT5OR5Jj6KMjMkb3xbz2fcpMIrjob8M5CQESInv3MA--fcLD2ch5-3eQgTI3fbIJJjWsq0x0-jle-bQypoa-U0j2COMahkMal7xO-QO05CaD53yDNDqtjn-5TIX_CjJbnA_Hn7zepoxebtpbt-qJJjzMerW_Mc8QUJBH4tR-T3keh-83xbnBT5KaKO2-RnPXbcWjt_lWh_bLf_kQN3TbxuO5bRiL66I0h6jDn3oyT3VXp0n54nTqjDHfRuDVItXf-L_qtDk-PnVeUP3DhbZKxtqtDKjXJ7X2fclHJ7z-R3IBPCD0tjk-6JnWncKaRcI3poiqKtmjJb6XJkl2HQ405OT-6-O0KJcbRodobAwhPJvyT8DXnO7-fRTfJuJ_DDMJDD3fP36q4QV-JIehmT22jnT32JeaJ5n0-nnhP3mBTbA3JDYX-Oh-jjRX56GhfO_0R3jsJKRy66jK4JKjHKet6vP; ispeed_lsm=0; H_PS_PSSID=1421_24558_21120_17001_24880_22072; BD_UPN=123253; H_PS_645EC=44be6I1wqYYVvyugm2gc3PK9PoSa26pxhzOVbeQrn2rRadHvKoI%2BCbN5K%2Bg; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598',\n'Host':'www.baidu.com',\n'User-Agent':'Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/61.0.3163.100 Safari\/537.36'}\n\n# response=requests.get('https:\/\/www.baidu.com\/s?'+urlencode({'wd':'\u7f8e\u5973'}),headers=headers)\nresponse=requests.get('https:\/\/www.baidu.com\/s',params={'wd':'\u7f8e\u5973'},headers=headers) #params\u5185\u90e8\u5c31\u662f\u8c03\u7528urlencode\nprint(response.text)<\/code><\/pre>\n
\u4e94 Response<\/h1>\n
1\u3001\u54cd\u5e94\u72b6\u6001<\/h2>\n
200\uff1a\u4ee3\u8868\u6210\u529f\n301\uff1a\u4ee3\u8868\u8df3\u8f6c\n404\uff1a\u6587\u4ef6\u4e0d\u5b58\u5728\n403\uff1a\u6743\u9650\n502\uff1a\u670d\u52a1\u5668\u9519\u8bef<\/code><\/pre>\n
2\u3001Respone header<\/h2>\n
set-cookie\uff1a\u53ef\u80fd\u6709\u591a\u4e2a\uff0c\u662f\u6765\u544a\u8bc9\u6d4f\u89c8\u5668\uff0c\u628acookie\u4fdd\u5b58\u4e0b\u6765<\/code><\/pre>\n
3\u3001preview\u5c31\u662f\u7f51\u9875\u6e90\u4ee3\u7801<\/h2>\n
\u6700\u4e3b\u8981\u7684\u90e8\u5206\uff0c\u5305\u542b\u4e86\u8bf7\u6c42\u8d44\u6e90\u7684\u5185\u5bb9\n\u5982\u7f51\u9875html\uff0c\u56fe\u7247\n\u4e8c\u8fdb\u5236\u6570\u636e\u7b49<\/code><\/pre>\n
\u516d \u603b\u7ed3<\/h1>\n
1\u3001\u603b\u7ed3\u722c\u866b\u6d41\u7a0b\uff1a<\/h2>\n
\u722c\u53d6--->\u89e3\u6790--->\u5b58\u50a8<\/code><\/pre>\n
2\u3001\u722c\u866b\u6240\u9700\u5de5\u5177\uff1a<\/h2>\n
\u8bf7\u6c42\u5e93\uff1arequests,selenium\n\u89e3\u6790\u5e93\uff1a\u6b63\u5219\uff0cbeautifulsoup\uff0cpyquery\n\u5b58\u50a8\u5e93\uff1a\u6587\u4ef6\uff0cMySQL\uff0cMongodb\uff0cRedis<\/code><\/pre>\n
3\u3001\u722c\u866b\u5e38\u7528\u6846\u67b6\uff1a<\/h2>\n
scrapy<\/code><\/pre>\n
\u722c\u53d6\u6821\u82b1\u7f51\u89c6\u9891<\/strong><\/p>\n
import requests\nimport re\nimport time\nimport hashlib\n\ndef get_page(url):\n    print('GET %s' %url)\n    try:\n        response=requests.get(url)\n        if response.status_code == 200:\n            return response.content\n    except Exception:\n        pass\n\ndef parse_index(res):\n    obj=re.compile('class=\"items.*? 0:\n        movie_url=res[0]\n        return movie_url\n\ndef save(movie_url):\n    response=requests.get(movie_url,stream=False)\n    if response.status_code == 200:\n        m=hashlib.md5()\n        m.update(('%s%s.mp4' %(movie_url,time.time())).encode('utf-8'))\n        filename=m.hexdigest()\n        with open(r'.\/movies\/%s.mp4' %filename,'wb') as f:\n            f.write(response.content)\n            f.flush()\n\ndef main():\n    index_url='http:\/\/www.xiaohuar.com\/list-3-{0}.html'\n    for i in range(5):\n        print('*'*50,i)\n        #\u722c\u53d6\u4e3b\u9875\u9762\n        index_page=get_page(index_url.format(i,))\n        #\u89e3\u6790\u4e3b\u9875\u9762,\u62ff\u5230\u89c6\u9891\u6240\u5728\u7684\u5730\u5740\u5217\u8868\n        detail_urls=parse_index(index_page)\n        #\u5faa\u73af\u722c\u53d6\u89c6\u9891\u9875\n        for detail_url in detail_urls:\n            #\u722c\u53d6\u89c6\u9891\u9875\n            detail_page=get_page(detail_url)\n            #\u62ff\u5230\u89c6\u9891\u7684url\n            movie_url=parse_detail(detail_page)\n            if movie_url:\n                #\u4fdd\u5b58\u89c6\u9891\n                save(movie_url)\n\nif __name__ == '__main__':\n    main()\n\n#\u5e76\u53d1\u722c\u53d6\nfrom concurrent.futures import ThreadPoolExecutor\nimport queue\nimport requests\nimport re\nimport time\nimport hashlib\nfrom threading import current_thread\n\np=ThreadPoolExecutor(50)\n\ndef get_page(url):\n    print('%s GET %s' %(current_thread().getName(),url))\n    try:\n        response=requests.get(url)\n        if response.status_code == 200:\n            return response.content\n    except Exception as e:\n        print(e)\n\ndef parse_index(res):\n    print('%s parse index ' %current_thread().getName())\n    res=res.result()\n    obj=re.compile('class=\"items.*? 0:\n        movie_url=res[0]\n        print('MOVIE_URL: ',movie_url)\n        with open('db.txt','a') as f:\n            f.write('%s\\n' %movie_url)\n        # save(movie_url)\n        p.submit(save,movie_url)\n        print('%s\u4e0b\u8f7d\u4efb\u52a1\u5df2\u7ecf\u63d0\u4ea4' %movie_url)\ndef save(movie_url):\n    print('%s SAVE: %s' %(current_thread().getName(),movie_url))\n    try:\n        response=requests.get(movie_url,stream=False)\n        if response.status_code == 200:\n            m=hashlib.md5()\n            m.update(('%s%s.mp4' %(movie_url,time.time())).encode('utf-8'))\n            filename=m.hexdigest()\n            with open(r'.\/movies\/%s.mp4' %filename,'wb') as f:\n                f.write(response.content)\n                f.flush()\n    except Exception as e:\n        print(e)\n\ndef main():\n    index_url='http:\/\/www.xiaohuar.com\/list-3-{0}.html'\n    for i in range(5):\n        p.submit(get_page,index_url.format(i,)).add_done_callback(parse_index)\n\nif __name__ == '__main__':\n    main()<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
\u4e00 \u722c\u866b\u662f\u4ec0\u4e48 1\u3001\u4ec0\u4e48\u662f\u4e92\u8054\u7f51\uff1f \u4e92\u8054\u7f51\u662f\u7531\u7f51\u7edc\u8bbe\u5907\uff08\u7f51\u7ebf\uff0c\u8def\u7531\u5668\uff0c\u4ea4\u6362\u673a\uff0c\u9632\u706b\u5899\u7b49\u7b49\uff09\u548c\u4e00\u53f0\u53f0\u8ba1\u7b97\u673a\u8fde\u63a5\u800c […]<\/p>\n","protected":false},"author":3,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":[],"categories":[342,350],"tags":[],"_links":{"self":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts\/4562"}],"collection":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/users\/3"}],"replies":[{"embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=4562"}],"version-history":[{"count":0,"href":"https:\/\/egonlin.com\/index.php?rest_route=\/wp\/v2\/posts\/4562\/revisions"}],"wp:attachment":[{"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=4562"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=4562"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/egonlin.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=4562"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}