\u968f\u673a\u53d8\u91cf\u4e0d\u786e\u5b9a\u6027\u7684\u5ea6\u91cf<\/code>\u3002\u5047\u8bbe\u79bb\u6563\u968f\u673a\u53d8\u91cf$X$\u53ef\u4ee5\u53d6\u5230$n$\u4e2a\u503c\uff0c\u5176\u6982\u7387\u5206\u5e03\u4e3a
\n$$
\nP(X=x_i)=pi, \\quad i = 1,2,\\ldots,n
\n$$
\n\u5219$X$\u7684\u71b5\u5b9a\u4e49\u4e3a
\n$$
\nH(X) = -\\sum<\/em>{i=1}^n p_i log{pi}
\n$$
\n\u7531\u4e8e\u71b5\u53ea\u4f9d\u8d56$X$\u7684\u5206\u5e03\uff0c\u4e0e$X$\u672c\u8eab\u7684\u503c\u6ca1\u6709\u5173\u7cfb\uff0c\u6240\u4ee5\u71b5\u4e5f\u53ef\u4ee5\u5b9a\u4e49\u4e3a
\n$$
\nH(p) = -\\sum<\/em>{i=1}^n p_i log{p_i}
\n$$
\n \u71b5\u8d8a\u5927\uff0c\u5219\u968f\u673a\u53d8\u91cf\u7684\u4e0d\u786e\u5b9a\u6027\u8d8a\u5927\uff0c\u5e76\u4e14$0\\geq{H(p)}\\leq\\log{n}$\u3002<\/p>\n \u5f53\u968f\u673a\u53d8\u91cf\u53ea\u53d6\u4e24\u4e2a\u503c$0$\u548c$1$\u7684\u65f6\u5019\uff0c$X$\u7684\u5206\u5e03\u4e3a
\n$$
\nP(X=1)=p, \\quad P(x=0)=1-p, \\quad 0\\geq{p}\\leq{1}
\n$$
\n\u71b5\u5219\u662f
\n$$
\nH(p) = -p\\log_2 p-(1-p) \\log_2(1-p)
\n$$
\n\u6b64\u65f6\u968f\u673a\u53d8\u91cf\u4e3a\u4f2f\u52aa\u5229\u5206\u5e03\uff0c\u71b5\u968f\u6982\u7387\u53d8\u5316\u7684\u66f2\u7ebf\u5982\u4e0b\u56fe\u6240\u793a<\/p>\n
import numpy as np\nfrom math import log\nimport matplotlib.pyplot as plt\nfrom matplotlib.font_manager import FontProperties\n%matplotlib inline\nfont = FontProperties(fname='\/Library\/Fonts\/Heiti.ttc')\n\np = np.arange(0.01, 1, 0.01)\nentro = -p*np.log2(p) - (1-p)*np.log2(1-p)\n\nplt.plot(p, entro)\nplt.title('\u4f2f\u52aa\u5229\u5206\u5e03\u65f6\u71b5\u548c\u6982\u7387\u7684\u5173\u7cfb', fontproperties=font)\nplt.xlabel('p')\nplt.ylabel('H(p)')\nplt.show()<\/code><\/pre>\n![\"\"](\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\")
<\/div><\/p>\n \u5f53$p=0$\u548c$p=1$\u65f6\u71b5\u503c\u4e3a$0$\uff0c\u6b64\u65f6\u968f\u673a\u53d8\u91cf\u5b8c\u5168\u6ca1\u6709\u4e0d\u786e\u5b9a\u6027\uff1b\u5f53$p=0.5$\u65f6\u7684\u71b5\u503c\u6700\u5927\uff0c\u968f\u673a\u53d8\u91cf\u7684\u4e0d\u786e\u5b9a\u6027\u6700\u5927\u3002<\/p>\n
\u6761\u4ef6\u71b5\uff08Conditional Entropy\uff09<\/h1>\n
\u5047\u8bbe\u6709\u968f\u673a\u53d8\u91cf$(X,Y)$\uff0c\u5176\u8054\u5408\u6982\u7387\u4e3a
\n$$
\np(X=x_i,Y=yi), \\quad i=1,2,\\ldots,n; \\quad j=1,2,\\ldots,m
\n$$
\n \u6761\u4ef6\u71b5$H(Y|X)$\u8868\u793a\u5728\u5df2\u77e5\u968f\u673a\u53d8\u91cf$X$\u7684\u6761\u4ef6\u4e0b\u968f\u673a\u53d8\u91cf$Y$\u7684\u4e0d\u786e\u5b9a\u6027\uff0c\u5b9a\u4e49\u4e3a
\n$$
\nH(Y|X) = \\sum<\/em>{i=1}^n P(X=x_i) H(Y|X=x_i)
\n$$
\n\u901a\u8fc7\u516c\u5f0f\u53ef\u4ee5\u628a\u6761\u4ef6\u71b5\u7406\u89e3\u4e3a\u5728\u5f97\u77e5\u67d0\u4e00\u786e\u5b9a\u4fe1\u606f\u7684\u57fa\u7840\u4e0a\u83b7\u53d6\u53e6\u5916\u4e00\u4e2a\u4fe1\u606f\u65f6\u6240\u83b7\u5f97\u7684\u4fe1\u606f\u91cf<\/code>\u3002<\/p>\n \u5f53\u71b5\u548c\u6761\u4ef6\u71b5\u4e2d\u7684\u6982\u7387\u7531\u6570\u636e\u4f30\u8ba1\u83b7\u5f97\u65f6\uff0c\u6240\u5bf9\u5e94\u7684\u71b5\u4e0e\u6761\u4ef6\u71b5\u5206\u522b\u79f0\u4e3a\u7ecf\u9a8c\u71b5\uff08empirical entropy\uff09\u548c\u7ecf\u9a8c\u6761\u4ef6\u71b5\uff08empirical conditional entropy\uff09\u3002<\/p>\n
\u8054\u5408\u71b5\uff08Joint Entropy\uff09<\/h1>\n
<\/p>
<\/div>\u67e5\u770b\u66f4\u591a<\/h2>\u8054\u7cfb\u7ba1\u7406\u5458\u5fae\u4fe1tutu19192010\uff0c\u6ce8\u518c\u8d26\u53f7<\/p>\n<\/div>
<\/div>
![\"\"](\"https:\/\/egonlin.com\/wp-content\/uploads\/2022\/02\/\u6982\u7387\u8bba-\u71b5\u548c\u4fe1\u606f\u589e\u76ca_3_0.png\")
<\/div><\/p>\n