로지스틱 회귀모델의 모수 추정

Sigmoid 함수 (logistic function)

$sigmoid\left(x\right)=\frac{1}{1+e^{-x}}$

Sigmoid 함수 미분

$\frac{d}{dx}sigmoid\left(x\right)=\frac{d}{dx}{\left(1+e^{-x}\right)}^{-1}$

$=\left(-1\right)\frac{1}{{\left(1+e^{-x}\right)}^2}\frac{d}{dx}\left(1+e^{-x}\right)$

$=\left(-1\right)\frac{1}{{\left(1+e^{-x}\right)}^2}\left(0+e^{-x}\right)\frac{d}{dx}\left(-x\right)$

$=\left(-1\right)\frac{1}{{\left(1+e^{-x}\right)}^2}{e}^{-x}\left(-1\right)$

$=\left(-1\right)\frac{1}{{\left(1+e^{-x}\right)}^2}{e}^{-x}\left(-1\right)$

$=\left(-1\right)\frac{1}{{\left(1+e^{-x}\right)}^2}{e}^{-x}\left(-1\right)$

$=\frac{\left(1+e^{-x}\right)}{{\left(1+e^{-x}\right)}^2}-\frac{1}{{\left(1+e^{-x}\right)}^2}$

$=\frac{1}{1+e^{-x}}-\frac{1}{{\left(1+e^{-x}\right)}^2}$

$=\frac{1}{1+e^{-x}}\left(1-\frac{1}{1+e^{-x}}\right)$

$=sigmoid\left(x\right)\left(1-sigmoid\left(x\right)\right)$

$=\sigma \left(x\right)\text{'}=\sigma \left(x\right)(1-\sigma (x\left)\right)$

Cost 함수

$Cost\left(h_{\theta }\right(x),y)=-ylog\left(h_{\theta }\right(x\left)\right)-(1-y)log(1-h_{\theta }(x\left)\right)$

전체 Cost 함수

$j\left(\theta \right)=-\frac{1}{m}{\sum }_{i=1}^m\left[y^{\left(i\right)}\log \left(h_{\theta }\left(x^{\left(i\right)}\right)\right)+\left(1-y^{\left(i\right)}\right)\log \left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)\right]$

Cost 함수 미분

$\frac{\partial }{\partial {\theta }_j}j\left(\theta \right)=\frac{\partial }{\partial {\theta }_j}\frac{-1}{m}{\sum }_{i=1}^m\left[y^{\left(i\right)}\log \left(h_{\theta }\left(x^{\left(i\right)}\right)\right)+\left(1-y^{\left(i\right)}\right)\log \left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)\right]$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[y^{\left(i\right)}\frac{\partial }{\partial {\theta }_j}\log \left(h_{\theta }\left(x^{\left(i\right)}\right)\right)+\left(1-y^{\left(i\right)}\right)\frac{\partial }{\partial {\theta }_j}\log \left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)\right]$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[\frac{y^{\left(i\right)}\frac{\partial }{\partial {\theta }_j}{h}_{\theta }\left(x^{\left(i\right)}\right)}{h_{\theta }\left(x^{\left(i\right)}\right)}+\frac{\left(1-y^{\left(i\right)}\right)\frac{\partial }{\partial {\theta }_j}\left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)}{1-h_{\theta }\left(x^{\left(i\right)}\right)}\right]$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[\frac{y^{\left(i\right)}\frac{\partial }{\partial {\theta }_j}\sigma \left({\theta }^T{x}^{\left(i\right)}\right)}{h_{\theta }\left(x^{\left(i\right)}\right)}+\frac{\left(1-y^{\left(i\right)}\right)\frac{\partial }{\partial {\theta }_j}\left(1-\sigma \left({\theta }^T{x}^{\left(i\right)}\right)\right)}{1-h_{\theta }\left(x^{\left(i\right)}\right)}\right]$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[\frac{y^{\left(i\right)}\sigma \left({\theta }^T{x}^{\left(i\right)}\right)\left(1-\sigma \left({\theta }^T{x}^{\left(i\right)}\right)\right)\frac{\partial }{\partial {\theta }_j}{\theta }^T{x}^{\left(i\right)}}{h_{\theta }\left(x^{\left(i\right)}\right)}+\frac{-\left(1-y^{\left(i\right)}\right)\sigma \left({\theta }^T{x}^{\left(i\right)}\right)\left(1-\sigma \left({\theta }^T{x}^{\left(i\right)}\right)\right)\frac{\partial }{\partial {\theta }_j}{\theta }^T{x}^{\left(i\right)}}{1-h_{\theta }\left(x^{\left(i\right)}\right)}\right]$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[\frac{y^{\left(i\right)}{h}_{\theta }\left(x^{\left(i\right)}\right)\left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)\frac{\partial }{\partial {\theta }_j}{\theta }^T{x}^{\left(i\right)}}{h_{\theta }\left(x^{\left(i\right)}\right)}+\frac{-\left(1-y^{\left(i\right)}\right)h_{\theta }\left(x^{\left(i\right)}\right)\left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)\frac{\partial }{\partial {\theta }_j}{\theta }^T{x}^{\left(i\right)}}{1-h_{\theta }\left(x^{\left(i\right)}\right)}\right]$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[y^{\left(i\right)}{h}_{\theta }\left(x^{\left(i\right)}\right)\left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)x_j^{\left(i\right)}+-\left(1-y^{\left(i\right)}\right)h_{\theta }\left(x^{\left(i\right)}\right)x_j^{\left(i\right)}\right]$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[y^{\left(i\right)}{h}_{\theta }\left(x^{\left(i\right)}\right)\left(1-h_{\theta }\left(x^{\left(i\right)}\right)\right)+-\left(1-y^{\left(i\right)}\right)h_{\theta }\left(x^{\left(i\right)}\right)\right]x_j^{\left(i\right)}$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[y^{\left(i\right)}-y^{\left(i\right)}{h}_{\theta }\left(x^{\left(i\right)}\right)-h_{\theta }\left(x^{\left(i\right)}\right)+y^{\left(i\right)}{h}_{\theta }\left(x^{\left(i\right)}\right)\right]x_j^{\left(i\right)}$

$=-\frac{1}{m}{\sum }_{i=1}^m\left[y^{\left(i\right)}-h_{\theta }\left(x^{\left(i\right)}\right)\right]x_j^{\left(i\right)}$

$=\frac{1}{m}{\sum }_{i=1}^m\left[h_{\theta }\left(x^{\left(i\right)}\right)-y^{\left(i\right)}\right]x_j^{\left(i\right)}$

Gradient Desent

$Repeat\left\{{\theta }_j:={\theta }_j-\alpha \frac{\partial }{\partial {\theta }_j}J\left(\theta \right)\right\}$

↓

$Repeat\left\{{\theta }_j:={\theta }_j-\frac{\alpha }{m}{\sum }_{i=1}^m\left(h_{\theta }\left(x^{\left(i\right)}\right)-y^{\left(i\right)}\right)x_j^{\left(i\right)}\right\}$