update
This commit is contained in:
Morten Hjorth-Jensen
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,57 @@
|
||||
digraph Tree {
|
||||
node [shape=box, style="filled, rounded", color="black", fontname=helvetica] ;
|
||||
edge [fontname=helvetica] ;
|
||||
0 [label="worst perimeter <= 106.05\ngini = 0.465\nsamples = 426\nvalue = [[269, 157]\n[157, 269]]", fillcolor="#e5813908"] ;
|
||||
1 [label="worst concave points <= 0.159\ngini = 0.067\nsamples = 259\nvalue = [[250, 9]\n[9, 250]]", fillcolor="#e58139db"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="worst concave points <= 0.135\ngini = 0.031\nsamples = 253\nvalue = [[249, 4]\n[4, 249]]", fillcolor="#e58139ee"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="radius error <= 0.643\ngini = 0.008\nsamples = 242\nvalue = [[241, 1]\n[1, 241]]", fillcolor="#e58139fb"] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="worst symmetry <= 0.208\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
3 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 6 ;
|
||||
7 [label="gini = 0.0\nsamples = 2\nvalue = [[2, 0]\n[0, 2]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 7 ;
|
||||
8 [label="worst texture <= 29.455\ngini = 0.397\nsamples = 11\nvalue = [[8, 3]\n[3, 8]]", fillcolor="#e581392c"] ;
|
||||
2 -> 8 ;
|
||||
9 [label="gini = 0.0\nsamples = 8\nvalue = [[8, 0]\n[0, 8]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 9 ;
|
||||
10 [label="gini = 0.0\nsamples = 3\nvalue = [[0, 3]\n[3, 0]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 10 ;
|
||||
11 [label="mean texture <= 16.22\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
1 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 12 ;
|
||||
13 [label="gini = 0.0\nsamples = 5\nvalue = [[0, 5]\n[5, 0]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 13 ;
|
||||
14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ;
|
||||
0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
15 [label="worst radius <= 17.74\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
14 -> 15 ;
|
||||
16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ;
|
||||
15 -> 16 ;
|
||||
17 [label="mean texture <= 13.745\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
15 -> 17 ;
|
||||
18 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 18 ;
|
||||
19 [label="gini = 0.0\nsamples = 4\nvalue = [[0, 4]\n[4, 0]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 19 ;
|
||||
20 [label="mean concave points <= 0.049\ngini = 0.088\nsamples = 151\nvalue = [[7, 144]\n[144, 7]]", fillcolor="#e58139d0"] ;
|
||||
14 -> 20 ;
|
||||
21 [label="concave points error <= 0.01\ngini = 0.48\nsamples = 15\nvalue = [[6, 9]\n[9, 6]]", fillcolor="#e5813900"] ;
|
||||
20 -> 21 ;
|
||||
22 [label="gini = 0.0\nsamples = 9\nvalue = [[0, 9]\n[9, 0]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 22 ;
|
||||
23 [label="gini = 0.0\nsamples = 6\nvalue = [[6, 0]\n[0, 6]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 23 ;
|
||||
24 [label="worst smoothness <= 0.096\ngini = 0.015\nsamples = 136\nvalue = [[1, 135]\n[135, 1]]", fillcolor="#e58139f7"] ;
|
||||
20 -> 24 ;
|
||||
25 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 25 ;
|
||||
26 [label="gini = 0.0\nsamples = 135\nvalue = [[0, 135]\n[135, 0]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 26 ;
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 216 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
@@ -0,0 +1,15 @@
|
||||
Outlook,Temperature,Humidity,Wind,Ride
|
||||
0,0,0,0,0
|
||||
0,0,0,1,1
|
||||
1,0,0,0,1
|
||||
2,1,0,0,1
|
||||
2,2,1,0,1
|
||||
2,2,1,1,0
|
||||
1,2,1,1,1
|
||||
0,1,0,0,0
|
||||
0,2,1,0,1
|
||||
2,1,1,0,1
|
||||
0,1,1,1,1
|
||||
1,1,0,1,1
|
||||
1,0,1,0,1
|
||||
2,1,0,1,0
|
||||
|
@@ -0,0 +1,13 @@
|
||||
digraph Tree {
|
||||
node [shape=box, style="filled, rounded", color="black", fontname=helvetica] ;
|
||||
edge [fontname=helvetica] ;
|
||||
0 [label="X[7] <= 0.5\ngini = 0.48\nsamples = 15\nvalue = [4, 10, 1]", fillcolor="#39e5818b"] ;
|
||||
1 [label="X[1] <= 0.5\ngini = 0.408\nsamples = 14\nvalue = [4, 10, 0]", fillcolor="#39e58199"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="gini = 0.48\nsamples = 10\nvalue = [4, 6, 0]", fillcolor="#39e58155"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="gini = 0.0\nsamples = 4\nvalue = [0, 4, 0]", fillcolor="#39e581ff"] ;
|
||||
1 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 1\nvalue = [0, 0, 1]", fillcolor="#8139e5ff"] ;
|
||||
0 -> 4 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
Day,Outlook,Temperature,Humidity,Wind,Ride
|
||||
1,Sunny,Hot,High,Weak,0
|
||||
2,Sunny,Hot,High,Strong,1
|
||||
3,Overcast,Hot,High,Weak,1
|
||||
4,Rain,Mild,High,Weak,1
|
||||
5,Rain,Cool,Normal,Weak,1
|
||||
6,Rain,Cool,Normal,Strong,0
|
||||
7,Overcast,Cool,Normal,Strong,1
|
||||
8,Sunny,Mild,High,Weak,0
|
||||
9,Sunny,Cool,Normal,Weak,1
|
||||
10,Rain,Mild,Normal,Weak,1
|
||||
11,Sunny,Mild,Normal,Strong,1
|
||||
12,Overcast,Mild,High,Strong,1
|
||||
13,Overcast,Hot,Normal,Weak,1
|
||||
14,Rain,Mild,High,Strong,0
|
||||
|
@@ -0,0 +1,101 @@
|
||||
aardvark,1,0,0,1,0,0,1,1,1,1,0,0,4,0,0,1,1
|
||||
antelope,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
bass,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
bear,1,0,0,1,0,0,1,1,1,1,0,0,4,0,0,1,1
|
||||
boar,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
buffalo,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
calf,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
carp,0,0,1,0,0,1,0,1,1,0,0,1,0,1,1,0,4
|
||||
catfish,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
cavy,1,0,0,1,0,0,0,1,1,1,0,0,4,0,1,0,1
|
||||
cheetah,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
chicken,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,2
|
||||
chub,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
clam,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,7
|
||||
crab,0,0,1,0,0,1,1,0,0,0,0,0,4,0,0,0,7
|
||||
crayfish,0,0,1,0,0,1,1,0,0,0,0,0,6,0,0,0,7
|
||||
crow,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,0,2
|
||||
deer,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
dogfish,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,4
|
||||
dolphin,0,0,0,1,0,1,1,1,1,1,0,1,0,1,0,1,1
|
||||
dove,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,2
|
||||
duck,0,1,1,0,1,1,0,0,1,1,0,0,2,1,0,0,2
|
||||
elephant,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
flamingo,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,1,2
|
||||
flea,0,0,1,0,0,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
frog,0,0,1,0,0,1,1,1,1,1,0,0,4,0,0,0,5
|
||||
frog,0,0,1,0,0,1,1,1,1,1,1,0,4,0,0,0,5
|
||||
fruitbat,1,0,0,1,1,0,0,1,1,1,0,0,2,1,0,0,1
|
||||
giraffe,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
girl,1,0,0,1,0,0,1,1,1,1,0,0,2,0,1,1,1
|
||||
gnat,0,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
goat,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
gorilla,1,0,0,1,0,0,0,1,1,1,0,0,2,0,0,1,1
|
||||
gull,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,2
|
||||
haddock,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,4
|
||||
hamster,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,0,1
|
||||
hare,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,0,1
|
||||
hawk,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,0,2
|
||||
herring,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
honeybee,1,0,1,0,1,0,0,0,0,1,1,0,6,0,1,0,6
|
||||
housefly,1,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
kiwi,0,1,1,0,0,0,1,0,1,1,0,0,2,1,0,0,2
|
||||
ladybird,0,0,1,0,1,0,1,0,0,1,0,0,6,0,0,0,6
|
||||
lark,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
leopard,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
lion,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
lobster,0,0,1,0,0,1,1,0,0,0,0,0,6,0,0,0,7
|
||||
lynx,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
mink,1,0,0,1,0,1,1,1,1,1,0,0,4,1,0,1,1
|
||||
mole,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,0,1
|
||||
mongoose,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
moth,1,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
newt,0,0,1,0,0,1,1,1,1,1,0,0,4,1,0,0,5
|
||||
octopus,0,0,1,0,0,1,1,0,0,0,0,0,8,0,0,1,7
|
||||
opossum,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,0,1
|
||||
oryx,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
ostrich,0,1,1,0,0,0,0,0,1,1,0,0,2,1,0,1,2
|
||||
parakeet,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,2
|
||||
penguin,0,1,1,0,0,1,1,0,1,1,0,0,2,1,0,1,2
|
||||
pheasant,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
pike,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,4
|
||||
piranha,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
pitviper,0,0,1,0,0,0,1,1,1,1,1,0,0,1,0,0,3
|
||||
platypus,1,0,1,1,0,1,1,0,1,1,0,0,4,1,0,1,1
|
||||
polecat,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
pony,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
porpoise,0,0,0,1,0,1,1,1,1,1,0,1,0,1,0,1,1
|
||||
puma,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
pussycat,1,0,0,1,0,0,1,1,1,1,0,0,4,1,1,1,1
|
||||
raccoon,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
reindeer,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
rhea,0,1,1,0,0,0,1,0,1,1,0,0,2,1,0,1,2
|
||||
scorpion,0,0,0,0,0,0,1,0,0,1,1,0,8,1,0,0,7
|
||||
seahorse,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,4
|
||||
seal,1,0,0,1,0,1,1,1,1,1,0,1,0,0,0,1,1
|
||||
sealion,1,0,0,1,0,1,1,1,1,1,0,1,2,1,0,1,1
|
||||
seasnake,0,0,0,0,0,1,1,1,1,0,1,0,0,1,0,0,3
|
||||
seawasp,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,7
|
||||
skimmer,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,2
|
||||
skua,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,2
|
||||
slowworm,0,0,1,0,0,0,1,1,1,1,0,0,0,1,0,0,3
|
||||
slug,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,7
|
||||
sole,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,4
|
||||
sparrow,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
squirrel,1,0,0,1,0,0,0,1,1,1,0,0,2,1,0,0,1
|
||||
starfish,0,0,1,0,0,1,1,0,0,0,0,0,5,0,0,0,7
|
||||
stingray,0,0,1,0,0,1,1,1,1,0,1,1,0,1,0,1,4
|
||||
swan,0,1,1,0,1,1,0,0,1,1,0,0,2,1,0,1,2
|
||||
termite,0,0,1,0,0,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
toad,0,0,1,0,0,1,0,1,1,1,0,0,4,0,0,0,5
|
||||
tortoise,0,0,1,0,0,0,0,0,1,1,0,0,4,1,0,1,3
|
||||
tuatara,0,0,1,0,0,0,1,1,1,1,0,0,4,1,0,0,3
|
||||
tuna,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,4
|
||||
vampire,1,0,0,1,1,0,0,1,1,1,0,0,2,1,0,0,1
|
||||
vole,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,0,1
|
||||
vulture,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,1,2
|
||||
wallaby,1,0,0,1,0,0,0,1,1,1,0,0,2,1,0,1,1
|
||||
wasp,1,0,1,0,1,0,0,0,0,1,1,0,6,0,0,0,6
|
||||
wolf,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
worm,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,7
|
||||
wren,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
|
@@ -0,0 +1,78 @@
|
||||
import numpy as np
|
||||
|
||||
class DecisionStump:
|
||||
def fit(self, X, y, weights):
|
||||
m, n = X.shape
|
||||
self.alpha = 0
|
||||
self.threshold = None
|
||||
self.polarity = 1
|
||||
|
||||
min_error = float('inf')
|
||||
|
||||
for feature in range(n):
|
||||
feature_values = np.unique(X[:, feature])
|
||||
|
||||
for threshold in feature_values:
|
||||
for polarity in [1, -1]:
|
||||
predictions = np.ones(m)
|
||||
predictions[X[:, feature] < threshold] = -1
|
||||
predictions *= polarity
|
||||
|
||||
error = sum(weights[predictions != y])
|
||||
|
||||
if error < min_error:
|
||||
min_error = error
|
||||
self.alpha = 0.5 * np.log((1 - error) / (error + 1e-10))
|
||||
self.threshold = threshold
|
||||
self.feature_index = feature
|
||||
self.polarity = polarity
|
||||
|
||||
def predict(self, X):
|
||||
m = X.shape[0]
|
||||
predictions = np.ones(m)
|
||||
if self.polarity == 1:
|
||||
predictions[X[:, self.feature_index] < self.threshold] = -1
|
||||
else:
|
||||
predictions[X[:, self.feature_index] >= self.threshold] = -1
|
||||
return predictions
|
||||
|
||||
class AdaBoost:
|
||||
def fit(self, X, y, n_estimators):
|
||||
m = X.shape[0]
|
||||
self.alphas = []
|
||||
self.models = []
|
||||
|
||||
weights = np.ones(m) / m
|
||||
|
||||
for _ in range(n_estimators):
|
||||
stump = DecisionStump()
|
||||
stump.fit(X, y, weights)
|
||||
predictions = stump.predict(X)
|
||||
|
||||
error = sum(weights[predictions != y])
|
||||
if error == 0:
|
||||
break
|
||||
|
||||
self.models.append(stump)
|
||||
self.alphas.append(stump.alpha)
|
||||
|
||||
weights *= np.exp(-stump.alpha * y * predictions)
|
||||
weights /= np.sum(weights)
|
||||
|
||||
def predict(self, X):
|
||||
final_predictions = np.zeros(X.shape[0])
|
||||
for alpha, model in zip(self.alphas, self.models):
|
||||
final_predictions += alpha * model.predict(X)
|
||||
return np.sign(final_predictions)
|
||||
|
||||
# Example dataset (X, y)
|
||||
X = np.array([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]])
|
||||
y = np.array([-1, -1, -1, -1, 1, 1, 1, 1, 1, 1]) # Labels must be -1 or 1
|
||||
|
||||
# Train AdaBoost
|
||||
ada = AdaBoost()
|
||||
ada.fit(X, y, n_estimators=10)
|
||||
|
||||
# Predictions
|
||||
predictions = ada.predict(X)
|
||||
print("Predictions:", predictions)
|
||||
@@ -1,65 +1,65 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.datasets import load_breast_cancer
|
||||
from sklearn.svm import SVC
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.ensemble import AdaBoostClassifier
|
||||
e:
|
||||
return self.Node(value=self._most_common_label(y))
|
||||
left_indices = X[:, best_feature] < best_threshold
|
||||
right_indices = X[:, best_feature] >= best_threshold
|
||||
left_subtree = self._grow_tree(X[left_indices], y[left_indices], depth + 1)
|
||||
right_subtree = self._grow_tree(X[right_indices], y[right_indices], depth + 1)
|
||||
return self.Node(feature=best_feature, threshold=best_threshold, left=left_subtree, right=right_subtree)
|
||||
def _best_split(self, X, y, num_features):
|
||||
best_gain = -1
|
||||
best_feature, best_threshold = None, None
|
||||
|
||||
def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
|
||||
x1s = np.linspace(axes[0], axes[1], 100)
|
||||
x2s = np.linspace(axes[2], axes[3], 100)
|
||||
x1, x2 = np.meshgrid(x1s, x2s)
|
||||
X_new = np.c_[x1.ravel(), x2.ravel()]
|
||||
y_pred = clf.predict(X_new).reshape(x1.shape)
|
||||
custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
|
||||
plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
|
||||
if contour:
|
||||
custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
|
||||
plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
|
||||
plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
|
||||
plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
|
||||
plt.axis(axes)
|
||||
plt.xlabel(r"$x_1$", fontsize=18)
|
||||
plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
|
||||
for feature in range(num_features):
|
||||
thresholds, classes = zip(*sorted(zip(X[:, feature], y)))
|
||||
num_samples = len(y)
|
||||
for i in range(1, num_samples):
|
||||
if classes[i] == classes[i - 1]:
|
||||
continue
|
||||
|
||||
# Load the data
|
||||
cancer = load_breast_cancer()
|
||||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||||
#now scale the data
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
threshold = (thresholds[i] + thresholds[i - 1]) / 2
|
||||
left_indices = X[:, feature] < threshold
|
||||
right_indices = X[:, feature] >= threshold
|
||||
|
||||
ada_clf = AdaBoostClassifier(
|
||||
DecisionTreeClassifier(max_depth=1), n_estimators=200,
|
||||
algorithm="SAMME.R", learning_rate=0.5, random_state=42)
|
||||
|
||||
ada_clf.fit(X_train_scaled, y_train)
|
||||
plot_decision_boundary(ada_clf, cancer.data,cancer.target)
|
||||
|
||||
m = len(X_train_scaled)
|
||||
|
||||
plt.figure(figsize=(11, 4))
|
||||
for subplot, learning_rate in ((121, 1), (122, 0.5)):
|
||||
sample_weights = np.ones(m)
|
||||
plt.subplot(subplot)
|
||||
for i in range(5):
|
||||
svm_clf = SVC(kernel="rbf", C=0.05, gamma="auto", random_state=42)
|
||||
svm_clf.fit(X_train_scaled, y_train, sample_weight=sample_weights)
|
||||
y_pred = svm_clf.predict(X_train_scaled)
|
||||
sample_weights[y_pred != y_train] *= (1 + learning_rate)
|
||||
plot_decision_boundary(svm_clf, cancer.data,cancer.target, alpha=0.2)
|
||||
plt.title("learning_rate = {}".format(learning_rate), fontsize=16)
|
||||
if subplot == 121:
|
||||
plt.text(-0.7, -0.65, "1", fontsize=14)
|
||||
plt.text(-0.6, -0.10, "2", fontsize=14)
|
||||
plt.text(-0.5, 0.10, "3", fontsize=14)
|
||||
plt.text(-0.4, 0.55, "4", fontsize=14)
|
||||
plt.text(-0.3, 0.90, "5", fontsize=14)
|
||||
|
||||
plt.show()
|
||||
gain = self._information_gain(y, y[left_indices], y[right_indices])
|
||||
if gain > best_gain:
|
||||
best_gain = gain
|
||||
best_feature = feature
|
||||
best_threshold = threshold
|
||||
return best_feature, best_threshold
|
||||
def _information_gain(self, parent, left, right):
|
||||
total_samples = len(parent)
|
||||
if len(left) == 0 or len(right) == 0:
|
||||
return 0
|
||||
|
||||
parent_entropy = self._entropy(parent)
|
||||
left_entropy = self._entropy(left)
|
||||
right_entropy = self._entropy(right)
|
||||
weighted_entropy = (len(left) / total_samples) * left_entropy + (len(right) / total_samples) * right_entropy
|
||||
return parent_entropy - weighted_entropy
|
||||
def _entropy(self, y):
|
||||
class_counts = np.bincount(y)
|
||||
probabilities = class_counts / len(y)
|
||||
return -np.sum(probabilities * np.log(probabilities + 1e-10))
|
||||
def _most_common_label(self, y):
|
||||
return np.bincount(y).argmax()
|
||||
def predict(self, X):
|
||||
return np.array([self._predict(inputs) for inputs in X])
|
||||
def _predict(self, inputs):
|
||||
node = self.tree
|
||||
while node.value is None:
|
||||
if inputs[node.feature] < node.threshold:
|
||||
node = node.left
|
||||
else:
|
||||
node = node.right
|
||||
return node.value
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Example dataset
|
||||
X = np.array([[2.5], [1.0], [1.5], [3.0], [3.5], [2.0], [4.0], [2.2]])
|
||||
y = np.array([0, 0, 0, 1, 1, 0, 1, 0]) # Binary labels
|
||||
# Train decision tree
|
||||
tree = DecisionTree(max_depth=3)
|
||||
tree.fit(X, y)
|
||||
# Predictions
|
||||
predictions = tree.predict(X)
|
||||
print("Predictions:", predictions)~
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import numpy as np
|
||||
class DecisionTree:
|
||||
def __init__(self, min_samples_split=2, max_depth=float("inf")):
|
||||
self.min_samples_split = min_samples_split
|
||||
self.max_depth = max_depth
|
||||
self.tree = None
|
||||
class Node:
|
||||
def __init__(self, feature=None, threshold=None, left=None, right=None, value=None):
|
||||
self.feature = feature
|
||||
self.threshold = threshold
|
||||
self.left = left
|
||||
self.right = right
|
||||
self.value = value
|
||||
def fit(self, X, y):
|
||||
self.tree = self._grow_tree(X, y)
|
||||
def _grow_tree(self, X, y, depth=0):
|
||||
num_samples, num_features = X.shape
|
||||
unique_classes = np.unique(y)
|
||||
# Check stopping criteria
|
||||
if (num_samples < self.min_samples_split or
|
||||
depth >= self.max_depth or
|
||||
len(unique_classes) == 1):
|
||||
return self.Node(value=self._most_common_label(y))
|
||||
best_feature, best_threshold = self._best_split(X, y, num_features)
|
||||
if best_feature is None:
|
||||
return self.Node(value=self._most_common_label(y))
|
||||
left_indices = X[:, best_feature] < best_threshold
|
||||
right_indices = X[:, best_feature] >= best_threshold
|
||||
left_subtree = self._grow_tree(X[left_indices], y[left_indices], depth + 1)
|
||||
right_subtree = self._grow_tree(X[right_indices], y[right_indices], depth + 1)
|
||||
return self.Node(feature=best_feature, threshold=best_threshold, left=left_subtree, right=right_subtree)
|
||||
def _best_split(self, X, y, num_features):
|
||||
best_gain = -1
|
||||
best_feature, best_threshold = None, None
|
||||
|
||||
for feature in range(num_features):
|
||||
thresholds, classes = zip(*sorted(zip(X[:, feature], y)))
|
||||
num_samples = len(y)
|
||||
for i in range(1, num_samples):
|
||||
if classes[i] == classes[i - 1]:
|
||||
continue
|
||||
|
||||
threshold = (thresholds[i] + thresholds[i - 1]) / 2
|
||||
left_indices = X[:, feature] < threshold
|
||||
right_indices = X[:, feature] >= threshold
|
||||
gain = self._information_gain(y, y[left_indices], y[right_indices])
|
||||
if gain > best_gain:
|
||||
best_gain = gain
|
||||
best_feature = feature
|
||||
best_threshold = threshold
|
||||
return best_feature, best_threshold
|
||||
def _information_gain(self, parent, left, right):
|
||||
total_samples = len(parent)
|
||||
if len(left) == 0 or len(right) == 0:
|
||||
return 0
|
||||
|
||||
parent_entropy = self._entropy(parent)
|
||||
left_entropy = self._entropy(left)
|
||||
right_entropy = self._entropy(right)
|
||||
weighted_entropy = (len(left) / total_samples) * left_entropy + (len(right) / total_samples) * right_entropy
|
||||
return parent_entropy - weighted_entropy
|
||||
def _entropy(self, y):
|
||||
class_counts = np.bincount(y)
|
||||
probabilities = class_counts / len(y)
|
||||
return -np.sum(probabilities * np.log(probabilities + 1e-10))
|
||||
def _most_common_label(self, y):
|
||||
return np.bincount(y).argmax()
|
||||
def predict(self, X):
|
||||
return np.array([self._predict(inputs) for inputs in X])
|
||||
def _predict(self, inputs):
|
||||
node = self.tree
|
||||
while node.value is None:
|
||||
if inputs[node.feature] < node.threshold:
|
||||
node = node.left
|
||||
else:
|
||||
node = node.right
|
||||
return node.value
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Example dataset
|
||||
X = np.array([[2.5], [1.0], [1.5], [3.0], [3.5], [2.0], [4.0], [2.2]])
|
||||
y = np.array([0, 0, 0, 1, 1, 0, 1, 0]) # Binary labels
|
||||
# Train decision tree
|
||||
tree = DecisionTree(max_depth=3)
|
||||
tree.fit(X, y)
|
||||
# Predictions
|
||||
predictions = tree.predict(X)
|
||||
print("Predictions:", predictions)
|
||||
@@ -0,0 +1,61 @@
|
||||
import tensorflow as tf
|
||||
|
||||
from tensorflow.keras import datasets, layers, models
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# We import the data set
|
||||
(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
|
||||
|
||||
|
||||
train_images, test_images = train_images / 255.0, test_images / 255.0
|
||||
|
||||
class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
|
||||
'dog', 'frog', 'horse', 'ship', 'truck']
|
||||
plt.figure(figsize=(10,10))
|
||||
for i in range(25):
|
||||
plt.subplot(5,5,i+1)
|
||||
plt.xticks([])
|
||||
plt.yticks([])
|
||||
plt.grid(False)
|
||||
plt.imshow(train_images[i], cmap=plt.cm.binary)
|
||||
# The CIFAR labels happen to be arrays,
|
||||
# which is why you need the extra index
|
||||
plt.xlabel(class_names[train_labels[i][0]])
|
||||
plt.show()
|
||||
|
||||
model = models.Sequential()
|
||||
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))
|
||||
model.add(layers.MaxPooling2D((2, 2)))
|
||||
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
|
||||
model.add(layers.MaxPooling2D((2, 2)))
|
||||
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
|
||||
|
||||
# Let's display the architecture of our model so far.
|
||||
|
||||
model.summary()
|
||||
|
||||
model.add(layers.Flatten())
|
||||
model.add(layers.Dense(64, activation='relu'))
|
||||
model.add(layers.Dense(10))
|
||||
# Here's the complete architecture of our model
|
||||
|
||||
model.summary()
|
||||
|
||||
model.compile(optimizer='adam',
|
||||
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
|
||||
metrics=['accuracy'])
|
||||
|
||||
history = model.fit(train_images, train_labels, epochs=10,
|
||||
validation_data=(test_images, test_labels))
|
||||
|
||||
|
||||
plt.plot(history.history['accuracy'], label='accuracy')
|
||||
plt.plot(history.history['val_accuracy'], label = 'val_accuracy')
|
||||
plt.xlabel('Epoch')
|
||||
plt.ylabel('Accuracy')
|
||||
plt.ylim([0.5, 1])
|
||||
plt.legend(loc='lower right')
|
||||
|
||||
test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
|
||||
|
||||
print(test_acc)
|
||||
@@ -0,0 +1,134 @@
|
||||
|
||||
# import necessary packages
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn import datasets
|
||||
|
||||
|
||||
# ensure the same random numbers appear every time
|
||||
np.random.seed(0)
|
||||
|
||||
# display images in notebook
|
||||
plt.rcParams['figure.figsize'] = (12,12)
|
||||
|
||||
|
||||
# download MNIST dataset
|
||||
digits = datasets.load_digits()
|
||||
|
||||
# define inputs and labels
|
||||
inputs = digits.images
|
||||
labels = digits.target
|
||||
|
||||
# RGB images have a depth of 3
|
||||
# our images are grayscale so they should have a depth of 1
|
||||
inputs = inputs[:,:,:,np.newaxis]
|
||||
|
||||
print("inputs = (n_inputs, pixel_width, pixel_height, depth) = " + str(inputs.shape))
|
||||
print("labels = (n_inputs) = " + str(labels.shape))
|
||||
|
||||
|
||||
# choose some random images to display
|
||||
n_inputs = len(inputs)
|
||||
indices = np.arange(n_inputs)
|
||||
random_indices = np.random.choice(indices, size=5)
|
||||
|
||||
for i, image in enumerate(digits.images[random_indices]):
|
||||
plt.subplot(1, 5, i+1)
|
||||
plt.axis('off')
|
||||
plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
|
||||
plt.title("Label: %d" % digits.target[random_indices[i]])
|
||||
plt.show()
|
||||
|
||||
from tensorflow.keras import datasets, layers, models
|
||||
from tensorflow.keras.layers import Input
|
||||
from tensorflow.keras.models import Sequential #This allows appending layers to existing models
|
||||
from tensorflow.keras.layers import Dense #This allows defining the characteristics of a particular layer
|
||||
from tensorflow.keras import optimizers #This allows using whichever optimiser we want (sgd,adam,RMSprop)
|
||||
from tensorflow.keras import regularizers #This allows using whichever regularizer we want (l1,l2,l1_l2)
|
||||
from tensorflow.keras.utils import to_categorical #This allows using categorical cross entropy as the cost function
|
||||
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
# representation of labels
|
||||
labels = to_categorical(labels)
|
||||
|
||||
# split into train and test data
|
||||
# one-liner from scikit-learn library
|
||||
train_size = 0.8
|
||||
test_size = 1 - train_size
|
||||
X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
|
||||
test_size=test_size)
|
||||
|
||||
def create_convolutional_neural_network_keras(input_shape, receptive_field,
|
||||
n_filters, n_neurons_connected, n_categories,
|
||||
eta, lmbd):
|
||||
model = Sequential()
|
||||
model.add(layers.Conv2D(n_filters, (receptive_field, receptive_field), input_shape=input_shape, padding='same',
|
||||
activation='relu', kernel_regularizer=regularizers.l2(lmbd)))
|
||||
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(layers.Flatten())
|
||||
model.add(layers.Dense(n_neurons_connected, activation='relu', kernel_regularizer=regularizers.l2(lmbd)))
|
||||
model.add(layers.Dense(n_categories, activation='softmax', kernel_regularizer=regularizers.l2(lmbd)))
|
||||
|
||||
sgd = optimizers.SGD(learning_rate=eta)
|
||||
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
|
||||
|
||||
return model
|
||||
|
||||
epochs = 100
|
||||
batch_size = 100
|
||||
input_shape = X_train.shape[1:4]
|
||||
receptive_field = 3
|
||||
n_filters = 10
|
||||
n_neurons_connected = 50
|
||||
n_categories = 10
|
||||
|
||||
eta_vals = np.logspace(-5, 1, 7)
|
||||
lmbd_vals = np.logspace(-5, 1, 7)
|
||||
|
||||
CNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
|
||||
|
||||
for i, eta in enumerate(eta_vals):
|
||||
for j, lmbd in enumerate(lmbd_vals):
|
||||
CNN = create_convolutional_neural_network_keras(input_shape, receptive_field,
|
||||
n_filters, n_neurons_connected, n_categories,
|
||||
eta, lmbd)
|
||||
CNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0)
|
||||
scores = CNN.evaluate(X_test, Y_test)
|
||||
|
||||
CNN_keras[i][j] = CNN
|
||||
|
||||
print("Learning rate = ", eta)
|
||||
print("Lambda = ", lmbd)
|
||||
print("Test accuracy: %.3f" % scores[1])
|
||||
print()
|
||||
# visual representation of grid search
|
||||
# uses seaborn heatmap, could probably do this in matplotlib
|
||||
import seaborn as sns
|
||||
|
||||
sns.set()
|
||||
|
||||
train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
||||
test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
||||
|
||||
for i in range(len(eta_vals)):
|
||||
for j in range(len(lmbd_vals)):
|
||||
CNN = CNN_keras[i][j]
|
||||
|
||||
train_accuracy[i][j] = CNN.evaluate(X_train, Y_train)[1]
|
||||
test_accuracy[i][j] = CNN.evaluate(X_test, Y_test)[1]
|
||||
|
||||
|
||||
fig, ax = plt.subplots(figsize = (10, 10))
|
||||
sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
|
||||
ax.set_title("Training Accuracy")
|
||||
ax.set_ylabel("$\eta$")
|
||||
ax.set_xlabel("$\lambda$")
|
||||
plt.show()
|
||||
|
||||
fig, ax = plt.subplots(figsize = (10, 10))
|
||||
sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
|
||||
ax.set_title("Test Accuracy")
|
||||
ax.set_ylabel("$\eta$")
|
||||
ax.set_xlabel("$\lambda$")
|
||||
plt.show()
|
||||
Reference in New Issue
Block a user