update
This commit is contained in:
Morten Hjorth-Jensen
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,57 @@
|
||||
digraph Tree {
|
||||
node [shape=box, style="filled, rounded", color="black", fontname=helvetica] ;
|
||||
edge [fontname=helvetica] ;
|
||||
0 [label="worst perimeter <= 106.05\ngini = 0.465\nsamples = 426\nvalue = [[269, 157]\n[157, 269]]", fillcolor="#e5813908"] ;
|
||||
1 [label="worst concave points <= 0.159\ngini = 0.067\nsamples = 259\nvalue = [[250, 9]\n[9, 250]]", fillcolor="#e58139db"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="worst concave points <= 0.135\ngini = 0.031\nsamples = 253\nvalue = [[249, 4]\n[4, 249]]", fillcolor="#e58139ee"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="radius error <= 0.643\ngini = 0.008\nsamples = 242\nvalue = [[241, 1]\n[1, 241]]", fillcolor="#e58139fb"] ;
|
||||
2 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 239\nvalue = [[239, 0]\n[0, 239]]", fillcolor="#e58139ff"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="worst symmetry <= 0.208\ngini = 0.444\nsamples = 3\nvalue = [[2, 1]\n[1, 2]]", fillcolor="#e5813913"] ;
|
||||
3 -> 5 ;
|
||||
6 [label="gini = 0.0\nsamples = 1\nvalue = [[0, 1]\n[1, 0]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 6 ;
|
||||
7 [label="gini = 0.0\nsamples = 2\nvalue = [[2, 0]\n[0, 2]]", fillcolor="#e58139ff"] ;
|
||||
5 -> 7 ;
|
||||
8 [label="worst texture <= 29.455\ngini = 0.397\nsamples = 11\nvalue = [[8, 3]\n[3, 8]]", fillcolor="#e581392c"] ;
|
||||
2 -> 8 ;
|
||||
9 [label="gini = 0.0\nsamples = 8\nvalue = [[8, 0]\n[0, 8]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 9 ;
|
||||
10 [label="gini = 0.0\nsamples = 3\nvalue = [[0, 3]\n[3, 0]]", fillcolor="#e58139ff"] ;
|
||||
8 -> 10 ;
|
||||
11 [label="mean texture <= 16.22\ngini = 0.278\nsamples = 6\nvalue = [[1, 5]\n[5, 1]]", fillcolor="#e581396b"] ;
|
||||
1 -> 11 ;
|
||||
12 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 12 ;
|
||||
13 [label="gini = 0.0\nsamples = 5\nvalue = [[0, 5]\n[5, 0]]", fillcolor="#e58139ff"] ;
|
||||
11 -> 13 ;
|
||||
14 [label="worst texture <= 20.645\ngini = 0.202\nsamples = 167\nvalue = [[19, 148]\n[148, 19]]", fillcolor="#e5813994"] ;
|
||||
0 -> 14 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
15 [label="worst radius <= 17.74\ngini = 0.375\nsamples = 16\nvalue = [[12, 4]\n[4, 12]]", fillcolor="#e5813938"] ;
|
||||
14 -> 15 ;
|
||||
16 [label="gini = 0.0\nsamples = 11\nvalue = [[11, 0]\n[0, 11]]", fillcolor="#e58139ff"] ;
|
||||
15 -> 16 ;
|
||||
17 [label="mean texture <= 13.745\ngini = 0.32\nsamples = 5\nvalue = [[1, 4]\n[4, 1]]", fillcolor="#e5813955"] ;
|
||||
15 -> 17 ;
|
||||
18 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 18 ;
|
||||
19 [label="gini = 0.0\nsamples = 4\nvalue = [[0, 4]\n[4, 0]]", fillcolor="#e58139ff"] ;
|
||||
17 -> 19 ;
|
||||
20 [label="mean concave points <= 0.049\ngini = 0.088\nsamples = 151\nvalue = [[7, 144]\n[144, 7]]", fillcolor="#e58139d0"] ;
|
||||
14 -> 20 ;
|
||||
21 [label="concave points error <= 0.01\ngini = 0.48\nsamples = 15\nvalue = [[6, 9]\n[9, 6]]", fillcolor="#e5813900"] ;
|
||||
20 -> 21 ;
|
||||
22 [label="gini = 0.0\nsamples = 9\nvalue = [[0, 9]\n[9, 0]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 22 ;
|
||||
23 [label="gini = 0.0\nsamples = 6\nvalue = [[6, 0]\n[0, 6]]", fillcolor="#e58139ff"] ;
|
||||
21 -> 23 ;
|
||||
24 [label="worst smoothness <= 0.096\ngini = 0.015\nsamples = 136\nvalue = [[1, 135]\n[135, 1]]", fillcolor="#e58139f7"] ;
|
||||
20 -> 24 ;
|
||||
25 [label="gini = 0.0\nsamples = 1\nvalue = [[1, 0]\n[0, 1]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 25 ;
|
||||
26 [label="gini = 0.0\nsamples = 135\nvalue = [[0, 135]\n[135, 0]]", fillcolor="#e58139ff"] ;
|
||||
24 -> 26 ;
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 216 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
@@ -0,0 +1,11 @@
|
||||
Grade Trend,Hours slept,Hours Studied,Grade
|
||||
1,0,1,1
|
||||
0,1,0,0
|
||||
1,0,1,1
|
||||
1,1,1,1
|
||||
0,0,1,0
|
||||
1,0,0,0
|
||||
0,1,1,0
|
||||
0,0,1,0
|
||||
1,0,0,0
|
||||
1,1,1,1
|
||||
|
@@ -0,0 +1,15 @@
|
||||
Outlook,Temperature,Humidity,Wind,Ride
|
||||
0,0,0,0,0
|
||||
0,0,0,1,1
|
||||
1,0,0,0,1
|
||||
2,1,0,0,1
|
||||
2,2,1,0,1
|
||||
2,2,1,1,0
|
||||
1,2,1,1,1
|
||||
0,1,0,0,0
|
||||
0,2,1,0,1
|
||||
2,1,1,0,1
|
||||
0,1,1,1,1
|
||||
1,1,0,1,1
|
||||
1,0,1,0,1
|
||||
2,1,0,1,0
|
||||
|
@@ -0,0 +1,13 @@
|
||||
digraph Tree {
|
||||
node [shape=box, style="filled, rounded", color="black", fontname=helvetica] ;
|
||||
edge [fontname=helvetica] ;
|
||||
0 [label="X[7] <= 0.5\ngini = 0.48\nsamples = 15\nvalue = [4, 10, 1]", fillcolor="#39e5818b"] ;
|
||||
1 [label="X[1] <= 0.5\ngini = 0.408\nsamples = 14\nvalue = [4, 10, 0]", fillcolor="#39e58199"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="gini = 0.48\nsamples = 10\nvalue = [4, 6, 0]", fillcolor="#39e58155"] ;
|
||||
1 -> 2 ;
|
||||
3 [label="gini = 0.0\nsamples = 4\nvalue = [0, 4, 0]", fillcolor="#39e581ff"] ;
|
||||
1 -> 3 ;
|
||||
4 [label="gini = 0.0\nsamples = 1\nvalue = [0, 0, 1]", fillcolor="#8139e5ff"] ;
|
||||
0 -> 4 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
Day,Outlook,Temperature,Humidity,Wind,Ride
|
||||
1,Sunny,Hot,High,Weak,0
|
||||
2,Sunny,Hot,High,Strong,1
|
||||
3,Overcast,Hot,High,Weak,1
|
||||
4,Rain,Mild,High,Weak,1
|
||||
5,Rain,Cool,Normal,Weak,1
|
||||
6,Rain,Cool,Normal,Strong,0
|
||||
7,Overcast,Cool,Normal,Strong,1
|
||||
8,Sunny,Mild,High,Weak,0
|
||||
9,Sunny,Cool,Normal,Weak,1
|
||||
10,Rain,Mild,Normal,Weak,1
|
||||
11,Sunny,Mild,Normal,Strong,1
|
||||
12,Overcast,Mild,High,Strong,1
|
||||
13,Overcast,Hot,Normal,Weak,1
|
||||
14,Rain,Mild,High,Strong,0
|
||||
|
@@ -0,0 +1,101 @@
|
||||
aardvark,1,0,0,1,0,0,1,1,1,1,0,0,4,0,0,1,1
|
||||
antelope,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
bass,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
bear,1,0,0,1,0,0,1,1,1,1,0,0,4,0,0,1,1
|
||||
boar,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
buffalo,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
calf,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
carp,0,0,1,0,0,1,0,1,1,0,0,1,0,1,1,0,4
|
||||
catfish,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
cavy,1,0,0,1,0,0,0,1,1,1,0,0,4,0,1,0,1
|
||||
cheetah,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
chicken,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,2
|
||||
chub,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
clam,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,7
|
||||
crab,0,0,1,0,0,1,1,0,0,0,0,0,4,0,0,0,7
|
||||
crayfish,0,0,1,0,0,1,1,0,0,0,0,0,6,0,0,0,7
|
||||
crow,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,0,2
|
||||
deer,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
dogfish,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,4
|
||||
dolphin,0,0,0,1,0,1,1,1,1,1,0,1,0,1,0,1,1
|
||||
dove,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,2
|
||||
duck,0,1,1,0,1,1,0,0,1,1,0,0,2,1,0,0,2
|
||||
elephant,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
flamingo,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,1,2
|
||||
flea,0,0,1,0,0,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
frog,0,0,1,0,0,1,1,1,1,1,0,0,4,0,0,0,5
|
||||
frog,0,0,1,0,0,1,1,1,1,1,1,0,4,0,0,0,5
|
||||
fruitbat,1,0,0,1,1,0,0,1,1,1,0,0,2,1,0,0,1
|
||||
giraffe,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
girl,1,0,0,1,0,0,1,1,1,1,0,0,2,0,1,1,1
|
||||
gnat,0,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
goat,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
gorilla,1,0,0,1,0,0,0,1,1,1,0,0,2,0,0,1,1
|
||||
gull,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,2
|
||||
haddock,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,4
|
||||
hamster,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,0,1
|
||||
hare,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,0,1
|
||||
hawk,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,0,2
|
||||
herring,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
honeybee,1,0,1,0,1,0,0,0,0,1,1,0,6,0,1,0,6
|
||||
housefly,1,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
kiwi,0,1,1,0,0,0,1,0,1,1,0,0,2,1,0,0,2
|
||||
ladybird,0,0,1,0,1,0,1,0,0,1,0,0,6,0,0,0,6
|
||||
lark,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
leopard,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
lion,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
lobster,0,0,1,0,0,1,1,0,0,0,0,0,6,0,0,0,7
|
||||
lynx,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
mink,1,0,0,1,0,1,1,1,1,1,0,0,4,1,0,1,1
|
||||
mole,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,0,1
|
||||
mongoose,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
moth,1,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
newt,0,0,1,0,0,1,1,1,1,1,0,0,4,1,0,0,5
|
||||
octopus,0,0,1,0,0,1,1,0,0,0,0,0,8,0,0,1,7
|
||||
opossum,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,0,1
|
||||
oryx,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,1
|
||||
ostrich,0,1,1,0,0,0,0,0,1,1,0,0,2,1,0,1,2
|
||||
parakeet,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,2
|
||||
penguin,0,1,1,0,0,1,1,0,1,1,0,0,2,1,0,1,2
|
||||
pheasant,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
pike,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,4
|
||||
piranha,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,4
|
||||
pitviper,0,0,1,0,0,0,1,1,1,1,1,0,0,1,0,0,3
|
||||
platypus,1,0,1,1,0,1,1,0,1,1,0,0,4,1,0,1,1
|
||||
polecat,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
pony,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
porpoise,0,0,0,1,0,1,1,1,1,1,0,1,0,1,0,1,1
|
||||
puma,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
pussycat,1,0,0,1,0,0,1,1,1,1,0,0,4,1,1,1,1
|
||||
raccoon,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
reindeer,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,1
|
||||
rhea,0,1,1,0,0,0,1,0,1,1,0,0,2,1,0,1,2
|
||||
scorpion,0,0,0,0,0,0,1,0,0,1,1,0,8,1,0,0,7
|
||||
seahorse,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,4
|
||||
seal,1,0,0,1,0,1,1,1,1,1,0,1,0,0,0,1,1
|
||||
sealion,1,0,0,1,0,1,1,1,1,1,0,1,2,1,0,1,1
|
||||
seasnake,0,0,0,0,0,1,1,1,1,0,1,0,0,1,0,0,3
|
||||
seawasp,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,7
|
||||
skimmer,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,2
|
||||
skua,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,2
|
||||
slowworm,0,0,1,0,0,0,1,1,1,1,0,0,0,1,0,0,3
|
||||
slug,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,7
|
||||
sole,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,4
|
||||
sparrow,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
squirrel,1,0,0,1,0,0,0,1,1,1,0,0,2,1,0,0,1
|
||||
starfish,0,0,1,0,0,1,1,0,0,0,0,0,5,0,0,0,7
|
||||
stingray,0,0,1,0,0,1,1,1,1,0,1,1,0,1,0,1,4
|
||||
swan,0,1,1,0,1,1,0,0,1,1,0,0,2,1,0,1,2
|
||||
termite,0,0,1,0,0,0,0,0,0,1,0,0,6,0,0,0,6
|
||||
toad,0,0,1,0,0,1,0,1,1,1,0,0,4,0,0,0,5
|
||||
tortoise,0,0,1,0,0,0,0,0,1,1,0,0,4,1,0,1,3
|
||||
tuatara,0,0,1,0,0,0,1,1,1,1,0,0,4,1,0,0,3
|
||||
tuna,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,4
|
||||
vampire,1,0,0,1,1,0,0,1,1,1,0,0,2,1,0,0,1
|
||||
vole,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,0,1
|
||||
vulture,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,1,2
|
||||
wallaby,1,0,0,1,0,0,0,1,1,1,0,0,2,1,0,1,1
|
||||
wasp,1,0,1,0,1,0,0,0,0,1,1,0,6,0,0,0,6
|
||||
wolf,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,1
|
||||
worm,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,7
|
||||
wren,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,2
|
||||
|
+166
-21
@@ -15,7 +15,7 @@ DATE: November 18-22, 2024
|
||||
o Basics of decision trees, classification and regression algorithms and ensemble models
|
||||
o Readings and Videos:
|
||||
o These lecture notes at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week47/ipynb/week47.ipynb"
|
||||
o See also lecture notes from week 46 at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week46.ipynb"
|
||||
o See also lecture notes from week 46 at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/pub/week46/ipynb/week46.ipynb". The lecture on Monday starts with a repetition on how to make a decision tree.
|
||||
# * "Video of Lecture":"https://youtu.be/SpWXsvn5I9E"
|
||||
# * "Whiteboard notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesNov23.pdf"
|
||||
o Video on Decision trees URL:"https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn"
|
||||
@@ -468,29 +468,10 @@ In computations we will translate all classes into numbers. Being
|
||||
these binary classes, they can easily be split into ones and zeros.
|
||||
|
||||
!bblock Gini index for Average trend
|
||||
"See handwritten notes November 3":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2022/NotesNov32022.pdf"
|
||||
See whiteboard notes from lecture November 11 at URL:"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesNovember11.pdf"
|
||||
!eblock
|
||||
|
||||
|
||||
!split
|
||||
===== Computing the various Gini Indices, Hours slept =====
|
||||
|
||||
|
||||
!bblock Gini index for hour slept
|
||||
"See handwritten notes November 3":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2022/NotesNov32022.pdf"
|
||||
!eblock
|
||||
|
||||
|
||||
!split
|
||||
===== Computing the various Gini Indices, Hours studied =====
|
||||
|
||||
|
||||
!bblock Gini index for hour studied
|
||||
"See handwritten notes November 3":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2022/NotesNov32022.pdf"
|
||||
!eblock
|
||||
|
||||
For final tree, see the above handwritten notes
|
||||
|
||||
|
||||
!split
|
||||
===== A possible code using Scikit-Learn =====
|
||||
@@ -1714,7 +1695,90 @@ skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Making an ADAboost code yourself =====
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
|
||||
class DecisionStump:
|
||||
def fit(self, X, y, weights):
|
||||
m, n = X.shape
|
||||
self.alpha = 0
|
||||
self.threshold = None
|
||||
self.polarity = 1
|
||||
|
||||
min_error = float('inf')
|
||||
|
||||
for feature in range(n):
|
||||
feature_values = np.unique(X[:, feature])
|
||||
|
||||
for threshold in feature_values:
|
||||
for polarity in [1, -1]:
|
||||
predictions = np.ones(m)
|
||||
predictions[X[:, feature] < threshold] = -1
|
||||
predictions *= polarity
|
||||
|
||||
error = sum(weights[predictions != y])
|
||||
|
||||
if error < min_error:
|
||||
min_error = error
|
||||
self.alpha = 0.5 * np.log((1 - error) / (error + 1e-10))
|
||||
self.threshold = threshold
|
||||
self.feature_index = feature
|
||||
self.polarity = polarity
|
||||
|
||||
def predict(self, X):
|
||||
m = X.shape[0]
|
||||
predictions = np.ones(m)
|
||||
if self.polarity == 1:
|
||||
predictions[X[:, self.feature_index] < self.threshold] = -1
|
||||
else:
|
||||
predictions[X[:, self.feature_index] >= self.threshold] = -1
|
||||
return predictions
|
||||
|
||||
class AdaBoost:
|
||||
def fit(self, X, y, n_estimators):
|
||||
m = X.shape[0]
|
||||
self.alphas = []
|
||||
self.models = []
|
||||
|
||||
weights = np.ones(m) / m
|
||||
|
||||
for _ in range(n_estimators):
|
||||
stump = DecisionStump()
|
||||
stump.fit(X, y, weights)
|
||||
predictions = stump.predict(X)
|
||||
|
||||
error = sum(weights[predictions != y])
|
||||
if error == 0:
|
||||
break
|
||||
|
||||
self.models.append(stump)
|
||||
self.alphas.append(stump.alpha)
|
||||
|
||||
weights *= np.exp(-stump.alpha * y * predictions)
|
||||
weights /= np.sum(weights)
|
||||
|
||||
def predict(self, X):
|
||||
final_predictions = np.zeros(X.shape[0])
|
||||
for alpha, model in zip(self.alphas, self.models):
|
||||
final_predictions += alpha * model.predict(X)
|
||||
return np.sign(final_predictions)
|
||||
|
||||
# Example dataset (X, y)
|
||||
X = np.array([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]])
|
||||
y = np.array([-1, -1, -1, -1, 1, 1, 1, 1, 1, 1]) # Labels must be -1 or 1
|
||||
|
||||
# Train AdaBoost
|
||||
ada = AdaBoost()
|
||||
ada.fit(X, y, n_estimators=10)
|
||||
|
||||
# Predictions
|
||||
predictions = ada.predict(X)
|
||||
print("Predictions:", predictions)
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent =====
|
||||
@@ -2024,4 +2088,85 @@ plt.show()
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Gradient boosting, making our own code for a regression case =====
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
class DecisionTreeRegressor:
|
||||
def __init__(self, max_depth=3):
|
||||
self.max_depth = max_depth
|
||||
self.tree = None
|
||||
def fit(self, X, y):
|
||||
self.tree = self._grow_tree(X, y)
|
||||
def _grow_tree(self, X, y, depth=0):
|
||||
n_samples, n_features = X.shape
|
||||
if depth < self.max_depth:
|
||||
best_feature, best_threshold = self._best_split(X, y)
|
||||
if best_feature is not None:
|
||||
left_indices = X[:, best_feature] < best_threshold
|
||||
right_indices = X[:, best_feature] >= best_threshold
|
||||
left_child = self._grow_tree(X[left_indices], y[left_indices], depth + 1)
|
||||
right_child = self._grow_tree(X[right_indices], y[right_indices], depth + 1)
|
||||
return (best_feature, best_threshold, left_child, right_child)
|
||||
return np.mean(y)
|
||||
def _best_split(self, X, y):
|
||||
best_mse = float('inf')
|
||||
best_feature, best_threshold = None, None
|
||||
n_samples, n_features = X.shape
|
||||
|
||||
for feature in range(n_features):
|
||||
thresholds = np.unique(X[:, feature])
|
||||
for threshold in thresholds:
|
||||
left_indices = X[:, feature] < threshold
|
||||
right_indices = X[:, feature] >= threshold
|
||||
if len(y[left_indices]) > 0 and len(y[right_indices]) > 0:
|
||||
left_mse = np.mean((y[left_indices] - np.mean(y[left_indices])) ** 2)
|
||||
right_mse = np.mean((y[right_indices] - np.mean(y[right_indices])) ** 2)
|
||||
mse = (len(y[left_indices]) * left_mse + len(y[right_indices]) * right_mse) / n_samples
|
||||
|
||||
if mse < best_mse:
|
||||
best_mse = mse
|
||||
best_feature = feature
|
||||
best_threshold = threshold
|
||||
return best_feature, best_threshold
|
||||
def predict(self, X):
|
||||
return np.array([self._predict_sample(sample, self.tree) for sample in X])
|
||||
def _predict_sample(self, sample, node):
|
||||
if isinstance(node, tuple):
|
||||
feature, threshold, left_child, right_child = node
|
||||
if sample[feature] < threshold:
|
||||
return self._predict_sample(sample, left_child)
|
||||
else:
|
||||
return self._predict_sample(sample, right_child)
|
||||
return node
|
||||
class GradientBoostingRegressor:
|
||||
def __init__(self, n_estimators=100, learning_rate=0.1, max_depth=3):
|
||||
self.n_estimators = n_estimators
|
||||
self.learning_rate = learning_rate
|
||||
self.max_depth = max_depth
|
||||
self.models = []
|
||||
def fit(self, X, y):
|
||||
y_pred = np.zeros(y.shape)
|
||||
for _ in range(self.n_estimators):
|
||||
residuals = y - y_pred
|
||||
model = DecisionTreeRegressor(max_depth=self.max_depth)
|
||||
model.fit(X, residuals)
|
||||
y_pred += self.learning_rate * model.predict(X)
|
||||
self.models.append(model)
|
||||
def predict(self, X):
|
||||
y_pred = np.zeros(X.shape[0])
|
||||
for model in self.models:
|
||||
y_pred += self.learning_rate * model.predict(X)
|
||||
return y_pred
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Sample data
|
||||
X = np.array([[1], [2], [3], [4], [5]])
|
||||
y = np.array([1.5, 1.7, 3.5, 3.7, 5.0])
|
||||
model = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, max_depth=2)
|
||||
model.fit(X, y)
|
||||
predictions = model.predict(X)
|
||||
print("Predictions:", predictions)
|
||||
|
||||
!ec
|
||||
|
||||
Reference in New Issue
Block a user