lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jupyter files

Browse Source
This commit is contained in:
mhjensen
2018-05-06 22:19:59 -04:00
parent c497ee89f2
commit 11d2a0f8fc
38 changed files with 9017 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/Programs/JupyterFiles/Examples/Youtube Tutorials/Deep Learning Sentdex.ipynb
+157
View File
@@ -0,0 +1,157 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
"Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
"Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
"Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
"Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
"Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
"Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
"Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n",
"WARNING:tensorflow:From <ipython-input-1-92265976c34d>:45: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"\n",
"Future major versions of TensorFlow will allow gradients to flow\n",
"into the labels input on backprop by default.\n",
"\n",
"See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
"\n",
"WARNING:tensorflow:From /anaconda3/lib/python3.6/site-packages/tensorflow/python/util/tf_should_use.py:118: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
"Instructions for updating:\n",
"Use `tf.global_variables_initializer` instead.\n",
"Epoch 0 completed out of 10 loss: 1664569.60834\n",
"Epoch 1 completed out of 10 loss: 414550.968437\n",
"Epoch 2 completed out of 10 loss: 229022.354944\n",
"Epoch 3 completed out of 10 loss: 136420.393392\n",
"Epoch 4 completed out of 10 loss: 88019.3560204\n",
"Epoch 5 completed out of 10 loss: 56024.820509\n",
"Epoch 6 completed out of 10 loss: 37434.4423951\n",
"Epoch 7 completed out of 10 loss: 29640.3100017\n",
"Epoch 8 completed out of 10 loss: 24399.9572706\n",
"Epoch 9 completed out of 10 loss: 23351.0056713\n",
"Accuracy: 0.9534\n"
]
}
],
"source": [
"import tensorflow as tf\n",
"from tensorflow.examples.tutorials.mnist import input_data\n",
"mnist=input_data.read_data_sets(\"/tmp/data/\", one_hot=True) #one component is on, all others are off\n",
"#10 classes, 0 through 9\n",
"#one-hot outputs 0=[1,0,0,0,0,0,0,0,0], being the 1 is in the algorithm's guess (0)\n",
"#3=[0,0,0,1,0,0,0,0,0]\n",
"n_nodes_hl1=500 #hl1 = hidden layer 1\n",
"n_nodes_hl2=500\n",
"n_nodes_hl3=500\n",
"n_classes=10 #number of categories\n",
"batch_size=100 #divies up the data to be more efficient, as opposed to loading all samples at once\n",
"\n",
"x=tf.placeholder('float',[None, 784])\n",
"y=tf.placeholder('float')\n",
"\n",
"def neural_network_model(data):\n",
" #(inputs*weights)+biases\n",
" hidden_1_layer={'weights':tf.Variable(tf.random_normal([784, n_nodes_hl1])), \n",
" 'biases': tf.Variable(tf.random_normal([n_nodes_hl1]))}\n",
" \n",
" hidden_2_layer={'weights':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])), \n",
" 'biases': tf.Variable(tf.random_normal([n_nodes_hl2]))}\n",
" \n",
" hidden_3_layer={'weights':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])), \n",
" 'biases': tf.Variable(tf.random_normal([n_nodes_hl3]))}\n",
" \n",
" output_layer={'weights':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])),\n",
" 'biases': tf.Variable(tf.random_normal([n_classes]))}\n",
" \n",
" l1=tf.add(tf.matmul(data, hidden_1_layer['weights']), hidden_1_layer['biases'])\n",
" l1=tf.nn.relu(l1)\n",
" \n",
" l2=tf.add(tf.matmul(l1, hidden_2_layer['weights']), hidden_2_layer['biases'])\n",
" l2=tf.nn.relu(l2)\n",
" \n",
" l3=tf.add(tf.matmul(l2, hidden_3_layer['weights']), hidden_3_layer['biases'])\n",
" l3=tf.nn.relu(l3)\n",
" \n",
" output=tf.matmul(l3, output_layer['weights'])+ output_layer['biases']\n",
" \n",
" return output\n",
" \n",
"def train_neural_network(x):\n",
" prediction=neural_network_model(x)\n",
" cost=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction,labels=y))\n",
" optimizer=tf.train.AdamOptimizer().minimize(cost)\n",
" \n",
" hm_epochs=10\n",
" \n",
" with tf.Session() as sess:\n",
" sess.run(tf.initialize_all_variables())\n",
" \n",
" for epoch in range(hm_epochs):\n",
" epoch_loss=0\n",
" for _ in range(int(mnist.train.num_examples/batch_size)):\n",
" epoch_x,epoch_y=mnist.train.next_batch(batch_size)\n",
" _,c=sess.run([optimizer,cost], feed_dict={x:epoch_x, y:epoch_y})\n",
" epoch_loss+=c\n",
" print('Epoch', epoch, 'completed out of ', hm_epochs, 'loss:', epoch_loss)\n",
" \n",
" correct=tf.equal(tf.argmax(prediction,1), tf.argmax(y,1))\n",
" \n",
" accuracy=tf.reduce_mean(tf.cast(correct, 'float'))\n",
" print('Accuracy:', accuracy.eval({x:mnist.test.images, y:mnist.test.labels}))\n",
" \n",
"\n",
" \n",
" \n",
"train_neural_network(x)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
doc/Programs/JupyterFiles/Examples/Youtube Tutorials/Digit Recognition Sentdex.ipynb
+101
View File
@@ -0,0 +1,101 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[ 0. 0. 5. ..., 0. 0. 0.]\n",
" [ 0. 0. 0. ..., 10. 0. 0.]\n",
" [ 0. 0. 0. ..., 16. 9. 0.]\n",
" ..., \n",
" [ 0. 0. 1. ..., 6. 0. 0.]\n",
" [ 0. 0. 2. ..., 12. 0. 0.]\n",
" [ 0. 0. 10. ..., 12. 1. 0.]]\n",
"[[ 0. 0. 5. ..., 0. 0. 0.]\n",
" [ 0. 0. 0. ..., 10. 0. 0.]\n",
" [ 0. 0. 0. ..., 16. 9. 0.]\n",
" ..., \n",
" [ 0. 0. 1. ..., 6. 0. 0.]\n",
" [ 0. 0. 2. ..., 12. 0. 0.]\n",
" [ 0. 0. 10. ..., 12. 1. 0.]]\n",
"(1796, 64)\n",
"prediction: [0 1 2 ..., 8 9 8]\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACvtJREFUeJzt3X+o3XUdx/HXy+vm2pwz0kJ2Z0uU\nkVQ6GTMZCW0VM0Un9McGCo3ggqE4CkTrHwv6V+yPEGQ6JZdSU0PMtJGKSrbc5kzn3WQtbbepU8LU\naZub7/64Z7DWjfO9O5/vj/vu+YCL98fhft6H+bzf7z33nO/HESEAOZ3Q9gAA6kPgQGIEDiRG4EBi\nBA4kRuBAYgQOJEbgQGIEDiR2Yh3fdLpPihmaVce3bpVnnNToegfnurG14sOhxtaavnd/Y2tl9S/t\n18E40Pd/kFoCn6FZutDL6vjWrRo6e0Gj673241r+eSb08YtzGlvrzJv/0NhaWW2K31e6HafoQGIE\nDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRWKXDby23vtL3L9o11DwWgjL6B2x6S9DNJl0g6V9Iq2+fW\nPRiAwVU5gi+WtCsidkfEQUn3Sbqi3rEAlFAl8LmS9hz18VjvcwA6rsqrGSZ6xcp/XUzd9oikEUma\noZkDjgWghCpH8DFJ8476eFjS3mNvFBG3R8SiiFg0Tc2+rBLAxKoE/pykc2x/zvZ0SSslPVTvWABK\n6HuKHhGHbF8r6TFJQ5LujIjttU8GYGCVrigQEY9IeqTmWQAUxjPZgMQIHEiMwIHECBxIjMCBxAgc\nSIzAgcQIHEisua0zEhj59W8aXW/FrPebW+yi5pZ6ZXVzWxetWXxlY2tJ0uE39zW6Xj8cwYHECBxI\njMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxKrsbHKn7X22X2piIADlVDmC3yVpec1zAKhB38Aj4ilJ\n/2hgFgCF8Ts4kFixV5OxdRHQPcWO4GxdBHQPp+hAYlX+THavpGclLbA9Zvs79Y8FoIQqe5OtamIQ\nAOVxig4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYlN+66IPrrywsbVWzNrW2FqS9Pnbv9vYWsNP\nfNjYWhvvXdfYWn+95uzG1pKkM29m6yIADSFwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCCx\nKhddnGf7Cdujtrfbvr6JwQAMrspz0Q9J+n5EbLU9W9IW2xsj4uWaZwMwoCp7k70eEVt7778naVTS\n3LoHAzC4Sb2azPZ8SQslbZrga2xdBHRM5QfZbJ8s6X5JayLi3WO/ztZFQPdUCtz2NI3HvT4iHqh3\nJAClVHkU3ZLukDQaEbfUPxKAUqocwZdIulrSUtvbem/frHkuAAVU2ZvsGUluYBYAhfFMNiAxAgcS\nI3AgMQIHEiNwIDECBxIjcCAxAgcSm/J7kx2Yk/dn1Alf/Gdja41pTmNrNem0Fw63PUKr8tYBgMCB\nzAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSKzKRRdn2P6T7Rd6Wxf9qInBAAyuylNVD0haGhHv9y6f\n/Izt30bEH2ueDcCAqlx0MSS93/twWu8t6hwKQBlVNz4Ysr1N0j5JGyNiwq2LbG+2vfkjHSg9J4Dj\nUCnwiDgcEedLGpa02PYXJrgNWxcBHTOpR9Ej4h1JT0paXss0AIqq8ij66bZP7b3/CUlfk7Sj7sEA\nDK7Ko+hnSLrb9pDGfyD8MiIerncsACVUeRT9zxrfExzAFMMz2YDECBxIjMCBxAgcSIzAgcQIHEiM\nwIHECBxIbMpvXfTJu55tbK3FuqaxtSTpJz/8eXOLfam5pdAcjuBAYgQOJEbgQGIEDiRG4EBiBA4k\nRuBAYgQOJEbgQGKVA+9dG/1521yPDZgiJnMEv17SaF2DACiv6s4mw5IulbS23nEAlFT1CH6rpBsk\nfVzjLAAKq7LxwWWS9kXElj63Y28yoGOqHMGXSLrc9quS7pO01PY9x96IvcmA7ukbeETcFBHDETFf\n0kpJj0fEVbVPBmBg/B0cSGxSV3SJiCc1vrsogCmAIziQGIEDiRE4kBiBA4kROJAYgQOJETiQGIED\niU35rYua1OQ2SZJ0211nN7peU1bs3dbYWrNfeaextSTpcKOr9ccRHEiMwIHECBxIjMCBxAgcSIzA\ngcQIHEiMwIHECBxIrNIz2XpXVH1P40/UORQRi+ocCkAZk3mq6lcj4u3aJgFQHKfoQGJVAw9Jv7O9\nxfZInQMBKKfqKfqSiNhr+9OSNtreERFPHX2DXvgjkjRDMwuPCeB4VDqCR8Te3n/3SXpQ0uIJbsPW\nRUDHVNl8cJbt2Ufel/QNSS/VPRiAwVU5Rf+MpAdtH7n9LyLi0VqnAlBE38AjYrek8xqYBUBh/JkM\nSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcTYumgSPrjywkbXe/u8oUbXa05zWxf9v+MIDiRG4EBi\nBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kVilw26fa3mB7h+1R2xfVPRiAwVV9qupPJT0aEd+yPV3i\nwufAVNA3cNunSLpY0rclKSIOSjpY71gASqhyin6WpLckrbP9vO21veujA+i4KoGfKOkCSbdFxEJJ\n+yXdeOyNbI/Y3mx780c6UHhMAMejSuBjksYiYlPv4w0aD/4/sHUR0D19A4+INyTtsb2g96llkl6u\ndSoARVR9FP06Set7j6DvlrS6vpEAlFIp8IjYJmlRzbMAKIxnsgGJETiQGIEDiRE4kBiBA4kROJAY\ngQOJETiQGIEDibE32SQcmNPsz8MvL3+xsbXWnfl0Y2ut/ttXGlvr8Padja3VRRzBgcQIHEiMwIHE\nCBxIjMCBxAgcSIzAgcQIHEiMwIHE+gZue4HtbUe9vWt7TRPDARhM36eqRsROSedLku0hSX+X9GDN\ncwEoYLKn6Msk/SUiXqtjGABlTfbFJisl3TvRF2yPSBqRpBlsPgp0QuUjeG/Tg8sl/Wqir7N1EdA9\nkzlFv0TS1oh4s65hAJQ1mcBX6X+cngPopkqB254p6euSHqh3HAAlVd2b7ANJn6p5FgCF8Uw2IDEC\nBxIjcCAxAgcSI3AgMQIHEiNwIDECBxJzRJT/pvZbkib7ktLTJL1dfJhuyHrfuF/t+WxEnN7vRrUE\nfjxsb46IRW3PUYes94371X2cogOJETiQWJcCv73tAWqU9b5xvzquM7+DAyivS0dwAIV1InDby23v\ntL3L9o1tz1OC7Xm2n7A9anu77evbnqkk20O2n7f9cNuzlGT7VNsbbO/o/dtd1PZMg2j9FL13rfVX\nNH7FmDFJz0laFREvtzrYgGyfIemMiNhqe7akLZJWTPX7dYTt70laJOmUiLis7XlKsX23pKcjYm3v\nQqMzI+Kdtuc6Xl04gi+WtCsidkfEQUn3Sbqi5ZkGFhGvR8TW3vvvSRqVNLfdqcqwPSzpUklr256l\nJNunSLpY0h2SFBEHp3LcUjcCnytpz1EfjylJCEfYni9poaRN7U5SzK2SbpD0cduDFHaWpLckrev9\n+rHW9qy2hxpEFwL3BJ9L89C+7ZMl3S9pTUS82/Y8g7J9maR9EbGl7VlqcKKkCyTdFhELJe2XNKUf\nE+pC4GOS5h318bCkvS3NUpTtaRqPe31EZLki7RJJl9t+VeO/Ti21fU+7IxUzJmksIo6caW3QePBT\nVhcCf07SObY/13tQY6Wkh1qeaWC2rfHf5UYj4pa25yklIm6KiOGImK/xf6vHI+KqlscqIiLekLTH\n9oLep5ZJmtIPik52b7LiIuKQ7WslPSZpSNKdEbG95bFKWCLpakkv2t7W+9wPIuKRFmdCf9dJWt87\n2OyWtLrleQbS+p/JANSnC6foAGpC4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBi/wYWKZOShpwGCQAA\nAABJRU5ErkJggg==\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x1a17854da0>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import matplotlib.pyplot as plt\n",
"import numpy\n",
"from sklearn import datasets\n",
"from sklearn import svm\n",
"digits=datasets.load_digits()\n",
"\n",
"clf=svm.SVC(gamma=0.001, C=100)\n",
"print(digits.data)\n",
"x,y=digits.data[:-1], digits.target[:-1]\n",
"clf.fit(x,y)\n",
"\n",
"print (digits.data)\n",
"print (x.shape)\n",
"\n",
"print(\"prediction:\", clf.predict(digits.data))\n",
"plt.imshow(digits.images[-2])\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
doc/Programs/JupyterFiles/Examples/Youtube Tutorials/Pandas Tutorial Sentdex.ipynb
+117
View File
@@ -0,0 +1,117 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Bounce_Rate Visitors\n",
"Day \n",
"1 65 43\n",
"2 72 53\n",
"3 62 34\n",
"4 64 45\n",
"5 54 64\n",
"6 66 34\n",
"Day\n",
"1 43\n",
"2 53\n",
"3 34\n",
"4 45\n",
"5 64\n",
"6 34\n",
"Name: Visitors, dtype: int64\n",
" Bounce_Rate Visitors\n",
"Day \n",
"1 65 43\n",
"2 72 53\n",
"3 62 34\n",
"4 64 45\n",
"5 54 64\n",
"6 66 34\n",
"[43, 53, 34, 45, 64, 34]\n",
"[[65 43]\n",
" [72 53]\n",
" [62 34]\n",
" [64 45]\n",
" [54 64]\n",
" [66 34]]\n"
]
}
],
"source": [
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"from matplotlib import style\n",
"style.use('ggplot')\n",
"import numpy as np\n",
"\n",
"web_stats={'Day':[1,2,3,4,5,6],\n",
" 'Visitors':[43,53,34,45,64,34],\n",
" 'Bounce_Rate':[65,72,62,64,54,66]}\n",
"\n",
"df=pd.DataFrame(web_stats)\n",
"\n",
"#print(df) #df stands for data frame\n",
"#print(df.head()) #prints the first 5 rows\n",
"#print(df.tail()) #prints the last 5 rows\n",
"#Specifying the number in the parentheses gives that number of rows\n",
"#print(df.head(2))\n",
"#print(df.tail(2))\n",
"\n",
"#df=df.set_index('Day')\n",
"\n",
"#OR you can do this:\n",
"df.set_index('Day', inplace=True)\n",
"print(df)\n",
"\n",
"#print (df['Visitors']) #prints specific column OR\n",
"print (df.Visitors)\n",
"\n",
"#referencing multiple columns\n",
"print (df[['Bounce_Rate','Visitors']])\n",
"\n",
"#making a list out of a column; this only work with one column because more than one would\n",
"#treat the dictionary like an array, which it isn't\n",
"print (df.Visitors.tolist())\n",
"\n",
"#to make it an array\n",
"print (np.array(df[['Bounce_Rate','Visitors']]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
doc/Programs/JupyterFiles/Examples/Youtube Tutorials/Stocks Sentdex.ipynb
+142
View File
File diff suppressed because one or more lines are too long
doc/Programs/JupyterFiles/Examples/Youtube Tutorials/TensorFlow Tutorial Sentdex.ipynb
+106
View File
@@ -0,0 +1,106 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tensor(\"Mul_11:0\", shape=(4,), dtype=int32)\n",
"[ 5 12 21 32]\n",
"[ 5 12 21 32]\n",
"30\n",
"30\n",
"30\n",
"30\n"
]
}
],
"source": [
"# Import `tensorflow`\n",
"import tensorflow as tf\n",
"import os\n",
"\n",
"# Initialize two constants\n",
"x1 = tf.constant([1,2,3,4])\n",
"x2 = tf.constant([5,6,7,8])\n",
"\n",
"# Multiply\n",
"result = tf.multiply(x1, x2)\n",
"\n",
"# Print the result\n",
"print(result)\n",
"\n",
"# Intialize the Session\n",
"sess = tf.Session()\n",
"\n",
"# Print the result\n",
"print(sess.run(result))\n",
"\n",
"# Close the session\n",
"sess.close()\n",
"\n",
"#Or you can run the session like so:\n",
"with tf.Session() as sess:\n",
" output = sess.run(result)\n",
" print(output)\n",
"\n",
" \n",
"y1=tf.constant(5)\n",
"y2=tf.constant(6)\n",
"result=tf.multiply(y1, y2)\n",
"sess=tf.Session()\n",
"print(sess.run(result))\n",
"sess.close()\n",
"\n",
"#or\n",
"\n",
"with tf.Session() as sess:\n",
" print (sess.run(result))\n",
" \n",
"#this closes the session automatically\n",
"\n",
"#try this:\n",
"with tf.Session() as sess:\n",
" output=sess.run(result)\n",
" print (output)\n",
" \n",
"print (output)\n",
"#you can't run sess.run(result) outside of the with action"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
doc/Programs/JupyterFiles/Examples/Youtube Tutorials/featuresets.ipynb
+112
View File
@@ -0,0 +1,112 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import nltk\n",
"from nltk.tokenize import word_tokenize\n",
"from nltk.stem import WordNetLemmatizer\n",
"import numpy as np\n",
"import random\n",
"import pickle\n",
"from collections import Counter\n",
"\n",
"lemmatizer=WordNetLemmatizer()\n",
"hm_lines=1000000\n",
"\n",
"def create_lexicon(pos,neg):\n",
" lexicon=[]\n",
" for fi in [pos,neg]:\n",
" with open(fi, 'ri') as f:\n",
" contents=f.readlines()\n",
" for l in contents[:hm_lines]:\n",
" all_words=word_tokenize(l.lower())\n",
" lexicon+=list(all_words)\n",
" \n",
" \n",
" lexicon=[lemmatizer.lemmatize(i) for i in lexicon] \n",
" w_counts=Counter(lexicon)\n",
" l2=[]\n",
" for w in w_counts:\n",
" if 1000 > w_counts[w] >50:\n",
" l2.append(w)\n",
" \n",
" return l2\n",
" \n",
" \n",
"def sample_handling(sample, lexicon, classification):\n",
" featureset=[]\n",
" with open(sample, 'ri') as f:\n",
" contents=f.readlines()\n",
" for l in contents[:hm_lines]:\n",
" current_words=word_tokenize(l.lower())\n",
" current_words=[lemmatizer.lemmatize(i) for i in current_words]\n",
" features=np.zeros(len(lexicon))\n",
" for word in current_words:\n",
" if word.lower() in lexicon:\n",
" index_value=lexicon.index(word.lower())\n",
" feature[index_value]+=1\n",
" features=list(features)\n",
" featureset.append([features, classification])\n",
" \n",
" return featureset\n",
" \n",
" \n",
" \n",
" \n",
"def creat_featuresets_and_labels(pos, neg, test_size=0.1):\n",
" lexicon=create_lexicon(pos,neg)\n",
" features=[]\n",
" features+=sample_handling('pos.txt', lexicon, [1,0])\n",
" features+=sample_handling('neg.txt', lexicon, [0,1])\n",
" random.shuffle(features)\n",
" features=np.array(features)\n",
" testing_size=int(test_size*len(features))\n",
" train_x=list(features[:,0][:-testing_size]) #creates a list of the 0th element of every list in the overall list\n",
" train_y=list(features[:,1][:-testing_size])\n",
" \n",
" test_x=list(features[:,0][-testing_size:]) \n",
" test_y=list(features[:,1][-testing_size:])\n",
" \n",
" return train_x, train_y, test_x, test_y\n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}