jupyter files

2018-05-06 22:19:59 -04:00
parent c497ee89f2
commit 11d2a0f8fc
38 changed files with 9017 additions and 0 deletions
@@ -0,0 +1,112 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.stem import WordNetLemmatizer\n",
+    "import numpy as np\n",
+    "import random\n",
+    "import pickle\n",
+    "from collections import Counter\n",
+    "\n",
+    "lemmatizer=WordNetLemmatizer()\n",
+    "hm_lines=1000000\n",
+    "\n",
+    "def create_lexicon(pos,neg):\n",
+    "    lexicon=[]\n",
+    "    for fi in [pos,neg]:\n",
+    "        with open(fi, 'ri') as f:\n",
+    "            contents=f.readlines()\n",
+    "            for l in contents[:hm_lines]:\n",
+    "                all_words=word_tokenize(l.lower())\n",
+    "                lexicon+=list(all_words)\n",
+    "                \n",
+    "                \n",
+    "    lexicon=[lemmatizer.lemmatize(i) for i in lexicon] \n",
+    "    w_counts=Counter(lexicon)\n",
+    "    l2=[]\n",
+    "    for w in w_counts:\n",
+    "        if 1000 > w_counts[w] >50:\n",
+    "            l2.append(w)\n",
+    "            \n",
+    "    return l2\n",
+    " \n",
+    "    \n",
+    "def sample_handling(sample, lexicon, classification):\n",
+    "    featureset=[]\n",
+    "    with open(sample, 'ri') as f:\n",
+    "        contents=f.readlines()\n",
+    "        for l in contents[:hm_lines]:\n",
+    "            current_words=word_tokenize(l.lower())\n",
+    "            current_words=[lemmatizer.lemmatize(i) for i in current_words]\n",
+    "            features=np.zeros(len(lexicon))\n",
+    "            for word in current_words:\n",
+    "                if word.lower() in lexicon:\n",
+    "                    index_value=lexicon.index(word.lower())\n",
+    "                    feature[index_value]+=1\n",
+    "                    features=list(features)\n",
+    "                    featureset.append([features, classification])\n",
+    "                    \n",
+    "    return featureset\n",
+    "            \n",
+    "            \n",
+    "            \n",
+    "            \n",
+    "def creat_featuresets_and_labels(pos, neg, test_size=0.1):\n",
+    "    lexicon=create_lexicon(pos,neg)\n",
+    "    features=[]\n",
+    "    features+=sample_handling('pos.txt', lexicon, [1,0])\n",
+    "    features+=sample_handling('neg.txt', lexicon, [0,1])\n",
+    "    random.shuffle(features)\n",
+    "    features=np.array(features)\n",
+    "    testing_size=int(test_size*len(features))\n",
+    "    train_x=list(features[:,0][:-testing_size]) #creates a list of the 0th element of every list in the overall list\n",
+    "    train_y=list(features[:,1][:-testing_size])\n",
+    "    \n",
+    "    test_x=list(features[:,0][-testing_size:]) \n",
+    "    test_y=list(features[:,1][-testing_size:])\n",
+    "    \n",
+    "    return train_x, train_y, test_x, test_y\n",
+    "            \n",
+    "            "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}