{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import numpy as np\n",
    "import random\n",
    "import pickle\n",
    "from collections import Counter\n",
    "\n",
    "lemmatizer=WordNetLemmatizer()\n",
    "hm_lines=1000000\n",
    "\n",
    "def create_lexicon(pos,neg):\n",
    "    lexicon=[]\n",
    "    for fi in [pos,neg]:\n",
    "        with open(fi, 'ri') as f:\n",
    "            contents=f.readlines()\n",
    "            for l in contents[:hm_lines]:\n",
    "                all_words=word_tokenize(l.lower())\n",
    "                lexicon+=list(all_words)\n",
    "                \n",
    "                \n",
    "    lexicon=[lemmatizer.lemmatize(i) for i in lexicon] \n",
    "    w_counts=Counter(lexicon)\n",
    "    l2=[]\n",
    "    for w in w_counts:\n",
    "        if 1000 > w_counts[w] >50:\n",
    "            l2.append(w)\n",
    "            \n",
    "    return l2\n",
    " \n",
    "    \n",
    "def sample_handling(sample, lexicon, classification):\n",
    "    featureset=[]\n",
    "    with open(sample, 'ri') as f:\n",
    "        contents=f.readlines()\n",
    "        for l in contents[:hm_lines]:\n",
    "            current_words=word_tokenize(l.lower())\n",
    "            current_words=[lemmatizer.lemmatize(i) for i in current_words]\n",
    "            features=np.zeros(len(lexicon))\n",
    "            for word in current_words:\n",
    "                if word.lower() in lexicon:\n",
    "                    index_value=lexicon.index(word.lower())\n",
    "                    feature[index_value]+=1\n",
    "                    features=list(features)\n",
    "                    featureset.append([features, classification])\n",
    "                    \n",
    "    return featureset\n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "def creat_featuresets_and_labels(pos, neg, test_size=0.1):\n",
    "    lexicon=create_lexicon(pos,neg)\n",
    "    features=[]\n",
    "    features+=sample_handling('pos.txt', lexicon, [1,0])\n",
    "    features+=sample_handling('neg.txt', lexicon, [0,1])\n",
    "    random.shuffle(features)\n",
    "    features=np.array(features)\n",
    "    testing_size=int(test_size*len(features))\n",
    "    train_x=list(features[:,0][:-testing_size]) #creates a list of the 0th element of every list in the overall list\n",
    "    train_y=list(features[:,1][:-testing_size])\n",
    "    \n",
    "    test_x=list(features[:,0][-testing_size:]) \n",
    "    test_y=list(features[:,1][-testing_size:])\n",
    "    \n",
    "    return train_x, train_y, test_x, test_y\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}