mhjensen
113 lines
3.4 KiB
Plaintext
113 lines
3.4 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import nltk\n",
|
|
"from nltk.tokenize import word_tokenize\n",
|
|
"from nltk.stem import WordNetLemmatizer\n",
|
|
"import numpy as np\n",
|
|
"import random\n",
|
|
"import pickle\n",
|
|
"from collections import Counter\n",
|
|
"\n",
|
|
"lemmatizer=WordNetLemmatizer()\n",
|
|
"hm_lines=1000000\n",
|
|
"\n",
|
|
"def create_lexicon(pos,neg):\n",
|
|
" lexicon=[]\n",
|
|
" for fi in [pos,neg]:\n",
|
|
" with open(fi, 'ri') as f:\n",
|
|
" contents=f.readlines()\n",
|
|
" for l in contents[:hm_lines]:\n",
|
|
" all_words=word_tokenize(l.lower())\n",
|
|
" lexicon+=list(all_words)\n",
|
|
" \n",
|
|
" \n",
|
|
" lexicon=[lemmatizer.lemmatize(i) for i in lexicon] \n",
|
|
" w_counts=Counter(lexicon)\n",
|
|
" l2=[]\n",
|
|
" for w in w_counts:\n",
|
|
" if 1000 > w_counts[w] >50:\n",
|
|
" l2.append(w)\n",
|
|
" \n",
|
|
" return l2\n",
|
|
" \n",
|
|
" \n",
|
|
"def sample_handling(sample, lexicon, classification):\n",
|
|
" featureset=[]\n",
|
|
" with open(sample, 'ri') as f:\n",
|
|
" contents=f.readlines()\n",
|
|
" for l in contents[:hm_lines]:\n",
|
|
" current_words=word_tokenize(l.lower())\n",
|
|
" current_words=[lemmatizer.lemmatize(i) for i in current_words]\n",
|
|
" features=np.zeros(len(lexicon))\n",
|
|
" for word in current_words:\n",
|
|
" if word.lower() in lexicon:\n",
|
|
" index_value=lexicon.index(word.lower())\n",
|
|
" feature[index_value]+=1\n",
|
|
" features=list(features)\n",
|
|
" featureset.append([features, classification])\n",
|
|
" \n",
|
|
" return featureset\n",
|
|
" \n",
|
|
" \n",
|
|
" \n",
|
|
" \n",
|
|
"def creat_featuresets_and_labels(pos, neg, test_size=0.1):\n",
|
|
" lexicon=create_lexicon(pos,neg)\n",
|
|
" features=[]\n",
|
|
" features+=sample_handling('pos.txt', lexicon, [1,0])\n",
|
|
" features+=sample_handling('neg.txt', lexicon, [0,1])\n",
|
|
" random.shuffle(features)\n",
|
|
" features=np.array(features)\n",
|
|
" testing_size=int(test_size*len(features))\n",
|
|
" train_x=list(features[:,0][:-testing_size]) #creates a list of the 0th element of every list in the overall list\n",
|
|
" train_y=list(features[:,1][:-testing_size])\n",
|
|
" \n",
|
|
" test_x=list(features[:,0][-testing_size:]) \n",
|
|
" test_y=list(features[:,1][-testing_size:])\n",
|
|
" \n",
|
|
" return train_x, train_y, test_x, test_y\n",
|
|
" \n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|