-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathhtml_bow_modular.py
173 lines (136 loc) · 7.78 KB
/
html_bow_modular.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from urllib.request import urlopen
import re
import os
import numpy as np
from random import shuffle
from sklearn.feature_extraction.text import CountVectorizer
from nltk.stem.porter import *
from nltk.tokenize import RegexpTokenizer
from sklearn.feature_extraction.text import TfidfTransformer
from nltk import sent_tokenize
from nltk import ne_chunk, pos_tag, word_tokenize
from nltk.tree import Tree
from nltk.corpus import stopwords
from nltk.tag.stanford import StanfordNERTagger
from sklearn.decomposition import TruncatedSVD
def term_frequency_matrix(documents, terms):
# Module to create the term frequency matrix
td_matrix = []
for itr in documents:
doc_terms = [itr[0].count(t) for t in terms]
td_matrix.append(doc_terms)
return np.array(td_matrix)
def tfidf_creation_module(root_pathname1, v1_pathname, root_pathname2, v2_pathname, principal_components, train_file_name, test_file_name):
# module to create train and test tfidf matrices and writes into file
text_container = [] # for storing the entire string of a webpage
unique_words = [] # stores the number of unique words in all the samples
path = root_pathname1 #'/home/sanjoy/Desktop/course-cotrain-data/fulltext/course/'
stemmer = PorterStemmer() # used for stemming
tokenizer = RegexpTokenizer(r'\w+') # for Regular expression
class_label = [] # holds the class labels
for filename in os.listdir(path):
filename = v1_pathname + filename
sock = urlopen(filename)
htmlSource = sock.read()
htmlSource = htmlSource.decode("windows-1252") # utf-8 could be used instead of "windows-1252"
sock.close()
class_label.append(0) # appends the class label
# for obtaining text inside <> tags
cleanr = re.compile('<.*?>')
htmlSource = re.sub(cleanr, '', htmlSource)
word_tokens = tokenizer.tokenize(htmlSource.lower()) # Changes to lower case
word_list = [stemmer.stem(line) for line in word_tokens if line not in ''] # stemming is being done
stop_words = set(stopwords.words('english')) # for stop word removal
word_tokens = [w for w in word_list if not w in stop_words]
unique_words += list(set(word_tokens))
unique_words = list(set(unique_words)) # updates unique word list
dummy_str = ""
for i in word_tokens:
dummy_str += i + " "
dummy_list = [dummy_str]
text_container.append(dummy_list) # Appends the entire text of a webpage into text_container
class_one_samples_count = len(class_label) # stores count of class one samples
path = root_pathname2
for filename in os.listdir(path):
# for every sample reads the data
print(filename)
filename = v2_pathname + filename
sock = urlopen(filename)
htmlSource = sock.read()
htmlSource = htmlSource.decode("windows-1252")
sock.close()
class_label.append(1)
# for obtaining text inside <> tags
cleanr = re.compile('<.*?>')
htmlSource = re.sub(cleanr, '', htmlSource)
# basic preprocessing
word_tokens = word_tokenize(htmlSource)
word_list = [stemmer.stem(line) for line in word_tokens if line not in '']
#stop_words = set(stopwords.words('english'))
word_tokens = [w for w in word_list if not w in stop_words]
unique_words += list(set(word_tokens))
unique_words = list(set(unique_words))
dummy_str = ""
for i in word_tokens:
dummy_str += i + " "
dummy_list = [dummy_str]
text_container.append(dummy_list)
class_two_samples_count = len(class_label) - class_one_samples_count
class_label = np.asarray(class_label)
class_label = class_label.reshape(class_label.shape[0], 1)
tf_matrix = term_frequency_matrix(text_container, unique_words)
tf = TfidfTransformer(norm='l2', use_idf=True, smooth_idf=True, sublinear_tf=False)
tf_idf_matrix = tf.fit_transform(tf_matrix).todense()
#Performs PCA to get top k principal components
svd = TruncatedSVD(n_components=principal_components, random_state=42)
tf_idf_matrix_SVD = svd.fit_transform(tf_idf_matrix)
tf_idf_matrix_with_labels = np.concatenate((tf_idf_matrix_SVD, class_label), axis=1)
class_one_test_samples = int(class_one_samples_count * 0.3) # Takes out 30% of samples for testing
test_tf_idf_matrix = tf_idf_matrix_with_labels[0:class_one_test_samples,:]
class_two_test_samples = int(class_two_samples_count * 0.3) # Takes out 30% of samples for testing
temp_test_tf_idf_matrix = tf_idf_matrix_with_labels[class_one_samples_count:(class_one_samples_count+class_two_test_samples),:]
test_tf_idf_matrix = np.concatenate((test_tf_idf_matrix,temp_test_tf_idf_matrix),axis = 0)
temp_matrix1 = tf_idf_matrix_with_labels[class_one_test_samples:class_one_samples_count,:]
temp_matrix2 = tf_idf_matrix_with_labels[(class_one_samples_count+class_two_test_samples):,:]
train_tf_idf_matrix = np.concatenate((temp_matrix1, temp_matrix2), axis=0)
# writes training tfidf into file
fp = open(train_file_name, 'w')
for i in range(train_tf_idf_matrix.shape[0]):
for j in range(train_tf_idf_matrix.shape[1]):
fp.write(str(train_tf_idf_matrix[i][j]) + " ")
fp.write("\n")
fp.close()
# writes test tfidf into file
fp = open(test_file_name, 'w')
for i in range(test_tf_idf_matrix.shape[0]):
for j in range(test_tf_idf_matrix.shape[1]):
fp.write(str(test_tf_idf_matrix[i][j]) + " ")
fp.write("\n")
fp.close()
def main():
#Main module to call subroutines
# for 1050 principal components
root_name1 = '/home/sanjoy/Desktop/course-cotrain-data/fulltext/course/'
view_name1 = 'file:///home/sanjoy/Desktop/course-cotrain-data/fulltext/course/'
root_name2 = '/home/sanjoy/Desktop/course-cotrain-data/fulltext/non-course/'
view_name2 = 'file:///home/sanjoy/Desktop/course-cotrain-data/fulltext/non-course/'
tfidf_creation_module(root_name1,view_name1,root_name2,view_name2, 1050, 'tfidf_matrix_fulltext_train_large.txt', 'tfidf_matrix_fulltext_test_large.txt')
root_name1 = '/home/sanjoy/Desktop/course-cotrain-data/inlinks/course/'
view_name1 = 'file:///home/sanjoy/Desktop/course-cotrain-data/inlinks/course/'
root_name2 = '/home/sanjoy/Desktop/course-cotrain-data/inlinks/non-course/'
view_name2 = 'file:///home/sanjoy/Desktop/course-cotrain-data/inlinks/non-course/'
tfidf_creation_module(root_name1,view_name1,root_name2,view_name2, 1050, 'tfidf_matrix_inlinks_train_large.txt', 'tfidf_matrix_inlinks_test_large.txt')
# for 100 principal components
root_name1 = '/home/sanjoy/Desktop/course-cotrain-data/fulltext/course/'
view_name1 = 'file:///home/sanjoy/Desktop/course-cotrain-data/fulltext/course/'
root_name2 = '/home/sanjoy/Desktop/course-cotrain-data/fulltext/non-course/'
view_name2 = 'file:///home/sanjoy/Desktop/course-cotrain-data/fulltext/non-course/'
tfidf_creation_module(root_name1,view_name1,root_name2,view_name2, 100, 'tfidf_matrix_fulltext_train_small.txt', 'tfidf_matrix_fulltext_test_small.txt')
root_name1 = '/home/sanjoy/Desktop/course-cotrain-data/inlinks/course/'
view_name1 = 'file:///home/sanjoy/Desktop/course-cotrain-data/inlinks/course/'
root_name2 = '/home/sanjoy/Desktop/course-cotrain-data/inlinks/non-course/'
view_name2 = 'file:///home/sanjoy/Desktop/course-cotrain-data/inlinks/non-course/'
tfidf_creation_module(root_name1,view_name1,root_name2,view_name2, 100, 'tfidf_matrix_inlinks_train_small.txt', 'tfidf_matrix_inlinks_test_small.txt')
main()