I have implemented the text classification using tf-idf and SVM by following the tutorial from this tutorial

The classification is working properly.

Now I want to plot the tf-idf values (i.e. features) and also see how the final hyperplane generated that classifies the data into two classes.

The code implemented is as follows:

import os

import numpy as np

from sklearn.naive_bayes import MultinomialNB

from sklearn.metrics import confusion_matrix

from sklearn.svm import LinearSVC

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.model_selection import StratifiedKFold

def make_Corpus(root_dir):

polarity_dirs = [os.path.join(root_dir,f) for f in os.listdir(root_dir)]

corpus = []

for polarity_dir in polarity_dirs:

reviews = [os.path.join(polarity_dir,f) for f in os.listdir(polarity_dir)]

for review in reviews:

doc_string = "";

with open(review) as rev:

for line in rev:

doc_string = doc_string + line

if not corpus:

corpus = [doc_string]

else:

corpus.append(doc_string)

return corpus

#Create a corpus with each document having one string

root_dir = 'txt_sentoken'

corpus = make_Corpus(root_dir)

#Stratified 10-cross fold validation with SVM and Multinomial NB

labels = np.zeros(2000);

labels[0:1000]=0;

labels[1000:2000]=1;

kf = StratifiedKFold(n_splits=10)

totalsvm = 0 # Accuracy measure on 2000 files

totalNB = 0

totalMatSvm = np.zeros((2,2)); # Confusion matrix on 2000 files

totalMatNB = np.zeros((2,2));

for train_index, test_index in kf.split(corpus,labels):

X_train = [corpus[i] for i in train_index]

X_test = [corpus[i] for i in test_index]

y_train, y_test = labels[train_index], labels[test_index]

vectorizer = TfidfVectorizer(min_df=5, max_df = 0.8, sublinear_tf=True, use_idf=True,stop_words='english')

train_corpus_tf_idf = vectorizer.fit_transform(X_train)

test_corpus_tf_idf = vectorizer.transform(X_test)

model1 = LinearSVC()

model2 = MultinomialNB()

model1.fit(train_corpus_tf_idf,y_train)

model2.fit(train_corpus_tf_idf,y_train)

result1 = model1.predict(test_corpus_tf_idf)

result2 = model2.predict(test_corpus_tf_idf)

totalMatSvm = totalMatSvm + confusion_matrix(y_test, result1)

totalMatNB = totalMatNB + confusion_matrix(y_test, result2)

totalsvm = totalsvm+sum(y_test==result1)

totalNB = totalNB+sum(y_test==result2)

print totalMatSvm, totalsvm/2000.0, totalMatNB, totalNB/2000.0

I have read how to plot the graphs, but couldn't find any tutorial related to plot the features of tf-idf and also the hyperplane generated by SVM.

解决方案

First, you need to select only 2 features in order to create the 2-dimensional decision surface plot.

Example using some synthetic data:

from sklearn.svm import SVC

import numpy as np

import matplotlib.pyplot as plt

from sklearn import svm, datasets

from sklearn.datasets import fetch_20newsgroups

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

from sklearn.pipeline import Pipeline

import matplotlib.pyplot as plt

newsgroups_train = fetch_20newsgroups(subset='train',

categories=['alt.atheism', 'sci.space'])

pipeline = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer())])

X = pipeline.fit_transform(newsgroups_train.data).todense()

# Select ONLY 2 features

X = np.array(X)

X = X[:, [0,1]]

y = newsgroups_train.target

def make_meshgrid(x, y, h=.02):

x_min, x_max = x.min() - 1, x.max() + 1

y_min, y_max = y.min() - 1, y.max() + 1

xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

return xx, yy

def plot_contours(ax, clf, xx, yy, **params):

Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

Z = Z.reshape(xx.shape)

out = ax.contourf(xx, yy, Z, **params)

return out

model = svm.SVC(kernel='linear')

clf = model.fit(X, y)

fig, ax = plt.subplots()

# title for the plots

title = ('Decision surface of linear SVC ')

# Set-up grid for plotting.

X0, X1 = X[:, 0], X[:, 1]

xx, yy = make_meshgrid(X0, X1)

plot_contours(ax, clf, xx, yy, cmap=plt.cm.coolwarm, alpha=0.8)

ax.scatter(X0, X1, c=y, cmap=plt.cm.coolwarm, s=20, edgecolors='k')

ax.set_ylabel('y label here')

ax.set_xlabel('x label here')

ax.set_xticks(())

ax.set_yticks(())

ax.set_title(title)

ax.legend()

plt.show()

RESULTS

The plot is not nice since we selected randomly only 2 features to create it. One way to make it nice is the following: You could use a univariate ranking method (e.g. ANOVA F-value test) and find the best top-2 features. Then using these top-2 you could create a nice separating surface plot.