Python Tf IDF算法

Python Tf IDF算法

这是使用scikit-learn的Tf-
idf算法的实现。在应用之前，你可以

word_tokenize()

和干你的话。

import pandas as pdfrom sklearn.feature_extraction.text import TfidfVectorizerfrom nltk import word_tokenizefrom nltk.stem.porter import PorterStemmerdef tokenize(text):    tokens = word_tokenize(text)    stems = []    for item in tokens: stems.append(PorterStemmer().stem(item))    return stems# your corpustext = ["This is your first text book", "This is the third text for analysis", "This is another text"]# word tokenize and stemtext = [" ".join(tokenize(txt.lower())) for txt in text]vectorizer = TfidfVectorizer()matrix = vectorizer.fit_transform(text).todense()# transform the matrix to a pandas dfmatrix = pd.Dataframe(matrix, columns=vectorizer.get_feature_names())# sum over each document (axis=0)top_words = matrix.sum(axis=0).sort_values(ascending=False)