Gensim is a free Python library designed to automatically extract semantic topics from documents, as efficiently (computer-wise) and painlessly (human-wise) as possible.
Gensim aims at processing raw, unstructured digital texts (“plain text”). The algorithms in gensim, such as Latent Semantic Analysis, Latent Dirichlet Allocation or Random Projections, discover semantic structure of documents, by examining word statistical co-occurrence patterns within a corpus of training documents. These algorithms are unsupervised, which means no human input is necessary – you only need a corpus of plain text documents.
Once these statistical patterns are found, any plain text documents can be succinctly expressed in the new, semantic representation, and queried for topical similarity against other documents.
Library documentation: https://radimrehurek.com/gensim/index.html
from gensim import corpora, models, similarities
documents = ["Human machine interface for lab abc computer applications",
"A survey of user opinion of computer system response time",
"The EPS user interface management system",
"System and human system engineering testing of EPS",
"Relation of user perceived response time to error measurement",
"The generation of random binary unordered trees",
"The intersection graph of paths in trees",
"Graph minors IV Widths of trees and well quasi ordering",
"Graph minors A survey"]
# remove common words and tokenize
stoplist = set('for a of the and to in'.split())
texts = [[word for word in document.lower().split() if word not in stoplist]
for document in documents]
# remove words that appear only once
from collections import defaultdict
frequency = defaultdict(int)
for text in texts:
for token in text:
frequency[token] += 1
texts = [[token for token in text if frequency[token] > 1]
for text in texts]
from pprint import pprint
pprint(texts)
# create a dictionary mapping between ids and unique words
dictionary = corpora.Dictionary(texts)
print(dictionary)
# mapping between ids and words
print(dictionary.token2id)
# convert the text to a bag-of-words corpus
corpus = [dictionary.doc2bow(text) for text in texts]
pprint(corpus)
# can convert to numpy/scipy matrices and back
from gensim import matutils
numpy_matrix = matutils.corpus2dense(corpus, num_terms=12)
print(numpy_matrix)
scipy_csc_matrix = matutils.corpus2csc(corpus)
numpy_corpus = matutils.Dense2Corpus(numpy_matrix)
scipy_corpus = matutils.Sparse2Corpus(scipy_csc_matrix)
# initialize a TF-IDF transformation
tfidf = models.TfidfModel(corpus)
# apply it to the whole corpus
corpus_tfidf = tfidf[corpus]
for doc in corpus_tfidf:
print(doc)
# initialize an LSI transformation
lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=2)
lsi.print_topics(2)
# create a double wrapper over the original corpus: bow->tfidf->lsi
corpus_lsi = lsi[corpus_tfidf]
for doc in corpus_lsi:
print(doc)
# random projection model
rp = models.RpModel(corpus_tfidf, num_topics=500)
# latent dirichlet allocation model
lda = models.LdaModel(corpus, id2word=dictionary, num_topics=100)
# convert a phrase into the LSI model space
doc = "Human computer interaction"
vec_bow = dictionary.doc2bow(doc.lower().split())
vec_lsi = lsi[vec_bow] # convert the query to LSI space
print(vec_lsi)
# index the transformed corpus from earlier
index = similarities.MatrixSimilarity(corpus_lsi)
# perform a similarity query against the corpus using cosine similarity
sims = index[vec_lsi]
print(list(enumerate(sims)))
# display in sorted order
sims = sorted(enumerate(sims), key=lambda item: -item[1])
pprint(sims)