In [20]:
#code for WP posting
<code><script>
var myUrl = 'http://analytics.business/?p=153&preview=true';
if(window.top.location.href !== myUrl) {
window.top.location.href = myUrl;
}
</script></code>
#Analyze article clean, keywords, count, lexical dispersion & diversity April 11, 2017 Christa Taylor
#ref article http://www.cbc.ca/news/technology/trump-climate-change-executive-order-1.4043650
import nltk
nltk.download('maxent_ne_chunker')
nltk.download('words')
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('stopwords')
nltk.download('genesis')
nltk.download('wordnet')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
nltk.download('vader_lexicon')
from nltk.collocations import * #nltk.download()
import numpy
import pandas as pd
import matplotlib
import sys #reload(sys) #sys.setdefaultencoding("utf-8")
import os
import pickle
#ref article http://www.cbc.ca/news/technology/trump-climate-change-executive-order-1.4043650
os.chdir('d:/text/') #folder
with open('cbcnewstrump.txt', 'r') as f: #filename
sample = f.read()
sentences = nltk.sent_tokenize(sample)
tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
chunked_sentences = nltk.ne_chunk_sents(tagged_sentences, binary=True)
def extract_entity_names(t):
entity_names = []
if hasattr(t, 'label') and t.label:
if t.label() == 'NE':
entity_names.append(' '.join([child[0] for child in t]))
else:
for child in t:
entity_names.extend(extract_entity_names(child))
return entity_names
entity_names = []
for tree in chunked_sentences:
# Print results per sentence
# print extract_entity_names(tree)
entity_names.extend(extract_entity_names(tree))
# Print unique entity names
#print set(entity_names)
set(['climate change', 'Trump', 'American', 'power plant', 'wind power industry', 'Environmental', 'coal miner', 'jobs', 'Environmental Protection Agency', 'America', 'Clean Power Plan', 'order'])
##Sentiment
##Using vader: http://www.nltk.org/_modules/nltk/sentiment/vader.html
sent = pd.DataFrame(index = range(0, 27),columns=["full_sentence","compound","negative","neutral","positive"])
sid = SentimentIntensityAnalyzer()
i=0
for sentence in sentences:
sent["full_sentence"][i] = sentence
ss = sid.polarity_scores(sentence)
sent["compound"][i] = ss['compound']
sent["negative"][i] = ss['neg']
sent["positive"][i] = ss['pos']
sent["neutral"][i] =ss['neu']
i = i + 1
#print sent
print ("Mean Neutral Score:", sent["neutral"].mean() )
print ("Mean Positive Score:" , sent["positive"].mean())
print ("Mean Negative Score:" , sent["negative"].mean())
In [18]:
#NLTK text ref: http://www.nltk.org/book/ch01.html -- Chapter 1
from nltk.corpus import stopwords
sample = sample.lower() #lowercase everything
tokens = nltk.word_tokenize(sample) # tokenize it
mytext = nltk.Text(tokens) # turn text into a NLTK Text object
words = [w.lower() for w in mytext if w.isalpha()] #remove punctuation
filtered_words = [word for word in words if word not in stopwords.words('english')] #get rid of stop words
mytext2 = nltk.Text(filtered_words)
print (mytext2.dispersion_plot(["climate", "energy", "order", "coal", "trump"]))
#Get parts of speech
#Meanings: http://nishutayaltech.blogspot.in/2015/02/penn-treebank-pos-tags-in-natural.html
pos = nltk.pos_tag(mytext2)
pos_freq = nltk.FreqDist(tag for (word, tag) in pos)
pos_freq.most_common()
##In order:There are singular nouns, then adjectives, then plural nouns,
# verb present participle, verb past tense, adverbs, Verbs non-3rd person singular present,
# verbs, verbs 3rd person singular present, verb past participle, cardinal number, preposition, modal and superlative
#Determine how many tokens there are
print ("Length of the Full Text:" , len(words))
#Determine how many tokens there are in filtered text
print ("How many tokens are in the Filtered Text:" , len(filtered_words))
#calculate a measure of the lexical richness of the text
from __future__ import division
In [12]:
def lexical_diversity(text):
return len(set(text)) / len(text)
def percentage(count, total):
return 100 * count / total
print ("This text has the following lexical diversity:" , lexical_diversity(mytext), "and Trump consitutes ", percentage(mytext.count("trump"), len(mytext)) , "% of the text")
#frequency distributions
fdist1 = nltk.FreqDist(filtered_words)
fdist1.most_common(25)
fdist1.plot(25, cumulative=True)
for word, frequency in fdist1.most_common(150):
print(u'{};{}'.format(word, frequency))
fdist1.plot(25, cumulative=False)
In [16]:
##Look at bigrams & collocations
##http://www.nltk.org/howto/collocations.html
bigram_measures = nltk.collocations.BigramAssocMeasures()
trigram_measures = nltk.collocations.TrigramAssocMeasures()
b = BigramCollocationFinder.from_words(mytext2)
b.apply_freq_filter(2)
b.nbest(bigram_measures.pmi, 20)
stemmer = nltk.stem.snowball.SnowballStemmer('english')
EC = pd.DataFrame(filtered_words,columns=["term"])
stems = [stemmer.stem(word) for word in filtered_words]
EC['stem'] = stems
print (EC.groupby(['stem']).agg(['count']))