NLP Using Python
1. 1. HandsOn- Simple Operations with text
a. Problem 1
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]()+"/nltk_data"
from [Link] import gutenberg
from [Link] import Text
#
# Complete the 'calculateWordCounts' function below.
#
#
#
def calculateWordCounts(text):
# Write your code here
n_words = len(text)
print(n_words)
n_unique_words = len(set(text))
print(n_unique_words)
word_coverage1 = [Link](n_words / n_unique_words)
print(word_coverage1)
if __name__ == '__main__':
text = input()
if not [Link]([Link]()+"/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
text = Text([Link](text))
calculateWordCounts(text)
b. Problem 2
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]() + "/nltk_data"
from [Link] import gutenberg
from [Link] import Text
#
# Complete the 'filterWords' function below.
#
#
#
def filterWords(text):
# Write your code here
unique_words = set(text)
ing_words = [word for word in list(unique_words) if [Link]('ing')]
big_words = [word for word in list(text) if len(word) > 15 ]
upper_words = [word for word in list(unique_words) if [Link]()]
return (ing_words, big_words, upper_words)
if __name__ == '__main__':
text = input()
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
text = Text([Link](text))
ing_words, big_words, upper_words = filterWords(text)
print(sorted(ing_words))
print(sorted(big_words))
print(sorted(upper_words))
c. Problem 3
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]() + "/nltk_data"
from [Link] import gutenberg
from [Link] import Text
import nltk
#
# Complete the 'findWordFreq' function below.
#
#
def findWordFreq(text, word):
# Write your code here
wordfreq = 0
for w in text:
if(w == word):
wordfreq = wordfreq+1
textfreq = [Link](word for word in list(text) if [Link]())
mostfreq = [Link]()
return(wordfreq, mostfreq)
if __name__ == '__main__':
text = input()
word = input()
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
text = Text([Link](text))
word_freq, max_freq = findWordFreq(text, word)
print(word_freq)
print(max_freq)
2. Hands-On-Accessing Text Corpora
a. Problem 1
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]() + "/nltk_data"
import nltk
from [Link] import inaugural
#
# Complete the 'accessTextCorpora' function below.
#
# The function accepts following parameters:
# 1. STRING fileid
# 2. STRING word
#
def accessTextCorpora(fileid, word):
# Write your code here
file_words = [Link](fileid)
wordcoverage = int(len(file_words)/len(set(file_words)))
ed_words = [words for words in set(file_words) if [Link]
dswith('ed')]
textfreq2 = [[Link]() for word in file_words if word.i
salpha()]
textfreq = [Link](textfreq2)
wordfreq = textfreq[word]
return wordcoverage, ed_words, wordfreq
if __name__ == '__main__':
fileid = input()
word = input()
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
word_coverage, ed_words, word_freq = accessTextCorpora(fil
eid, word)
print(word_coverage)
print(sorted(ed_words))
print(word_freq)
b. Problem 2
#!/bin/python3
import math
import os
import random
import re
import sys
import nltk
from [Link] import PlaintextCorpusReader
#
# Complete the 'createUserTextCorpora' function below.
#
# The function accepts following parameters:
# 1. STRING filecontent1
# 2. STRING filecontent2
#
def createUserTextCorpora(filecontent1, filecontent2):
# Write your code here
with open([Link]('nltk_data/','[Link]'),"w") a
s file1:
[Link](filecontent1)
[Link]()
with open([Link]('nltk_data/','[Link]'),"w") a
s file2:
[Link](filecontent2)
[Link]()
text_corpus = PlaintextCorpusReader('nltk_data/','.*')
no_of_words_corpus1 = len(text_corpus.words('[Link]'
))
no_of_words_corpus2 = len(text_corpus.words('[Link]'
))
no_of_unique_words_corpus1 = len(set(text_corpus.words('co
[Link]')))
no_of_unique_words_corpus2 = len(set(text_corpus.words('co
[Link]')))
return text_corpus,no_of_words_corpus1,no_of_unique_words_
corpus1,no_of_words_corpus2,no_of_unique_words_corpus2
if __name__ == '__main__':
filecontent1 = input()
filecontent2 = input()
path = [Link]([Link](), "nltk_data")
[Link](path, exist_ok=True)
for file in [Link](path):
[Link](path+"\\"+file)
text_corpus, no_of_words_corpus1, no_of_unique_words_corpu
s1, no_of_words_corpus2, no_of_unique_words_corpus2 = createUs
erTextCorpora(filecontent1, filecontent2)
expected_corpus_files = ['[Link]', '[Link]']
if type(text_corpus) == [Link]
textCorpusReader and sorted(list(text_corpus.fileids())) == ex
pected_corpus_files:
print(no_of_words_corpus1)
print(no_of_unique_words_corpus1)
print(no_of_words_corpus2)
print(no_of_unique_words_corpus2)
3. Handson-Conditional Frequency distribution
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]()+"/nltk_data"
import nltk
from [Link] import brown,stopwords
#
# Complete the 'calculateCFD' function below.
#
# The function accepts following parameters:
# 1. STRING_ARRAY cfdconditions
# 2. STRING_ARRAY cfdevents
#
def calculateCFD(cfdconditions, cfdevents):
# Write your code here
stopword = set([Link]('english'))
cdev_cfd = [Link]([(genre, [Link]()) for genr
e in [Link]() for word in [Link](categories=genre) if no
t [Link]() in stopword])
cdev_cfd.tabulate(conditions = cfdconditions, samples = cfdevents)
inged_cfd = [ (genre, [Link]()) for genre in [Link]()
for word in [Link](categories=genre) if ([Link]().endswith('i
ng') or [Link]().endswith('ed')) ]
inged_cfd = [list(x) for x in inged_cfd]
for wd in inged_cfd:
if wd[1].endswith('ing') and wd[1] not in stopword:
wd[1] = 'ing'
elif wd[1].endswith('ed') and wd[1] not in stopword:
wd[1] = 'ed'
#print(inged_cfd)
inged_cfd = [Link](inged_cfd)
#print(inged_cfd.conditions())
inged_cfd.tabulate(conditions=cfdconditions, samples = ['ed','ing']
)
if __name__ == '__main__':
cfdconditions_count = int(input().strip())
cfdconditions = []
for _ in range(cfdconditions_count):
cfdconditions_item = input()
[Link](cfdconditions_item)
cfdevents_count = int(input().strip())
cfdevents = []
for _ in range(cfdevents_count):
cfdevents_item = input()
[Link](cfdevents_item)
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
calculateCFD(cfdconditions, cfdevents)
4. Hands-On Processing Raw text
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]() + "/nltk_data"
import nltk
import urllib
import requests
#
# Complete the 'processRawText' function below.
#
# The function accepts STRING textURL as parameter.
#
def processRawText(textURL):
# Write your code here
response = [Link](textURL)
textcontent = [Link]
tokenizedword = [Link].word_tokenize(textcontent)
#Step 2
tokenizedlcwords = [[Link]() for x in list(tokenizedword) if x !=
'']
noofwords = len(tokenizedlcwords)
noofunqwords = len(set(tokenizedlcwords))
wordcov = [Link]((noofwords)/(noofunqwords))
workfreq = [Link]([word for word in (tokenizedlcwords) if wo
[Link]()])
maxfreq = [Link]()
return (noofwords, noofunqwords, wordcov, maxfreq)
if __name__ == '__main__':
textURL = input()
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
noofwords, noofunqwords, wordcov, maxfreq = processRawText(textURL)
print(noofwords)
print(noofunqwords)
print(wordcov)
print(maxfreq)
5. Bigrams and ngrams
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]() + "/nltk_data"
import nltk
from [Link] import stopwords
from nltk import ConditionalFreqDist
#
# Complete the 'performBigramsAndCollocations' function below.
#
# The function accepts following parameters:
# 1. STRING textcontent
# 2. STRING word
#
def performBigramsAndCollocations(textcontent, word):
# Write your code here
tokenizedword = nltk.regexp_tokenize(textcontent, pattern = r'\w
*', gaps = False)
tokenizedwords = [[Link]() for x in tokenizedword if x != '']
tokenizedwordsbigrams=[Link](tokenizedwords)
stop_words= [Link]('english')
tokenizednonstopwordsbigrams=[(w1,w2) for w1 , w2 in tokenizedwo
rdsbigrams if (w1 not in stop_words and w2 not in stop_words)]
cfd_bigrams=[Link](tokenizednonstopwordsbigram
s)
mostfrequentwordafter=cfd_bigrams[word].most_common(3)
tokenizedwords = [Link](tokenizedwords)
collocationwords = tokenizedwords.collocation_list()
return mostfrequentwordafter ,collocationwords
if __name__ == '__main__':
textcontent = input()
word = input()
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
mostfrequentwordafter, collocationwords = performBigramsAndCollo
cations(textcontent, word)
print(sorted(mostfrequentwordafter, key=lambda element: (element
[1], element[0]), reverse=True))
print(sorted(collocationwords))
6. Stemming and Lemmatization
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]()+"/nltk_data"
import nltk
from [Link] import stopwords
#
# Complete the 'performStemAndLemma' function below.
#
# The function accepts STRING textcontent as parameter.
#
def performStemAndLemma(textcontent):
# Write your code here
#Step 1
tokenizedword = nltk.regexp_tokenize(textcontent, pattern = r'\w
*', gaps = False)
#Step 2
tokenizedwords = [y for y in tokenizedword if y != '']
unique_tokenizedwords = set(tokenizedwords)
tokenizedwords = [[Link]() for x in unique_tokenizedwords if x
!= '']
#Step 3
#unique_tokenizedwords = set(tokenizedwords)
stop_words = set([Link]('english'))
filteredwords = []
for x in tokenizedwords:
if x not in stop_words:
[Link](x)
#Steps 4, 5 , 6
ps = [Link]()
ls = [Link]()
wnl = [Link]()
porterstemmedwords =[]
lancasterstemmedwords = []
lemmatizedwords = []
for x in filteredwords:
[Link]([Link](x))
[Link]([Link](x))
[Link]([Link](x))
return porterstemmedwords, lancasterstemmedwords, lemmatizedword
s
if __name__ == '__main__':
textcontent = input()
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
porterstemmedwords, lancasterstemmedwords, lemmatizedwords = per
formStemAndLemma(textcontent)
print(sorted(porterstemmedwords))
print(sorted(lancasterstemmedwords))
print(sorted(lemmatizedwords))
7. POS Tagging
#!/bin/python3
import math
import os
import random
import re
import sys
import zipfile
[Link]['NLTK_DATA'] = [Link]() + "/nltk_data"
from [Link] import brown
import nltk
#
# Complete the 'tagPOS' function below.
#
# The function accepts following parameters:
# 1. STRING textcontent
# 2. STRING taggedtextcontent
#
def tagPOS(textcontent, taggedtextcontent, defined_tags):
# Write your code here
text = nltk.word_tokenize(textcontent)
nltk_pos_tags = nltk.pos_tag(text)
#text2 = nltk.word_tokenize(taggedtextcontent)
tagged_pos_tag = [[Link].str2tuple(word) for word in [Link]()]
tagger = [Link](model=defined_tags)
unigram_pos_tag = [Link](text)
return nltk_pos_tags, tagged_pos_tag, unigram_pos_tag
if __name__ == '__main__':
textcontent = input()
taggedtextcontent = input()
if not [Link]([Link]() + "/nltk_data"):
with [Link]("nltk_data.zip", 'r') as zip_ref:
zip_ref.extractall([Link]())
defined_tags = dict(brown.tagged_words(tagset='universal'))
nltk_pos_tags, tagged_pos_tag, unigram_pos_tag = tagPOS(textcontent, taggedtextcont
ent, defined_tags)
print(nltk_pos_tags)
print(tagged_pos_tag)
print(unigram_pos_tag)