Week 11 & 12
77. Write a program to explore the contents and metadata of a Gutenberg Corpus.
import nltk
from [Link] import gutenberg
# [Link]('gutenberg')
# [Link]('punkt')
print("Files in Gutenberg:", [Link]())
for fid in [Link]():
words = [Link](fid)
sents = [Link](fid)
raw_len = len(words)
sent_len = len(sents)
vocab_size = len(set([Link]() for w in words))
avg_word_len = sum(len(w) for w in words if [Link]()) / max(1, sum(1 for w in words if
[Link]()))
avg_sent_len = sum(len(s) for s in sents) / max(1, sent_len)
print(f"\n--- {fid} ---")
print(f"Tokens: {raw_len} | Sentences: {sent_len} | Vocab: {vocab_size}")
print(f"Avg word length: {avg_word_len:.2f} | Avg sentence length (tokens):
{avg_sent_len:.2f}")
fid = '[Link]'
print("\nSample sentences from:", fid)
for s in [Link](fid)[:2]:
print(" ".join(s))
Output :
Files in Gutenberg: ['[Link]', '[Link]', '[Link]',
'[Link]', '[Link]', '[Link]', '[Link]', 'carroll-
[Link]', '[Link]', '[Link]', '[Link]',
'[Link]', 'melville-moby_dick.txt', '[Link]', 'shakespeare-
[Link]', '[Link]', '[Link]', '[Link]']
--- [Link] ---
Tokens: 192427 | Sentences: 7752 | Vocab: 7344
Avg word length: 4.23 | Avg sentence length (tokens): 24.83
--- [Link] ---
Tokens: 98171 | Sentences: 3747 | Vocab: 5835
Avg word length: 4.34 | Avg sentence length (tokens): 26.20
--- [Link] ---
Tokens: 141576 | Sentences: 4999 | Vocab: 6403
Avg word length: 4.35 | Avg sentence length (tokens): 28.33
…………………………..
……………………………..
--- [Link] ---
Tokens: 37360 | Sentences: 3106 | Vocab: 4716
Avg word length: 4.04 | Avg sentence length (tokens): 12.03
--- [Link] ---
Tokens: 23140 | Sentences: 1907 | Vocab: 3464
Avg word length: 4.12 | Avg sentence length (tokens): 12.13
--- [Link] ---
Tokens: 154883 | Sentences: 4250 | Vocab: 12452
Avg word length: 4.29 | Avg sentence length (tokens): 36.44
Sample sentences from: [Link]
[ Emma by Jane Austen 1816 ]
VOLUME I
78. Write a program to find the most frequent words in a text corpus.
import nltk
from [Link] import brown
from collections import Counter
import re
# [Link]('brown')
tokens = [[Link]() for w in [Link](categories='news')]
tokens = [[Link](r"[^a-z]", "", w) for w in tokens]
tokens = [w for w in tokens if w]
freqs = Counter(tokens).most_common(30)
print("Top 30 words in Brown 'news':")
for w, c in freqs:
print(f"{w:15} {c}")
Output :
Top 30 words in Brown 'news':
the 6386
of 2861
and 2187
……………..
……………..
………………
a 2170
they 267
79. Write a program to compute TF-IDF scores of words in corpus documents.
from sklearn.feature_extraction.text import TfidfVectorizer
import nltk
from [Link] import gutenberg
# [Link]('gutenberg')
file_ids = ['[Link]', '[Link]', '[Link]']
docs = [[Link](fid) for fid in file_ids]
# set min_df <= number of documents
vectorizer = TfidfVectorizer(min_df=1, stop_words='english')
X = vectorizer.fit_transform(docs)
terms = vectorizer.get_feature_names_out()
def top_terms(doc_index, k=15):
row = [Link](doc_index).toarray().ravel()
top_idx = [Link]()[::-1][:k]
return [(terms[i], row[i]) for i in top_idx]
for i, fid in enumerate(file_ids):
print(f"\nTop TF-IDF terms for {fid}:")
for term, score in top_terms(i, 10):
print(f"{term:20} {score:.4f}")
Output :
Top TF-IDF terms for [Link]:
mr 0.3575
emma 0.3454
harriet 0.2656
weston 0.2310
mrs 0.2167
knightley 0.2042
80. Write a program to explore lexical semantics using WordNet.
import nltk
from [Link] import wordnet as wn
# [Link]('wordnet')
# [Link]('omw-1.4')
word = "bank"
synsets = [Link](word)
print(f"Synsets for '{word}':")
for s in synsets:
print(f"- {[Link]()} :: {[Link]()}")
if synsets:
s = synsets[0]
print("\nHypernyms:", [[Link]() for h in [Link]()])
print("Hyponyms (first 10):", [[Link]() for h in [Link]()[:10]])
print("Lemmas:", [[Link]() for l in [Link]()])
print("Examples:", [Link]())
dog = [Link]('dog.n.01')
cat = [Link]('cat.n.01')
print("\nSimilarity dog vs cat (path):", dog.path_similarity(cat))
Output :
Synsets for 'bank':
- bank.n.01 :: sloping land (especially the slope beside a body of water)
- depository_financial_institution.n.01 :: a financial institution that accepts deposits and
channels the money into lending activities
- bank.n.03 :: a long ridge or pile
- bank.n.04 :: an arrangement of similar objects in a row or in tiers
- bank.n.05 :: a supply or stock held in reserve for future use (especially in emergencies)
- bank.n.06 :: the funds held by a gambling house or the dealer in some gambling games
………………
……………………
81. Write a program to implement Sentiment Analysis using TextBlob.
from textblob import TextBlob
sentences = [
"I absolutely love this movie. The acting was brilliant!",
"It was okay, not great but not terrible either.",
"This is the worst product I have ever bought."
]
for s in sentences:
blob = TextBlob(s)
print(f"Text: {s}\n ->
Polarity={[Link]:.3f},Subjectivity={[Link]:.3f}\n")
Output :
Text: I absolutely love this movie. The acting was brilliant!
-> Polarity=0.500, Subjectivity=0.533
Text: It was okay, not great but not terrible either.
-> Polarity=0.200, Subjectivity=0.750
Text: This is the worst product I have ever bought.
-> Polarity=-1.000, Subjectivity=1.000
Week 12
82. Write a program to convert text to lowercase and remove punctuation.
import string
text = "Hello, World! NLP in 2025: Robust—yet simple."
lower = [Link]()
punct_table = [Link]('', '', [Link] + "—“”‘’")
clean = [Link](punct_table)
print("Original:", text)
print("Lowercase:", lower)
print("No punctuation:", clean)
print("Tokens:", [Link]())
Output :
Original: Hello, World! NLP in 2025: Robust—yet simple.
Lowercase: hello, world! nlp in 2025: robust—yet simple.
No punctuation: hello world nlp in 2025 robustyet simple
Tokens: ['hello', 'world', 'nlp', 'in', '2025', 'robustyet', 'simple']
83. Write a program to remove common stopwords from a sentence.
import nltk
from [Link] import stopwords
import re
# [Link]('stopwords')
sentence = "This is a simple example to demonstrate removal of common English stopwords."
tokens = [Link](r"\b\w+\b", [Link]())
stops = set([Link]('english'))
filtered = [w for w in tokens if w not in stops]
print("Original:", sentence)
print("Filtered:", " ".join(filtered))
Output :
Original: This is a simple example to demonstrate removal of common English stopwords.
Filtered: simple example demonstrate removal common english stopwords
84. Write a program to reduce words to their base (root) form using a stemmer.
import re
from [Link] import PorterStemmer, SnowballStemmer
text = "running runner runs easily fairly cared cars studies studying"
tokens = [Link](r"\b\w+\b", [Link]())
porter = PorterStemmer()
snow = SnowballStemmer("english")
print("Token Porter Snowball")
for t in tokens:
print(f"{t:8} {[Link](t):7} {[Link](t):8}")
Output :
Token Porter Snowball
running run run
runner runner runner
runs run run
easily easili easili
fairly fairli fair
cared care care
cars car car
studies studi studi
studying studi studi
85. Write a program to use WordNet lemmatizer to convert words to dictionary form.
import nltk
from [Link] import WordNetLemmatizer
from nltk import pos_tag
from [Link] import wordnet
import re
# [Link]('punkt'); [Link]('averaged_perceptron_tagger')
# [Link]('wordnet'); [Link]('omw-1.4')
def to_wn_pos(treebank_tag):
if treebank_tag.startswith('J'): return [Link]
if treebank_tag.startswith('V'): return [Link]
if treebank_tag.startswith('N'): return [Link]
if treebank_tag.startswith('R'): return [Link]
return [Link]
text = "The striped bats are hanging on their feet and ate best fishes"
tokens = [Link](r"\b\w+\b", text)
tags = pos_tag(tokens)
lemmatizer = WordNetLemmatizer()
lemmas = [[Link](w, to_wn_pos(t)) for w, t in tags]
print("Original:", text)
print("Lemmas :", " ".join(lemmas))
Output :
Original: The striped bats are hanging on their feet and ate best fishes
Lemmas : The striped bat be hang on their foot and ate best fish
86. Write a program to implement Normalize text end-to-end.
import re
import unicodedata
import nltk
from [Link] import stopwords, wordnet
from nltk import pos_tag
from [Link] import WordNetLemmatizer
# [Link]('stopwords'); [Link]('punkt')
# [Link]('averaged_perceptron_tagger'); [Link]('wordnet'); [Link]('omw-1.4')
def to_wn_pos(tag):
if [Link]('J'): return [Link]
if [Link]('V'): return [Link]
if [Link]('N'): return [Link]
if [Link]('R'): return [Link]
return [Link]
def normalize(text):
text = [Link]("NFKC", text)
text = [Link]()
text = [Link](r"[^\w\s']+", " ", text)
tokens = [Link](r"\b\w+(?:'\w+)?\b", text)
stops = set([Link]('english'))
tokens = [t for t in tokens if t not in stops and len(t) > 2]
tags = pos_tag(tokens)
lemmatizer = WordNetLemmatizer()
lemmas = [[Link](w, to_wn_pos(p)) for w, p in tags]
return lemmas
sample = "Caf\u00e9s were AMAZING—yet pricey! We're testing normalisation vs. normalization..."
print("Input:", sample)
print("Normalized tokens:", normalize(sample))
Output :
Input: Cafés were AMAZING—yet pricey! We're testing normalisation vs. normalization...
Normalized tokens: ['cafés', 'amaze', 'yet', 'pricey', 'test', 'normalisation', 'normalization']
Assessment:POS Tag Frequency Counter (Students only must write this program)
Objective: Count how many times each POS tag appears in a given text.
Task: After POS tagging a paragraph, output the frequency of each tag.