1.
Python program to perform Tokenization and Stop word Removal
import nltk
# Download required NLTK data files
[Link]('punkt')
[Link]('punkt_tab')
[Link]('stopwords')
from [Link] import word_tokenize
from [Link] import stopwords
import string
def process_text(text):
"""
Performs tokenization and stop word removal on input text.
"""
# Convert text to lowercase
text = [Link]()
# Tokenize text
tokens = word_tokenize(text)
# Load English stop words
stop_words = set([Link]('english'))
# Remove stop words and punctuation
filtered_tokens = [
word for word in tokens
if word not in stop_words and word not in [Link]
return filtered_tokens
sample_text = "This is a sample sentence, showing how to use NLTK for tokenization and stop
word removal effectively."
result = process_text(sample_text)
print("Original Text:")
print(sample_text)
print("\nTokens after stop word removal:")
print(result)
Output:
Original Text:
This is a sample sentence, showing how to use NLTK for tokenization and stop word
removal effectively.
Tokens after stop word removal:
['sample', 'sentence', 'showing', 'use', 'nltk', 'tokenization', 'stop', 'word', 'removal',
'effectively']
2. Write a python program to implement Porter stemmer Algorithm for stemming.
import nltk
[Link]('punkt')
from [Link] import PorterStemmer
from [Link] import word_tokenize
def porter_stemming(text):
"""
Applies Porter Stemmer to the given text.
"""
# Create a PorterStemmer object
stemmer = PorterStemmer()
# Tokenize the input text
words = word_tokenize(text)
# Apply stemming to each word
stemmed_words = [[Link](word) for word in words]
return stemmed_words
sample_text = "The students were studying studies and studied harder"
result = porter_stemming(sample_text)
print("Original Text:")
print(sample_text)
print("\nStemmed Words:")
print(result)
Output:
['the', 'student', 'were', 'studi', 'studi', 'and', 'studi', 'harder']
3. Write a python program for Word Analysis and Word Generation
import nltk
from [Link] import word_tokenize
from [Link] import ngrams
from collections import Counter
[Link]('punkt')
[Link]('punkt_tab')
def word_analysis(text):
# Convert text to lowercase
text = [Link]()
# Tokenize text
tokens = word_tokenize(text)
# Count word frequency
word_freq = Counter(tokens)
return tokens, word_freq
def word_generation(text):
# Tokenize text
tokens = word_tokenize([Link]())
# Generate bigrams
bigrams = list(ngrams(tokens, 2))
return bigrams
text = "Natural language processing enables machines to understand language"
tokens, frequencies = word_analysis(text)
bigrams = word_generation(text)
print("Tokens:")
print(tokens)
print("\nWord Frequencies:")
print(frequencies)
print("\nGenerated Bigrams:")
print(bigrams)
Output:
Tokens:
['natural', 'language', 'processing', 'enables', 'machines', 'to', 'understand', 'language']
Word Frequencies:
Counter({'language': 2, 'natural': 1, 'processing': 1, 'enables': 1, 'machines': 1, 'to': 1,
'understand': 1})
Generated Bigrams:
[('natural', 'language'), ('language', 'processing'), ('processing', 'enables'),
('enables', 'machines'), ('machines', 'to'), ('to', 'understand'), ('understand', 'language')]
4. Create a sample list for at least 5 words with ambiguous sense and write a python
program to implement WSD.
import nltk
from [Link] import lesk
from [Link] import word_tokenize
from [Link] import wordnet
[Link]('punkt')
[Link]('wordnet')
[Link]('omw-1.4')
def wsd_lesk(sentence, ambiguous_word):
# Tokenize the sentence
words = word_tokenize(sentence)
# Apply Lesk algorithm
sense = lesk(words, ambiguous_word)
return sense
sentence1 = "He deposited money in the bank"
sentence2 = "The children played with the bat"
sentence3 = "The crane is used at the construction site"
print("Sentence:", sentence1)
print("Sense:", wsd_lesk(sentence1, "bank"))
print("Definition:", wsd_lesk(sentence1, "bank").definition())
print("\nSentence:", sentence2)
print("Sense:", wsd_lesk(sentence2, "bat"))
print("Definition:", wsd_lesk(sentence2, "bat").definition())
print("\nSentence:", sentence3)
print("Sense:", wsd_lesk(sentence3, "crane"))
print("Definition:", wsd_lesk(sentence3, "crane").definition())
Output:
Sentence: He deposited money in the bank
Sense: Synset('depository_financial_institution.n.01')
Definition: a financial institution that accepts deposits and channels the money into lending
activities
Sentence: The children played with the bat
Sense: Synset('bat.n.01')
Definition: nocturnal mouselike mammal with forelimbs modified to form membranous
wings and anatomical adaptations for echolocation
Sentence: The crane is used at the construction site
Sense: Synset('crane.n.04')
Definition: lifts and moves heavy objects; lifting tackle is suspended from a pivoted boom
that rotates around a vertical axis
5. Install nltk tool kit and perform stemming.
!pip install nltk
import nltk
[Link]('punkt')
import nltk
from [Link] import word_tokenize
from [Link] import PorterStemmer, LancasterStemmer, SnowballStemmer
# Sample text
text = "The boys are playing, running and studying happily"
# Tokenize the text
words = word_tokenize(text)
# Initialize stemmers
porter = PorterStemmer()
lancaster = LancasterStemmer()
snowball = SnowballStemmer("english")
print("Original Words:", words)
print("\nPorter Stemming:")
for word in words:
print(word, "->", [Link](word))
print("\nLancaster Stemming:")
for word in words:
print(word, "->", [Link](word))
print("\nSnowball Stemming:")
for word in words:
print(word, "->", [Link](word))
Output:
Original Words:
['The', 'students', 'are', 'studying', 'and', 'studied', 'their', 'studies']
Stemmed Words:
['the', 'student', 'are', 'studi', 'and', 'studi', 'their', 'studi']
6. Create Sample list of at least 10 words POS tagging and find the POS for any given
word
import nltk
# Download required resources
[Link]('punkt')
[Link]('averaged_perceptron_tagger')
[Link]('averaged_perceptron_tagger_eng')
# Sample list of at least 10 words
words = [
"run", "beautiful", "quickly", "computer", "she",
"is", "happy", "write", "book", "and", "play"
]
# Perform POS tagging
pos_tags = nltk.pos_tag(words)
print("POS Tagging for Sample Words:\n")
for word, tag in pos_tags:
print(word, "->", tag)
def find_pos(word):
tagged = nltk.pos_tag([word])
return tagged[0][1]
# User input
user_word = input("Enter a word to find its POS: ")
pos = find_pos(user_word)
print("POS tag of", user_word, "is:", pos)
Output:
POS Tagging for Sample Words:
run -> VB
beautiful -> JJ
quickly -> RB
computer -> NN
she -> PRP
is -> VBZ
happy -> JJ
write -> VB
book -> NN
and -> CC
play -> VB
Enter a word to find its POS: quickly
POS tag of quickly is:
7: Write a Python program to
a) Perform Morphological Analysis using NLTK library
b) Generate n-grams using NLTK N-Grams library
c) Implement N-Grams Smoothing
a)Perform Morphological Analysis using NLTK library
import nltk
from [Link] import PorterStemmer
[Link]('punkt')
stemmer = PorterStemmer()
words = ["playing", "plays", "played", "player", "happiness", "running"]
print("Morphological Analysis using Stemming:\n")
for word in words:
stem_word = [Link](word)
print(word, "->", stem_word)
Ouput:
Morphological Analysis using Stemming:
playing -> play
plays -> play
played -> play
player -> player
happiness -> happi
running -> run
b) Generate n-grams using NLTK N-Grams library
# Import required libraries
import nltk
from [Link] import ngrams
from [Link] import word_tokenize
# Download tokenizer model (run once)
[Link]('punkt')
# Input sentence
text = "Natural Language Processing is an important field of Artificial Intelligence"
# Tokenize the sentence into words
tokens = word_tokenize(text)
print("Tokens:", tokens)
# Generate Unigrams
unigrams = list(ngrams(tokens, 1))
print("\nUnigrams:")
print(unigrams)
# Generate Bigrams
bigrams = list(ngrams(tokens, 2))
print("\nBigrams:")
print(bigrams)
# Generate Trigrams
trigrams = list(ngrams(tokens, 3))
print("\nTrigrams:")
print(trigrams)
Output:
Tokens: ['Natural', 'Language', 'Processing', 'is', 'an', 'important', 'field', 'of', 'Artificial',
'Intelligence']
Unigrams:
[('Natural',), ('Language',), ('Processing',), ('is',), ('an',), ('important',), ('field',), ('of',),
('Artificial',), ('Intelligence',)]
Bigrams:
[('Natural', 'Language'),
('Language', 'Processing'),
('Processing', 'is'),
('is', 'an'),
('an', 'important'),
('important', 'field'),
('field', 'of'),
('of', 'Artificial'),
('Artificial', 'Intelligence')]
Trigrams:
[('Natural', 'Language', 'Processing'),
('Language', 'Processing', 'is'),
('Processing', 'is', 'an'),
('is', 'an', 'important'),
('an', 'important', 'field'),
('important', 'field', 'of'),
('field', 'of', 'Artificial'),
('of', 'Artificial', 'Intelligence')]
c) Implement N-Grams Smoothing
import nltk
from [Link] import ngrams
from [Link] import word_tokenize
from collections import Counter
# Download tokenizer
[Link]('punkt')
# Training text
text = "Natural language processing is a part of artificial intelligence and language processing is
useful"
# Tokenize the text
tokens = word_tokenize([Link]())
# Generate Bigrams
bigrams = list(ngrams(tokens, 2))
# Count frequencies
bigram_counts = Counter(bigrams)
unigram_counts = Counter(tokens)
# Vocabulary size
V = len(set(tokens))
print("Unigram Counts:", unigram_counts)
print("\nBigram Counts:", bigram_counts)
# Function for Add-One (Laplace) smoothing
def laplace_smoothing(w1, w2):
bigram_count = bigram_counts[(w1, w2)]
unigram_count = unigram_counts[w1]
probability = (bigram_count + 1) / (unigram_count + V)
return probability
# Example probability calculation
print("\nProbability with Laplace Smoothing:")
print("P(language | natural) =", laplace_smoothing("natural", "language"))
print("P(processing | language) =", laplace_smoothing("language", "processing"))
print("P(ai | natural) =", laplace_smoothing("natural", "ai"))
Output:
unigram Counts: Counter({'language': 2, 'processing': 2, 'is': 2,
'natural': 1, 'a': 1, 'part': 1, 'of': 1, 'artificial': 1,
'intelligence': 1, 'and': 1, 'useful': 1})
Bigram Counts: Counter({('natural', 'language'): 1,
('language', 'processing'): 1,
('processing', 'is'): 1,
('is', 'a'): 1,
('a', 'part'): 1,
('part', 'of'): 1,
('of', 'artificial'): 1,
('artificial', 'intelligence'): 1,
('intelligence', 'and'): 1,
('and', 'language'): 1,
('language', 'processing'): 1,
('processing', 'is'): 1,
('is', 'useful'): 1})
Probability with Laplace Smoothing:
P(language | natural) = 0.16666666666666666
P(processing | language) = 0.16666666666666666
P(ai | natural) = 0.08333333333333333
8. Use NLTK package to convert audio files to text and text file to audio files.
import pyttsx3
# Initialize the engine
engine = [Link]()
# Read text from file
with open("[Link]", "r") as file:
text = [Link]()
# Convert text to speech
[Link](text)
# Save audio file
engine.save_to_file(text, "speech_output.mp3")
# Run the engine
[Link]()
print("Text converted to audio successfully")
Output:
The audio file will speak:
“Natural Language Processing is a branch of Artificial Intelligence.”