COIMBATORE INSTITUTE OF TECHNOLOGY

(Government Aided Autonomous Affiliated to Anna University)

COIMBATORE – 641014, TAMIL NADU, INDIA

DEPARTMENT OF COMPUTING
[Link]. ARTIFICIAL INTELLIGENCE AND MACHINE LEARNING

19MAMEL03 – NATURAL LANGUAGE PROCESSING

NAME : SWETHA KRITHIKA M

REG NO : 71762134052
BATCH : 2021 - 2026
SEMESTER : VIII
 INDEX
[Link] EXERCISE PG. NO
1 NLTK Download and Function 2

2 Stemming, Lemmatization and Stop words 3

3 Word tokenizer and analysis 5

4 Regular Expression NLTK 7

5 Text Corpus 9

6 Tokenizing Text 12

7 Patterns and substitution 14

8 Analyse sentence 16

9 Bigram and Trigram 20

10 Probability N-gram 21

11 Comparision analysis of N-grams 23

12 Audio to text converter 26

13 Language detection with langid 27

14 RNN language translation 29

15 Naïve Bayes Classifier 31

16 Skipgram 32

17 GLOVE Embedding 36

18 LSTM 39

19 LSTM and GRU 43

20 CBOW 47

21 Tagging methods 50

22 Viterbi Algorithm 52

23 CRF Implementation 55

24 French to English translation model from scratch 59

EX:1
NLTK Download and functions
AIM:
To explore functions in NLTK packages.
CODE:
import nltk

[Link]()
dir(nltk)
print([Link]("playing","v"))
print([Link]("communication","v"))
from nltk import pos_tag
from nltk import word_tokenize

text = "GeeksforGeeks is a Computer Science platform."

tokenized_text = word_tokenize(text)
tags = tokens_tag = pos_tag(tokenized_text)
tags

RESULT:
EX:2
Stemming. Lemmatization and Stop words
AIM:
To perform text processing tasks on given document containing at least 200 words.
CODE AND OUTPUT:
EX:3
Word tokenize and analysis
AIM:
To implement program using word tokenize
CODE:
import nltk
from [Link] import gutenberg
from [Link] import word_tokenize, sent_tokenize
from [Link] import FreqDist
import [Link] as plt

# Download necessary NLTK data

[Link]('gutenberg')
[Link]('punkt')

# Load a small corpus (we'll use a part of a text from the Gutenberg corpus)
text = [Link]('[Link]') # Load "Emma" by Jane Austen

# Tokenize the text into words and sentences

word_tokens = word_tokenize(text)
sent_tokens = sent_tokenize(text)

# Find the total number of tokens and sentences

total_tokens = len(word_tokens)
total_sentences = len(sent_tokens)

print(f"Total number of tokens: {total_tokens}")

print(f"Total number of sentences: {total_sentences}")

# Frequency Distribution of words

fdist = FreqDist(word_tokens)

# (i) Display frequency of all words

print("Frequency of all words:")
for word, frequency in [Link]():
print(f"{word}: {frequency}")

# (ii) Find and display frequency of a specific word

specific_word = 'Emma' # Example word to check
specific_word_frequency = fdist[specific_word]
print(f"\nFrequency of the word '{specific_word}': {specific_word_frequency}")

# Display the top 5 words with the highest frequency

top_5_words = fdist.most_common(5)
print("\nTop 5 words with highest frequency:")
for word, frequency in top_5_words:
print(f"{word}: {frequency}")

# Create a dispersion plot for the word 'Emma' and other words
search_words = ['Emma', 'Mr.', 'Miss', 'Harriet', 'Knightley']
# Get the indices of the words in the tokenized text for plotting
word_indices = [(word, idx) for idx, word in enumerate(word_tokens) if word in
search_words]

# Prepare data for the plot

[Link](figsize=(10, 6))
for word, idx in word_indices:
[Link](idx, search_words.index(word), label=word if word not in
[Link]().get_legend_handles_labels()[1] else "")

[Link](range(len(search_words)), search_words)
[Link]("Lexical Dispersion Plot")
[Link]("Word Offset")
[Link]("Words")
[Link]()

OUTPUT:
EX: 4
Regular expression in NLTK
AIM:
To implement the following in Regex from a sample corpus in NLTK
CODE:
import re
import nltk
from [Link] import gutenberg
from [Link] import sent_tokenize

# Download necessary NLTK data

[Link]('gutenberg')
[Link]('punkt')

# Load a sample corpus from NLTK (we'll use Jane Austen's "Emma" for
demonstration)
text = [Link]('[Link]')

# Tokenize text into sentences

sentences = sent_tokenize(text[:5000]) # Limit to first 5000 characters

# Combine sentences for regex operations

corpus_text = ' '.join(sentences)

# Task 1: Use of Hyphen [2-5] and [b-f]

hyphen_pattern = r'[2-5][b-f]-[b-f][2-5]'
hyphen_matches = [Link](hyphen_pattern, corpus_text)
print("Hyphen [2-5] and [b-f] matches:", hyphen_matches)

# Task 2: Caret symbol (^) for start of line

caret_pattern = r'^The'
caret_matches = [sent for sent in sentences if [Link](caret_pattern, sent)]
print("\nSentences starting with 'The':", caret_matches[:5]) # First 5 matches

# Task 3: ? operator (optional preceding characters)

question_mark_pattern = r'\b\w+ed?\b'
question_mark_matches = [Link](question_mark_pattern, corpus_text)
print("\nWords with optional 'ed':", question_mark_matches[:10])

# Task 4: Kleene * operator (zero or more preceding characters)

kleene_star_pattern = r'\bba*!\b' # For "baa!", "baaa!" etc. in the sheep example
kleene_star_matches = [Link](kleene_star_pattern, "baa! baaa! baaaa! baaaaa!")
print("\nKleene * matches (for 'baa!'):", kleene_star_matches)

# Task 5: Kleene + operator (one or more preceding characters)

kleene_plus_pattern = r'\bba+\b' # Example for "baa" or similar words
kleene_plus_matches = [Link](kleene_plus_pattern, "baa! baaa! baaaa! baaaaa!")
print("\nKleene + matches (for 'baa'): ", kleene_plus_matches)

# Task 6: Dot (.) operator for any character

dot_pattern = r'\b.a.\b' # Words with any character between 'a's
dot_matches = [Link](dot_pattern, corpus_text)
print("\nDot operator matches (any character between letters):", dot_matches[:10])

# Task 7: Pipe symbol (|) for alternatives

pipe_pattern = r'\b(Mr\.|Mrs\.)\b' # Matches "Mr." or "Mrs."
pipe_matches = [Link](pipe_pattern, corpus_text)
print("\nPipe symbol matches for 'Mr.' or 'Mrs.':", pipe_matches)

# Task 8: Word boundary (\b)

word_boundary_pattern = r'\b[Aa]rt\b'
word_boundary_matches = [Link](word_boundary_pattern, corpus_text)
print("\nWord boundary matches for 'Art' or 'art':", word_boundary_matches)

# Task 9: Non-word boundary (\B)

non_word_boundary_pattern = r'\Bing\b'
non_word_boundary_matches = [Link](non_word_boundary_pattern, corpus_text)
print("\nNon-word boundary matches for 'ing':", non_word_boundary_matches)

# Task 10: Regex explanation: /[ˆa-zA-Z][tT]he[ˆa-zA-Z]/

pattern = r'[^a-zA-Z][tT]he[^a-zA-Z]'
pattern_matches = [Link](pattern, corpus_text)
print("\nMatches for /[^a-zA-Z][tT]he[^a-zA-Z]/:", pattern_matches)

# Explanation:
# - [^a-zA-Z]: Non-letter character before 'T' or 't'
# - [tT]he: Match 'The' or 'the'
# - [^a-zA-Z]: Non-letter character after 'e'

OUTPUT:
EX:5
Text corpus
AIM:
To explore on inbuild corpus eg:. Brown data in NLTK

CODE:
import nltk
from [Link] import brown

# Ensure the Brown Corpus is downloaded

[Link]('brown')

# 1. List all sections (categories) in the Brown Corpus

categories = [Link]()
print(f"Categories in the Brown Corpus: {categories}")

# 2. List file IDs in the "news" category

news_files = [Link](categories=['news'])
print(f"File IDs in 'news' category: {news_files}")

# 3. Number of words and sentences in "news" category

news_words = [Link](categories=['news'])
news_sentences = [Link](categories=['news'])
print(f"Number of words in 'news': {len(news_words)}")
print(f"Number of sentences in 'news': {len(news_sentences)}")

# 4. Modal verbs frequency in "news" category

modals = ['can', 'could', 'may', 'might', 'must', 'shall', 'should', 'will',
'would']
modal_freq = {modal: news_words.count(modal) for modal in modals}
print(f"Modal frequencies in 'news': {modal_freq}")

# 5. Count 'wh' words in "news" category

wh_words = ['what', 'why', 'when', 'who', 'whom', 'whose', 'which']
wh_word_freq = {word: news_words.count(word) for word in wh_words}
print(f"'Wh' word frequencies in 'news': {wh_word_freq}")

# 6. Demonstrate other functionalities

# a. fileids(): List all file IDs in the corpus
all_file_ids = [Link]()
print(f"Total number of files: {len(all_file_ids)}")
print(f"Example file IDs: {all_file_ids[:10]}")

# b. fileids([categories]): File IDs for specific categories

news_file_ids = [Link](categories=['news'])
print(f"File IDs in 'news' category: {news_file_ids[:5]} (Total:
{len(news_file_ids)})")

# c. categories(): List all categories in the corpus

all_categories = [Link]()
print(f"Categories in the Brown Corpus: {all_categories}")

# d. categories([fileids]): Categories for specific files

categories_for_files = [Link](fileids=news_file_ids[:2])
print(f"Categories for the first two files in 'news': {categories_for_files}")

# e. raw(): Raw content of the whole corpus

print(f"First 500 characters of the raw content of the entire corpus:\
n{[Link]()[:500]}...")

# f. raw(fileids=[f1, f2, f3]): Raw content for specific files

raw_content_files = [Link](fileids=news_file_ids[:2])
print(f"Raw content for the first two 'news' files:\
n{raw_content_files[:500]}...")

# g. raw(categories=[c1, c2]): Raw content for specific categories

raw_content_categories = [Link](categories=['news', 'hobbies'])
print(f"First 500 characters of raw content for 'news' and 'hobbies':\
n{raw_content_categories[:500]}...")

# h. words(): Words of the whole corpus

all_words = [Link]()
print(f"First 20 words of the entire corpus: {all_words[:20]}")

# i. words(fileids=[f1, f2, f3]): Words for specific files

words_in_files = [Link](fileids=news_file_ids[:2])
print(f"First 20 words in the first two 'news' files: {words_in_files[:20]}")

# j. words(categories=[c1, c2]): Words for specific categories

words_in_categories = [Link](categories=['news', 'hobbies'])
print(f"First 20 words in 'news' and 'hobbies' categories:
{words_in_categories[:20]}")

# k. sents(): Sentences of the whole corpus

all_sentences = [Link]()
print(f"First sentence of the entire corpus: {all_sentences[0]}")

# l. sents(fileids=[f1, f2, f3]): Sentences for specific files

sentences_in_files = [Link](fileids=news_file_ids[:2])
print(f"First sentence in the first two 'news' files: {sentences_in_files[0]}")

# m. sents(categories=[c1, c2]): Sentences for specific categories

sentences_in_categories = [Link](categories=['news', 'hobbies'])
print(f"First sentence in 'news' and 'hobbies': {sentences_in_categories[0]}")

# n. abspath(fileid): Location of a specific file on disk

file_path = [Link](news_file_ids[0])
print(f"Absolute path of the first 'news' file: {file_path}")

# o. encoding(fileid): Encoding of a specific file

file_encoding = [Link](news_file_ids[0])
print(f"Encoding of the first 'news' file: {file_encoding}")
# p. open(fileid): Open a stream for a specific file
with [Link](news_file_ids[0]) as file_stream:
print(f"First 100 characters of the first 'news' file stream:\
n{file_stream.read(100)}")

# q. root(): Path to the root of the corpus

root_path = [Link]
print(f"Root path of the Brown Corpus: {root_path}")

# r. readme(): README file content of the corpus

readme_content = [Link]()
print(f"README content of the Brown Corpus:\n{readme_content}")

OUTPUT:
EX:6
Tokenizing text
AIM:
To implement the following by loading Shakespeare text corpus
CODE:
import nltk
from [Link] import word_tokenize, TreebankWordTokenizer
from [Link] import PorterStemmer
from collections import Counter

# Ensure required NLTK packages are downloaded

[Link]('gutenberg')
[Link]('punkt')

# 1. Load the Shakespeare text corpus

from [Link] import gutenberg

shakespeare_text = [Link]('[Link]')
print(f"First 500 characters of the Shakespeare corpus:\
n{shakespeare_text[:500]}")

# 2. Tokenize using the Penn Treebank tokenizer

treebank_tokenizer = TreebankWordTokenizer()
treebank_tokens = treebank_tokenizer.tokenize(shakespeare_text)
print(f"First 20 tokens (Treebank): {treebank_tokens[:20]}")

# 3. Implement Byte Pair Encoding (BPE) for tokenization

def byte_pair_encoding(tokens, num_merges):
vocab = Counter(tokens)
pairs = get_pairs(vocab)

for _ in range(num_merges):
if not pairs:
break
# Find the most frequent pair
best = max(pairs, key=[Link])
vocab = merge_vocab(best, vocab)
pairs = get_pairs(vocab)

return vocab

def get_pairs(vocab):
"""Find pairs of symbols in the vocabulary."""
pairs = Counter()
for word, freq in [Link]():
symbols = [Link]()
for i in range(len(symbols) - 1):
pairs[symbols[i], symbols[i + 1]] += freq
return pairs
def merge_vocab(pair, vocab):
"""Merge all occurrences of the most frequent pair in the vocabulary."""
new_vocab = {}
bigram = ' '.join(pair)
replacement = ''.join(pair)
for word, freq in [Link]():
new_word = [Link](bigram, replacement)
new_vocab[new_word] = freq
return new_vocab

# Example: Apply BPE to the Shakespeare tokens

# Prepare tokens for BPE
bpe_tokens = [' '.join(list(token)) for token in treebank_tokens]
bpe_vocab = byte_pair_encoding(bpe_tokens, num_merges=10)
print(f"BPE Vocabulary (after 10 merges): {list(bpe_vocab.items())[:10]}")

# 4. Use Porter Stemmer for stemming the words

porter_stemmer = PorterStemmer()
stemmed_words = [porter_stemmer.stem(token) for token in treebank_tokens]
print(f"First 20 stemmed words: {stemmed_words[:20]}")

OUTPUT:
EX:7
AIM:
Implement an ELIZA-like program, using substitutions such as those described below sample

CODE:
import re

class Eliza:
def __init__(self):
[Link] = [
(r"(.*) mother(.*)", "Tell me more about your mother."),
(r"(.*) father(.*)", "How do you feel about your father?"),
(r"I need (.*)", "Why do you need {0}?"),
(r"Why (.*)", "Why do you think {0}?"),
(r"I am feeling (.*)", "Why are you feeling {0}?"),
(r"I am (.*)", "How long have you been {0}?"),
(r"(.*) sorry (.*)", "There’s no need to apologize."),
(r"(.*) friend(.*)", "Tell me more about your friends."),
(r"(.*) you (.*)", "What makes you think I {0}?"),
(r"(.*)", "Can you elaborate on that?")
]

def reflect(self, statement):

"""
Reflects user statements to make responses more engaging.
"""
reflections = {
"am": "are",
"was": "were",
"I": "you",
"I'd": "you would",
"I've": "you have",
"I'll": "you will",
"my": "your",
"you are": "I am",
"you were": "I was",
"you've": "I've",
"you'll": "I'll",
"your": "my",
"yours": "mine",
"me": "you"
}
words = [Link]().split()
reflected = [[Link](word, word) for word in words]
return " ".join(reflected)

def respond(self, user_input):

"""
Generate a response based on user input.
"""
for pattern, response in [Link]:
match = [Link](pattern, user_input, [Link])
 if match:
groups = [Link]()
reflected_groups = [[Link](group) for group in groups]
return [Link](*reflected_groups)
return "I'm not sure I understand. Can you explain that in another way?"

# Run the ELIZA-like program

def run_eliza():
print("ELIZA: Hello, I'm your personal therapist. How can I help you today?")
while True:
user_input = input("You: ")
if user_input.lower() in ["quit", "exit", "bye"]:
print("ELIZA: Goodbye! Take care.")
break
eliza_bot = Eliza()
response = eliza_bot.respond(user_input)
print(f"ELIZA: {response}")

# Start the ELIZA-like conversation

run_eliza()

OUTPUT:
EX:8
AIM:

Translate the following sentences into propositional logic and verify that they

parse with LogicParser. Provide a key that shows how the propositional variables

in your translation correspond to expressions of English.

a. If Angus sings, it is not the case that Bertie sulks.

b. Cyril runs and barks.

c. It will snow if it doesn’t rain.

d. It’s not the case that Irene will be happy if Olive or Tofu comes.

e. Pat didn’t cough or sneeze.

f. If you don’t come if I call, I won’t come if you call.

2. ○ Translate the following sentences into predicate-argument formulas of first-order

logic.

a. Angus likes Cyril and Irene hates Cyril.

b. Tofu is taller than Bertie.

c. Bruce loves himself and Pat does too.

d. Cyril saw Bertie, but Angus didn’t.

e. Cyril is a four-legged friend.

f. Tofu and Olive are near each other.

3. ○ Translate the following sentences into quantified formulas of first-order logic.

a. Angus likes someone and someone likes Julia.

b. Angus loves a dog who loves him.

c. Nobody smiles at Pat.

d. Somebody coughs and sneezes.

e. Nobody coughed or sneezed.

f. Bruce loves somebody other than Bruce.

g. Nobody other than Matthew loves Pat.

h. Cyril likes everyone except for Irene.

i. Exactly one person is asleep.

4. ○ Translate the following verb phrases using λ-abstracts and quantified formulas
of first-order logic.

a. feed Cyril and give a capuccino to Angus

b. be given ‘War and Peace’ by Pat

c. be loved by everyone

d. be loved or detested by everyone

e. be loved by everyone and detested by no-one

CODE:
from [Link] import LogicParser

parser = LogicParser()

def parse_propositional_logic():
print("Propositional Logic Translations:")
sentences = {
"If Angus sings, it is not the case that Bertie sulks.": "A -> ~B",
"Cyril runs and barks.": "C & D",
"It will snow if it doesn’t rain.": "~R -> S",
"Pat didn’t cough or sneeze.": "~C & ~S",
}

for english, logic in [Link]():

print(f"{english} \nLogic: {logic}")
try:
parsed = [Link](logic)
print(f"Parsed Successfully: {parsed}\n")
except Exception as e:
print(f"Error Parsing: {e}\n")

parse_propositional_logic()
def parse_predicate_logic():
print("Predicate-Argument Logic Translations:")
sentences = {
"Angus likes Cyril and Irene hates Cyril.": "likes(angus, cyril) &
hates(irene, cyril)",
"Tofu is taller than Bertie.": "taller(tofu, bertie)",
"Bruce loves himself and Pat does too.": "loves(bruce, bruce) & loves(pat,
pat)",
"Cyril saw Bertie, but Angus didn’t.": "saw(cyril, bertie) & ~saw(angus,
bertie)",
"Cyril is a four-legged friend.": "four_legged(cyril) & friend(cyril)",
"Tofu and Olive are near each other.": "near(tofu, olive)"
}
 for english, logic in [Link]():
print(f"{english} \nLogic: {logic}")
try:
parsed = [Link](logic)
print(f"Parsed Successfully: {parsed}\n")
except Exception as e:
print(f"Error Parsing: {e}\n")
parse_predicate_logic()

def parse_quantified_logic():
print("Quantified Logic Translations:")
sentences = {
"Angus likes someone and someone likes Julia.": "exists x. (likes(angus,
x) & likes(x, julia))",
"Angus loves a dog who loves him.": "exists x. (dog(x) & loves(angus, x) &
loves(x, angus))",
"Somebody coughs and sneezes.": "exists x. (coughs(x) & sneezes(x))",
"Bruce loves somebody other than Bruce.": "exists x. (x != bruce &
loves(bruce, x))",
"Nobody other than Matthew loves Pat.": "forall x. (x != matthew ->
~loves(x, pat))",
"Cyril likes everyone except for Irene.": "forall x. (x != irene ->
likes(cyril, x))",
"Exactly one person is asleep.": "exists x. (asleep(x) & forall y. (y != x
-> ~asleep(y)))"
}

for english, logic in [Link]():

print(f"{english} \nLogic: {logic}")
try:
parsed = [Link](logic)
print(f"Parsed Successfully: {parsed}\n")
except Exception as e:
print(f"Error Parsing: {e}\n")
parse_quantified_logic()
OUTPUT:
EX:9
AIM:
Using collocations ,Implement Bigram and trigram from the given corpus data

CODE:
def extract_bigrams_and_trigrams():
wsj_sents = [Link]()

words = [word for sent in wsj_sents for word in sent]

print("Total words in the corpus:", len(words))

bigram_finder = BigramCollocationFinder.from_words(words)
bigram_finder.apply_freq_filter(3) # Only consider bigrams with a frequency
>= 3
bigrams = bigram_finder.nbest(BigramAssocMeasures.likelihood_ratio, 10) # Top
10 bigrams

print("Top 10 Bigrams:")
for bigram in bigrams:
print(bigram)

trigram_finder = TrigramCollocationFinder.from_words(words)
trigram_finder.apply_freq_filter(3) # Only consider trigrams with a frequency
>= 3
trigrams = trigram_finder.nbest(TrigramAssocMeasures.likelihood_ratio, 10) #
Top 10 trigrams

print("\nTop 10 Trigrams:")
for trigram in trigrams:
print(trigram)

OUTPUT:
EX:10
AIM:
Implement bigram for berkeley restaurant project corpus file and perform the below

CODE:
if __name__ == "__main__":
file_path = "berkeley_restaurant.txt"
tokens = get_text_from_file(file_path)
vocabulary = set(tokens)
bigrams = get_bigrams(tokens)
cpd = laplace_smoothing(bigrams, vocabulary)
word1 = input("Enter the first word of the bigram: ")
word2 = input("Enter the second word of the bigram: ")
random_bigram = (word1, word2)
print("Random Bigram:", random_bigram)
probability = get_bigram_probability(cpd, random_bigram)
print(f"Probability of the bigram {random_bigram}: {probability}")
kn_model = kneser_ney_smoothing([tokens])
kn_probability = kn_model.score(random_bigram[1], [random_bigram[0]])
print(f"Kneser-Ney probability of the bigram {random_bigram}:
{kn_probability}")
def get_bigram_probability(cpd, bigram):
w1, w2 = bigram
return cpd[w1].prob(w2)

def kneser_ney_smoothing(corpus):
train_data, padded_vocab = padded_everygram_pipeline(2, corpus)
model = KneserNeyInterpolated(order=2)
[Link](train_data, padded_vocab)
return model
def get_bigrams(tokens):
return list(ngrams(tokens, 2))

def laplace_smoothing(bigrams, vocabulary):

fd = ConditionalFreqDist()
for w1, w2 in bigrams:
fd[w1][w2] += 1
cpd = ConditionalProbDist(fd, [Link], bins=len(vocabulary))
return cpd
import pandas as pd

if __name__ == "__main__":
file_path = "berkeley_restaurant.txt"
tokens = get_text_from_file(file_path)
vocabulary = set(tokens)
vocab_size = len(vocabulary) # Calculate the vocabulary size
bigrams = get_bigrams(tokens)

# Ensure bigrams are tuples

bigrams = [tuple(bigram) for bigram in bigrams]

# Laplace (Add-One) Smoothing

cpd = laplace_smoothing(bigrams, vocabulary)

# Get all bigrams and their probabilities

bigram_list = list(bigrams)
add_one_probabilities = []
for bigram in bigram_list:
probability = get_bigram_probability(cpd, bigram)
add_one_probabilities.append((bigram, probability))

# Create a DataFrame for Add-One Smoothing Probabilities

add_one_df = [Link](add_one_probabilities, columns=['Bigram',
'Probability'])
print("Add-One Smoothing Probabilities:")
print(add_one_df)

# Kneser-Ney Smoothing
kn_model = kneser_ney_smoothing(bigrams, vocab_size) # Pass vocabulary size

kn_probabilities = []
for bigram in bigram_list:
kn_probability = kn_model(bigram[1], [bigram[0]])
kn_probabilities.append((bigram, kn_probability))

# Create a DataFrame for Kneser-Ney Smoothing Probabilities

kn_df = [Link](kn_probabilities, columns=['Bigram', 'Probability'])
print("\nKneser-Ney Smoothing Probabilities:")
print(kn_df)

OUTPUT:
EX:11
AIM:

.Run your n-gram program on two different small corpora of your choice (you

might use email text or newsgroups from NLTK). Now compare the statistics of the two

corpora. (in fig)What are the differences in the most common unigrams between the

two? How about interesting differences in bigrams?

(i) Add an option to your program to generate random sentences.

(ii) Add an option to your program to compute the perplexity of a test set.

CODE:
import random
import nltk
from nltk import word_tokenize, bigrams
from [Link] import FreqDist, MLEProbDist
from [Link] import reuters, brown
import numpy as np

[Link]('reuters')
[Link]('brown')
[Link]('punkt')
def get_corpus_text(corpus_name):
if corpus_name == "reuters":
return " ".join([Link]())
elif corpus_name == "brown":
return " ".join([Link]())
else:
raise ValueError("Invalid corpus name. Choose 'reuters' or 'brown'.")

def compute_ngrams(corpus_text, n=1):

tokens = word_tokenize(corpus_text.lower())
if n == 1:
return list(tokens)
elif n == 2:
return list(bigrams(tokens))
else:
raise ValueError("Only unigram and bigram supported")

def get_ngram_statistics(corpus_name):
text = get_corpus_text(corpus_name)
unigrams = compute_ngrams(text, 1)
bigram_list = compute_ngrams(text, 2)

unigram_freq = FreqDist(unigrams)
bigram_freq = FreqDist(bigram_list)

return unigram_freq, bigram_freq

def generate_sentence(unigram_freq, sentence_length=10):

words = list(unigram_freq.keys())
return " ".join([Link](words, weights=unigram_freq.values(),
k=sentence_length))

def compute_perplexity(test_set, unigram_freq):

test_tokens = word_tokenize(test_set.lower())
N = len(test_tokens)
prob_model = MLEProbDist(unigram_freq)

log_prob_sum = sum(np.log2(prob_model.prob(word)) for word in test_tokens if

prob_model.prob(word) > 0)
perplexity = 2 ** (-log_prob_sum / N)

return perplexity

# Compare corpora
corpora = ["reuters", "brown"]
for corpus in corpora:
unigram_freq, bigram_freq = get_ngram_statistics(corpus)
print(f"Most common unigrams in {corpus}: {unigram_freq.most_common(10)}")
print(f"Most common bigrams in {corpus}: {bigram_freq.most_common(10)}")

# Generate random sentence

print(f"Generated sentence from {corpus}: {generate_sentence(unigram_freq)}")
 # Compute perplexity (using a sample of test text from the corpus)
test_text = " ".join(get_corpus_text(corpus).split()[:50]) # Taking first 50
words as test set
print(f"Perplexity of {corpus} test set: {compute_perplexity(test_text,
unigram_freq)}")
print("-"*50)

OUTPUT:
EX:12
AIM:

Write a program to convert a simple audio file (.wav) into text using a speech-to-text library like
SpeechRecognition.

 Example Input: An audio file saying, "Hello, how are you?"

 Expected Output: Text: "Hello, how are you?"

CODE:
import speech_recognition as sr

def audio_to_text(audio_file):
"""Convert WAV audio to text using Google Speech Recognition"""
recognizer = [Link]()

with [Link](audio_file) as source:

print("Processing audio file...")
recognizer.adjust_for_ambient_noise(source) # Reduce noise
audio_data = [Link](source) # Record the entire audio

try:
text = recognizer.recognize_google(audio_data)
return text
except [Link]:
return "Speech Recognition could not understand the audio"
except [Link]:
return "Could not request results from Google Speech Recognition
service"

# Path to your .wav file

audio_file_path = "[Link]"

# Convert audio to text

if audio_file_path:
converted_text = audio_to_text(audio_file_path)
print("Converted Text:", converted_text)

OUTPUT:
EX:13
AIM:
Implement a program to detect the language and specify it using Langid for any 5 languages

CODE:
import langid

# Mapping of language codes to full language names

language_names = {
'af': 'Afrikaans', 'am': 'Amharic', 'an': 'Aragonese', 'ar': 'Arabic', 'as':
'Assamese', 'az': 'Azerbaijani',
'be': 'Belarusian', 'bg': 'Bulgarian', 'bn': 'Bengali', 'br': 'Breton', 'bs':
'Bosnian', 'ca': 'Catalan',
'cs': 'Czech', 'cy': 'Welsh', 'da': 'Danish', 'de': 'German', 'dz':
'Dzongkha', 'el': 'Greek', 'en': 'English',
'eo': 'Esperanto', 'es': 'Spanish', 'et': 'Estonian', 'eu': 'Basque', 'fa':
'Persian', 'fi': 'Finnish',
'fo': 'Faroese', 'fr': 'French', 'ga': 'Irish', 'gl': 'Galician', 'gu':
'Gujarati', 'he': 'Hebrew', 'hi': 'Hindi',
'hr': 'Croatian', 'ht': 'Haitian', 'hu': 'Hungarian', 'hy': 'Armenian', 'id':
'Indonesian', 'is': 'Icelandic',
'it': 'Italian', 'ja': 'Japanese', 'jv': 'Javanese', 'ka': 'Georgian', 'kk':
'Kazakh', 'km': 'Khmer', 'kn': 'Kannada',
'ko': 'Korean', 'lt': 'Lithuanian', 'lv': 'Latvian', 'mk': 'Macedonian', 'ml':
'Malayalam', 'mn': 'Mongolian',
'mr': 'Marathi', 'ms': 'Malay', 'my': 'Burmese', 'nb': 'Norwegian Bokmål',
'ne': 'Nepali', 'nl': 'Dutch',
'nn': 'Norwegian Nynorsk', 'no': 'Norwegian', 'oc': 'Occitan', 'or': 'Oriya',
'pa': 'Punjabi', 'pl': 'Polish',
'ps': 'Pashto', 'pt': 'Portuguese', 'qu': 'Quechua', 'ro': 'Romanian', 'ru':
'Russian', 'rw': 'Kinyarwanda',
'se': 'Northern Sami', 'si': 'Sinhala', 'sk': 'Slovak', 'sl': 'Slovenian',
'sq': 'Albanian', 'sr': 'Serbian',
'sv': 'Swedish', 'sw': 'Swahili', 'ta': 'Tamil', 'te': 'Telugu', 'th': 'Thai',
'tl': 'Tagalog', 'tr': 'Turkish',
'ug': 'Uighur', 'uk': 'Ukrainian', 'ur': 'Urdu', 'vi': 'Vietnamese', 'vo':
'Volapük', 'wa': 'Walloon',
'xh': 'Xhosa', 'zh': 'Chinese', 'zu': 'Zulu'
}

def detect_language(text):
detected_lang, confidence = [Link](text)
full_lang_name = language_names.get(detected_lang, "Unknown Language")
return f"Detected Language: {full_lang_name} (Confidence: {confidence})"

# Sample texts in different languages

texts = {
"Hello, how are you?",
"Bonjour, comment ça va?",
"Hola, ¿cómo estás?",
"Hallo, wie geht's dir?",
"नमस्ते, आप कैसे हैं?",
 "你好，你怎么样?",
"‫ كيف حالك؟‬،‫"مرحبا‬
}

# Detect language for each text

for sample_text in texts:
result = detect_language(sample_text)
print(f"Sample Text: {sample_text}\n{result}\n")

OUTPUT:
EX:14
AIM:
Detect the language of the text “mwanafunzi wa Taasisi ya Teknolojia ya Coimbatore” and translate the text
to English Language. Use RNN based machine translation system

CODE:
import torch
import [Link] as nn
import [Link] as optim
import numpy as np

input_text = "mwanafunzi wa Taasisi ya Teknolojia ya Coimbatore"

target_text = "student of the Coimbatore Institute of Technology"

input_words = input_text.split()
target_words = target_text.split()
vocab = list(set(input_words + target_words))
word2idx = {word: i for i, word in enumerate(vocab)}
idx2word = {i: word for word, i in [Link]()}

input_indices = [word2idx[word] for word in input_words]

target_indices = [word2idx[word] for word in target_words]

# Fix the class definition

class SimpleRNN([Link]):
def __init__(self, input_size, hidden_size, output_size): # Fixed __init__
super(SimpleRNN, self).__init__() # Fixed super() call
self.hidden_size = hidden_size
[Link] = [Link](input_size, hidden_size)
[Link] = [Link](hidden_size, hidden_size, batch_first=True)
[Link] = [Link](hidden_size, output_size)

def forward(self, x, hidden):

x = [Link](x).unsqueeze(0)
out, hidden = [Link](x, hidden)
out = [Link]([Link](0))
return out, hidden

input_size = len(vocab)
hidden_size = 16
output_size = len(vocab)

model = SimpleRNN(input_size, hidden_size, output_size) # Now it works!

criterion = [Link]()
optimizer = [Link]([Link](), lr=0.01)

epochs = 1000
for epoch in range(epochs):
hidden = [Link](1, 1, hidden_size)
model.zero_grad()
 loss = 0

for i in range(len(input_indices)):
x = [Link]([input_indices[i]], dtype=[Link])
y = [Link]([target_indices[i]], dtype=[Link])
output, hidden = model(x, hidden)
loss += criterion(output, y)

[Link]()
[Link]()

if epoch % 100 == 0:
print(f'Epoch {epoch}, Loss: {[Link]()}')

hidden = [Link](1, 1, hidden_size)

predicted_indices = []
for i in range(len(input_indices)):
x = [Link]([input_indices[i]], dtype=[Link])
output, hidden = model(x, hidden)
predicted_word_index = [Link](output).item()
predicted_indices.append(predicted_word_index)

predicted_translation = ' '.join([idx2word[idx] for idx in predicted_indices])

print(f'Translated Text: {predicted_translation}')

OUTPUT:
EX:15
AIM:
Build a ML Model to classify 20 News group Dataset using Naive bayes for NLTK Corpus

CODE:
def get_labeled_data():
labeled_data = []
for category in movie_reviews.categories():
for fileid in movie_reviews.fileids(category):
words = movie_reviews.words(fileid)
labeled_data.append((words, category))
return labeled_data
def extract_features(words):
return {word: True for word in words}

labeled_data = get_labeled_data()
[Link](labeled_data)
featuresets = [(extract_features(words), label) for words, label in labeled_data]
train_size = int(len(featuresets) * 0.8)
train_data, test_data = featuresets[:train_size], featuresets[train_size:]
classifier = [Link](train_data)
print(f"Model Accuracy: {accuracy(classifier, test_data) * 100:.2f}%")
def classify_review(review):
words = nltk.word_tokenize([Link]())
features = extract_features(words)
return [Link](features)
test_reviews = [
"This movie was fantastic! The story was engaging and the characters were
great.",
"Absolutely terrible. The plot made no sense and the acting was awful.",
"The movie was very excellent and was very excited to watch."
]

for review in test_reviews:

sentiment = classify_review(review)
print(f"Review: \"{review}\" -> Sentiment: {sentiment}")

OUTPUT:
EX:16
AIM:

By using Skip-gram train a logistic regression classifier to compute the probability that two words are
‘likely to occur nearby in text’. Use dataset from WSJ corpus

CODE:
import nltk
import random
import numpy as np
import seaborn as sns
import [Link] as plt
from [Link] import treebank
from sklearn.linear_model import LogisticRegression
from [Link] import Word2Vec
from [Link] import PCA
from [Link] import confusion_matrix

# Download WSJ corpus (Penn Treebank)

# [Link]('treebank')

# Load sentences from WSJ corpus

sentences = list([Link]())

# Train Skip-gram model using Word2Vec

word2vec_model = Word2Vec(sentences, vector_size=100, window=5, sg=1, min_count=2,
workers=4)

# Function to generate Skip-gram training data

def generate_skipgram_data(sentences, window_size=5):
training_data = []
vocab = set(word2vec_model.wv.index_to_key)

for sentence in sentences:

for i, word in enumerate(sentence):
if word in vocab:
# Define the context window
start, end = max(0, i - window_size), min(len(sentence), i +
window_size + 1)
for j in range(start, end):
if i != j and sentence[j] in vocab:
training_data.append((word, sentence[j], 1)) # Positive
pair
negative_sample = [Link](list(vocab))
training_data.append((word, negative_sample, 0)) #
Negative pair
return training_data

# Generate dataset
skipgram_data = generate_skipgram_data(sentences)

# Convert words to vectors

X, y = [], []
for word1, word2, label in skipgram_data:
vector1, vector2 = word2vec_model.wv[word1], word2vec_model.wv[word2]
[Link]([Link]((vector1, vector2))) # Concatenate vectors
[Link](label)

X = [Link](X)
y = [Link](y)

# ✅ Reduce dimensions using PCA

pca = PCA(n_components=2)
X_reduced = pca.fit_transform(X)

# ✅ Reshape target values into a 1D array

y = [Link]()

# ✅ Train logistic regression on reduced dimensions

classifier = LogisticRegression(max_iter=1000)
[Link](X_reduced, y)

# ✅ Predict and evaluate performance

y_pred = [Link](X_reduced)

# ✅ Generate confusion matrix

confusion = confusion_matrix(y, y_pred)

# ✅ Plot heatmap of actual vs predicted

[Link](figsize=(6, 4))
[Link](confusion, annot=True, fmt="d", cmap="Blues", xticklabels=['Not
Nearby', 'Nearby'], yticklabels=['Not Nearby', 'Nearby'])
[Link]('Predicted')
[Link]('Actual')
[Link]('Confusion Matrix for Skip-Gram Model')
[Link]()

# ✅ Test with dynamic input

while True:
print("\n🔎 Test Word Pair Probability")
word1 = input("Enter first word (or 'exit' to quit): ").strip()
if [Link]() == 'exit':
break

word2 = input("Enter second word: ").strip()

if word1 in word2vec_model.wv and word2 in word2vec_model.wv:

test_vector = [Link]((word2vec_model.wv[word1],
word2vec_model.wv[word2]))
test_vector_reduced = [Link]([test_vector])
probability = classifier.predict_proba(test_vector_reduced)[0][1]
print(f"Probability that '{word1}' and '{word2}' occur nearby:
{probability:.4f}\n")
else:
print("One or both words not in vocabulary. Try again.\n")
OUTPUT:
EX:17
AIM:

Find similar vectors on TNSE - class with GLOVE and visualize it with plots

CODE:
import numpy as np
import [Link] as plt
from [Link] import TSNE

# Step 1: Load GloVe embeddings

def load_glove_embeddings(glove_file):
embeddings_index = {}
with open(glove_file, encoding='utf8') as f:
for line in f:
values = [Link]()
word = values[0]
vector = [Link](values[1:], dtype='float32')
embeddings_index[word] = vector
return embeddings_index

# Path to GloVe file (update path as needed)

glove_path = 'D:\\8TH SEM\\NLP Lab\\Glove\\[Link]'
embeddings_index = load_glove_embeddings(glove_path)

# Step 2: Select words for analysis

words = ['good', 'bad', 'nice', 'terrific', 'horrible', 'dislike', 'fantastic',
'worst', 'best', 'incredible', 'awful', 'happy', 'sad', 'amazing',
'great']

# Filter out missing words

word_vectors = [Link]([embeddings_index[word] for word in words if word in
embeddings_index])
labels = [word for word in words if word in embeddings_index]

# Step 3: Apply t-SNE for dimensionality reduction (adjust perplexity)

tsne = TSNE(n_components=2, perplexity=5, random_state=42)
reduced_vectors = tsne.fit_transform(word_vectors)

# Step 4: Plot the embeddings

[Link](figsize=(12, 8))

# Define color categories

color_map = {
'positive': 'green',
'negative': 'red',
'neutral': 'blue'
}

# Keep track of handles for the legend

handles = {}
for i, label in enumerate(labels):
x, y = reduced_vectors[i]
if label in ['good', 'nice', 'terrific', 'fantastic', 'amazing', 'great',
'happy', 'best']:
color = color_map['positive']
category = 'Positive'
elif label in ['bad', 'worst', 'horrible', 'dislike', 'awful', 'sad']:
color = color_map['negative']
category = 'Negative'
else:
color = color_map['neutral']
category = 'Neutral'

# Plot the point

scatter = [Link](x, y, color=color)

# Add text next to the point

[Link](x + 0.1, y + 0.1, label, fontsize=12)

# Add to handles if not already added

if category not in handles:
handles[category] = scatter

# Add legend
[Link]([Link](), [Link](), title="Sentiment", loc="best")

[Link]("t-SNE Projection of Word Vectors")

[Link]()

OUTPUT:
EX:18
AIM:
Sentiment Analysis of text reviews using LSTM

CODE:

# Load dataset
file_path = 'D:\\8TH SEM\\NLP Lab\\IMDB [Link]' # Update with your file
path
df = pd.read_csv(file_path)

# Preview the dataset

print([Link]())

# Assuming dataset has 'review' and 'sentiment' columns

df = df[['review', 'sentiment']]

# Drop missing values

[Link](inplace=True)

# Function to clean text

def clean_text(text):
text = [Link]() # Convert to lowercase
text = [Link](r'[^\w\s]', '', text) # Remove punctuation
text = [Link](r'\d+', '', text) # Remove numbers
text = [Link](r'\s+', ' ', text).strip() # Remove extra whitespaces
return text

# Clean the reviews

df['cleaned_review'] = df['review'].apply(clean_text)

# Split dataset into training and test sets (80-20 split)

train_texts, test_texts, train_labels, test_labels = train_test_split(
df['cleaned_review'], df['sentiment'], test_size=0.2, random_state=42
)

print(f"Training size: {len(train_texts)}, Test size: {len(test_texts)}")

# Set maximum number of words in vocabulary

MAX_WORDS = 1000
SEQUENCE_LENGTH = 10

# Initialize and fit tokenizer

tokenizer = Tokenizer(num_words=MAX_WORDS, oov_token='<OOV>')
tokenizer.fit_on_texts(train_texts)

# Convert text to sequences

train_sequences = tokenizer.texts_to_sequences(train_texts)
test_sequences = tokenizer.texts_to_sequences(test_texts)
# Pad sequences to a fixed length
train_sequences = pad_sequences(train_sequences, maxlen=SEQUENCE_LENGTH,
padding='post')
test_sequences = pad_sequences(test_sequences, maxlen=SEQUENCE_LENGTH,
padding='post')

train_labels = [Link](train_labels == 'positive', 1, 0).astype('int32')

test_labels = [Link](test_labels == 'positive', 1, 0).astype('int32')

print(f'Tokenized Training Data Shape: {train_sequences.shape}')

print(f'Tokenized Test Data Shape: {test_sequences.shape}')

# Define model
model = Sequential()
[Link](Embedding(input_dim=MAX_WORDS, output_dim=32,
input_length=SEQUENCE_LENGTH))
[Link](LSTM(254, return_sequences=False))
[Link](Dense(1, activation='sigmoid')) # Binary classification

# Compile model
[Link](optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

# Display model summary

[Link]()

# Train the model

history = [Link](
train_sequences, train_labels,
epochs=10,
batch_size=32,
validation_data=(test_sequences, test_labels)
)

# Predict on test data

y_pred = ([Link](test_sequences) > 0.5).astype("int32")

# Calculate metrics
accuracy = accuracy_score(test_labels, y_pred)
precision = precision_score(test_labels, y_pred)
recall = recall_score(test_labels, y_pred)
f1 = f1_score(test_labels, y_pred)

print(f"Accuracy: {accuracy:.4f}")
print(f"Precision: {precision:.4f}")
print(f"Recall: {recall:.4f}")
print(f"F1 Score: {f1:.4f}")
import [Link] as plt

# Plot training & validation accuracy values

[Link](figsize=(12, 4))
[Link](1, 2, 1)
[Link]([Link]['accuracy'], label='Train')
[Link]([Link]['val_accuracy'], label='Validation')
[Link]('Model Accuracy')
[Link]('Accuracy')
[Link]('Epoch')
[Link]()

# Plot training & validation loss values

[Link](1, 2, 2)
[Link]([Link]['loss'], label='Train')
[Link]([Link]['val_loss'], label='Validation')
[Link]('Model Loss')
[Link]('Loss')
[Link]('Epoch')
[Link]()

[Link]()

OUTPUT:
EX:19
AIM:

Next Word Prediction using LSTM and GRU, Compare the performance of both

CODE:
import numpy as np
import tensorflow as tf
from [Link] import Tokenizer
from [Link] import pad_sequences
from [Link] import Sequential
from [Link] import Embedding, LSTM, GRU, Dense
from [Link] import reuters
import nltk
import random

# Ensure the dataset is downloaded

[Link]('reuters')
[Link]('punkt')

# Load and clean text data

documents = [Link]()
text_data = " ".join([[Link](doc) for doc in [Link](documents, 100)])
# Using 100 random docs
text_data = text_data.lower()

# Tokenization
tokenizer = Tokenizer()
tokenizer.fit_on_texts([text_data])
total_words = len(tokenizer.word_index) + 1

# Create sequences
input_sequences = []
for line in nltk.sent_tokenize(text_data):
token_list = tokenizer.texts_to_sequences([line])[0]
for i in range(1, len(token_list)):
input_sequences.append(token_list[:i + 1])

# Apply padding
max_sequence_length = max(len(seq) for seq in input_sequences)
padded_sequences = pad_sequences(input_sequences, maxlen=max_sequence_length,
padding='pre')
X, y = padded_sequences[:, :-1], padded_sequences[:, -1]
y = [Link].to_categorical(y, num_classes=total_words)

# Define function to build LSTM and GRU models

def build_model(rnn_layer):
model = Sequential([
Embedding(total_words, 50, input_length=max_sequence_length - 1),
rnn_layer(100, return_sequences=True),
rnn_layer(100),
Dense(total_words, activation='softmax')
 ])
[Link](loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy'])
return model

# Train LSTM model

lstm_model = build_model(LSTM)
lstm_model.fit(X, y, epochs=10, verbose=1)

# Train GRU model

gru_model = build_model(GRU)
gru_model.fit(X, y, epochs=10, verbose=1)

# Function to predict next word

def predict_next_word(model, seed_text):
token_list = tokenizer.texts_to_sequences([seed_text])[0]
token_list = pad_sequences([token_list], maxlen=max_sequence_length - 1,
padding='pre')
predicted_probs = [Link](token_list, verbose=0)
predicted_index = [Link](predicted_probs)
return tokenizer.index_word.get(predicted_index, "<unknown>")

# Sample predictions
seed_text = "the market is"
print("LSTM Prediction:", predict_next_word(lstm_model, seed_text))
print("GRU Prediction:", predict_next_word(gru_model, seed_text))

# Evaluate performance
lstm_loss, lstm_acc = lstm_model.evaluate(X, y, verbose=0)
gru_loss, gru_acc = gru_model.evaluate(X, y, verbose=0)
print(f"LSTM Accuracy: {lstm_acc:.4f}, Loss: {lstm_loss:.4f}")
print(f"GRU Accuracy: {gru_acc:.4f}, Loss: {gru_loss:.4f}")

# Inference
if lstm_acc > gru_acc:
print("LSTM performed better than GRU for this dataset.")
elif gru_acc > lstm_acc:
print("GRU performed better than LSTM for this dataset.")
else:
print("Both models performed similarly.")
# Function to interactively predict next words
def interactive_prediction(model, tokenizer, max_sequence_length):
while True:
seed_text = input("\nEnter a sentence (or type 'exit' to quit): ").strip()
if seed_text.lower() == 'exit':
print("Exiting interactive mode.")
break
predicted_word = predict_next_word(model, seed_text)
print(f"Predicted Next Word for {seed_text}: {predicted_word}")

# Choose model (LSTM or GRU)

model_choice = input("Choose a model (LSTM/GRU): ").strip().lower()
if model_choice == "lstm":
print("Using LSTM Model for Prediction...")
 interactive_prediction(lstm_model, tokenizer, max_sequence_length)
elif model_choice == "gru":
print("Using GRU Model for Prediction...")
interactive_prediction(gru_model, tokenizer, max_sequence_length)
else:
print("Invalid choice. Please enter 'LSTM' or 'GRU'.")

OUTPUT:
EX:20
AIM:

Implement CBOW for a sample sentence/corpus using Neural Networks

Draw CBOW Architecture in your note

( Prepare data, then create input-output pairs, define a CBOW model architecture, train the model, and
finally evaluate its performance. )

CODE:
import numpy as np
import tensorflow as tf
from [Link] import Sequential
from [Link] import Embedding, Lambda, Dense
from [Link] import Tokenizer
from [Link] import pad_sequences

# Sample corpus
corpus = ["the quick brown fox jumps over the lazy dog"]

# Step 1: Tokenization
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
word2idx = tokenizer.word_index # Mapping words to indices
idx2word = {v: k for k, v in [Link]()} # Reverse mapping
vocab_size = len(word2idx) + 1 # Adding 1 for padding

print("Vocabulary:", word2idx)

# Convert text into sequence of word indices

tokens = [word2idx[word] for word in corpus[0].split()]

# Step 2: Create Input-Output Pairs (CBOW)

window_size = 2
data = []
labels = []

for i in range(window_size, len(tokens) - window_size):

context = [tokens[j] for j in range(i - window_size, i)] + \
[tokens[j] for j in range(i + 1, i + 1 + window_size)]
target = tokens[i]

[Link](context)
[Link](target)

# Convert to NumPy arrays

X_train = [Link](data)
y_train = [Link](labels)

print("\nSample Input-Output Pair:")

print("Context words:", X_train[0])
print("Target word:", y_train[0])

# Step 3: Define CBOW Model Architecture

embedding_dim = 10 # Dimension of word embeddings

model = Sequential([
Embedding(input_dim=vocab_size, output_dim=embedding_dim,
input_length=window_size * 2),
Lambda(lambda x: tf.reduce_mean(x, axis=1)), # Averaging embeddings
Dense(vocab_size, activation='softmax')
])

[Link](loss='sparse_categorical_crossentropy', optimizer='adam',
metrics=['accuracy'])
[Link]()

# Step 4: Train the Model

[Link](X_train, y_train, epochs=100, verbose=2)
0
# Step 5: Evaluate the Model
def predict_context(context_words):
context_vector = [Link]([[word2idx[word] for word in context_words]])
predicted_idx = [Link]([Link](context_vector))
return idx2word[predicted_idx]

# Example Test Case

sample_context = ["the", "quick", "jumps", "over"]
predicted_word = predict_context(sample_context)
print("\nPredicted Word for context", sample_context, ":", predicted_word)

OUTPUT:
EX:21
AIM:
1. Perform Named Entity Recognition (NER) using the spacy library.
2. Perform Part-of-Speech (POS) tagging using the NLTK library on a given sentence based on the
Penn Treebank tagset.
3. Implement a Hidden Markov Model (HMM) POS tagger using NLTK and train it on the Brown
corpus.
Steps:
 Load POS-tagged sentences from the Brown corpus/WSJ
 Split data into training and testing sets.
 Train an HMM-based POS tagger on the training set.
 Predict POS tags for a sample sentence.

CODE:
import spacy
import nltk
from nltk import pos_tag, word_tokenize
from [Link] import brown
from [Link] import hmm
import random

# [Link]('averaged_perceptron_tagger')
# [Link]('punkt')
# [Link]('brown')
# [Link]('universal_tagset')

def named_entity_recognition(text):
nlp = [Link]("en_core_web_sm")
doc = nlp(text)
print("\nNamed Entities:")
for ent in [Link]:
print(f"{[Link]} ({ent.label_})")

def pos_tagging_nltk(text):
tokens = word_tokenize(text)
tagged = pos_tag(tokens)
print("\nPOS Tagging using NLTK:")
print(tagged)

def train_hmm_pos_tagger():
tagged_sentences = list(brown.tagged_sents(tagset='universal')) # Convert to
list

[Link](tagged_sentences)
split = int(0.9 * len(tagged_sentences))
train_data = tagged_sentences[:split]
test_data = tagged_sentences[split:]

trainer = [Link]()
hmm_tagger = [Link](train_data)
 sample_sentence = "The quick brown fox jumps over the lazy dog"
sample_tokens = word_tokenize(sample_sentence)
predicted_tags = hmm_tagger.tag(sample_tokens)

print("\nHMM POS Tagging:")

print(predicted_tags)

if __name__ == "__main__":
text = "Apple Inc. was founded by Steve Jobs and Steve Wozniak in Cupertino."
named_entity_recognition(text)
pos_tagging_nltk(text)
train_hmm_pos_tagger()

OUTPUT:
EX:22
AIM:
Implementing the Viterbi Algorithm for POS Tagging

CODE:
import numpy as np

def viterbi_pos_tagging(words, tags, transition_prob, emission_prob):

num_tags = len(tags)
num_words = len(words)

# Initialize Viterbi and Backpointer matrices

viterbi = [Link]((num_tags, num_words))
backpointer = [Link]((num_tags, num_words), dtype=int)

# Step 1: Initialization (first word probabilities)

first_word = words[0]
for i, tag in enumerate(tags):
if first_word in emission_prob[tag]:
viterbi[i, 0] = transition_prob['START'].get(tag, 0) *
emission_prob[tag][first_word]
else:
viterbi[i, 0] = 0 # Handle unknown words (smoothing can be added)

# Step 2: Recursion (forward pass)

for t in range(1, num_words):
current_word = words[t]
for i, current_tag in enumerate(tags):
max_prob = -1
best_prev_tag = -1
for j, prev_tag in enumerate(tags):
prob = viterbi[j, t-1] *
transition_prob[prev_tag].get(current_tag, 0)
if current_word in emission_prob[current_tag]:
prob *= emission_prob[current_tag][current_word]
else:
prob *= 0 # Handle unknown words (smoothing can be added)

if prob > max_prob:

max_prob = prob
best_prev_tag = j

viterbi[i, t] = max_prob
backpointer[i, t] = best_prev_tag

# Step 3: Termination (find best path)

best_last_tag = [Link](viterbi[:, -1])
best_path = [best_last_tag]

# Step 4: Backtracking
for t in range(num_words-1, 0, -1):
best_prev_tag = backpointer[best_path[-1], t]
 best_path.append(best_prev_tag)

best_path.reverse()
best_tags = [tags[idx] for idx in best_path]

return best_tags

# Example Usage
if __name__ == "__main__":
# Define POS tags
tags = ['NNP', 'MD', 'VB', 'JJ', 'NN', 'DT']

# Define transition probabilities (A)

transition_prob = {
'START': {'NNP': 0.5, 'MD': 0.1, 'VB': 0.1, 'JJ': 0.1, 'NN': 0.1, 'DT':
0.1},
'NNP': {'MD': 0.3, 'VB': 0.1, 'NN': 0.1, 'DT': 0.1},
'MD': {'VB': 0.4, 'NN': 0.2, 'JJ': 0.1},
'VB': {'DT': 0.3, 'NN': 0.2},
'JJ': {'NN': 0.4},
'NN': {'END': 0.2},
'DT': {'NN': 0.5, 'JJ': 0.2}
}

# Define emission probabilities (B)

emission_prob = {
'NNP': {'Janet': 0.8, 'bill': 0.1},
'MD': {'will': 0.6},
'VB': {'back': 0.5, 'will': 0.01},
'JJ': {'back': 0.3},
'NN': {'back': 0.1, 'bill': 0.7},
'DT': {'the': 0.9}
}

# Input sentence
sentence = "Janet will back the bill"
words = [Link]()

# Run Viterbi algorithm

best_tags = viterbi_pos_tagging(words, tags, transition_prob, emission_prob)

print("Sentence:", sentence)
print("Predicted POS Tags:", best_tags)

OUTPUT:
EX:23
AIM:
Implement CRF with the following steps:
 Install and set up sklearn-crfsuite for training a CRF model.
 Load the CoNLL-2003 dataset and preprocess it for training a CRF-based NER model.
 Implement tokenization and POS tagging as preprocessing steps for NER.
 Extract word-level features (e.g., word shape, suffix, prefix, POS tag) for CRF training.
 Format the dataset to represent sentences as sequences of features and labels.
 Modify the feature extraction function to include word prefixes along with suffixes.
 Train a CRF model for Named Entity Recognition (NER) using a labeled dataset.
 Modify the CRF model to include contextual features from previous and next words.
 Use Part-of-Speech (POS) tags as additional features in the CRF-based NER model.
 Implement feature scaling techniques to improve CRF model performance.

CODE:
from datasets import load_dataset
import nltk
from [Link] import word_tokenize
from [Link] import pos_tag

# Download NLTK resources

# [Link]('punkt')
# [Link]('averaged_perceptron_tagger')

# Load CoNLL-2003 dataset

conll_dataset = load_dataset("conll2003")

# Preprocess function to extract tokens and NER tags

def preprocess_conll(example):
tokens = example['tokens']
ner_tags = example['ner_tags']
pos_tags = [tag for word, tag in pos_tag(tokens)]
return {'tokens': tokens, 'ner_tags': ner_tags, 'pos_tags': pos_tags}

# Apply preprocessing
train_data = [preprocess_conll(x) for x in conll_dataset['train']]
test_data = [preprocess_conll(x) for x in conll_dataset['test']]
def word2features(sent, i):
word = sent[i][0] # The word itself
pos_tag = sent[i][1] # POS tag (string)

features = {
'bias': '1.0', # Changed to string
'[Link]()': [Link](),
'word[-3:]': word[-3:] if len(word) >= 3 else word,
'word[-2:]': word[-2:] if len(word) >= 2 else word,
'[Link]()': str([Link]()), # Convert boolean to string
'[Link]()': str([Link]()), # Convert boolean to string
'[Link]()': str([Link]()), # Convert boolean to string
'postag': pos_tag,
'postag[:2]': pos_tag[:2],
}
 # Add prefix features
[Link]({
'word[:3]': word[:3] if len(word) >= 3 else word,
'word[:2]': word[:2] if len(word) >= 2 else word,
})

# Contextual features from previous and next words

if i > 0:
prev_word = sent[i-1][0]
prev_postag = sent[i-1][1]
[Link]({
'-1:[Link]()': prev_word.lower(),
'-1:postag': prev_postag,
})
else:
features['BOS'] = 'True' # Changed to string

if i < len(sent)-1:
next_word = sent[i+1][0]
next_postag = sent[i+1][1]
[Link]({
'+1:[Link]()': next_word.lower(),
'+1:postag': next_postag,
})
else:
features['EOS'] = 'True' # Changed to string

return features

def sent2features(sent):
return [word2features(sent, i) for i in range(len(sent))]

def sent2labels(sent):
return [str(label) for token, pos, label in sent] # Convert labels to strings

def sent2tokens(sent):
return [token for token, pos, label in sent]

# Prepare data for CRF - now including POS tags

train_sents = [list(zip(x['tokens'], x['pos_tags'], x['ner_tags'])) for x in
train_data]
test_sents = [list(zip(x['tokens'], x['pos_tags'], x['ner_tags'])) for x in
test_data]

X_train = [sent2features(s) for s in train_sents]

y_train = [sent2labels(s) for s in train_sents]

X_test = [sent2features(s) for s in test_sents]

y_test = [sent2labels(s) for s in test_sents]
import sklearn_crfsuite
from sklearn_crfsuite import metrics

# Train CRF model

crf = sklearn_crfsuite.CRF(
algorithm='lbfgs',
c1=0.1,
c2=0.1,
max_iterations=100,
all_possible_transitions=True
)

try:
[Link](X_train, y_train)
except AttributeError:
# Fallback for different versions
from sklearn_crfsuite import CRF
crf = CRF(
algorithm='lbfgs',
c1=0.1,
c2=0.1,
max_iterations=100,
all_possible_transitions=True
)
[Link](X_train, y_train)

# Evaluation
y_pred = [Link](X_test)
print(metrics.flat_classification_report(
y_test, y_pred, digits=3
))
def word2features_with_scaling(sent, i):
features = word2features(sent, i)

# Example of feature scaling - word length normalized

word = sent[i][0]
max_word_length = 20 # Assuming maximum word length
features['word.length_scaled'] = len(word) / max_word_length

return features

# Update feature extraction

X_train_scaled = [[word2features_with_scaling(s, i) for i in range(len(s))] for s
in train_sents]
X_test_scaled = [[word2features_with_scaling(s, i) for i in range(len(s))] for s
in test_sents]

# Retrain with scaled features

crf_scaled = sklearn_crfsuite.CRF(
algorithm='lbfgs',
c1=0.1,
c2=0.1,
max_iterations=100,
all_possible_transitions=True
)
crf_scaled.fit(X_train_scaled, y_train)

# Evaluate scaled model

y_pred_scaled = crf_scaled.predict(X_test_scaled)
print(metrics.flat_classification_report(
y_test, y_pred_scaled, digits=3
))
def predict_ner(text):
# Tokenize and POS tag
tokens = word_tokenize(text)
pos_tags = [tag for word, tag in pos_tag(tokens)]

# Prepare sentence for prediction

sent = list(zip(tokens, pos_tags))
features = [word2features(sent, i) for i in range(len(sent))]

# Predict
labels = crf.predict_single(features)

# Return tokens with predicted labels

return list(zip(tokens, labels))

# Example usage
example_text = "Apple is looking to buy U.K. startup for $1 billion"
print(predict_ner(example_text))

OUTPUT:
EX:24

AIM:

Implement a machine translation pipeline using the Transformer architecture from scratch
(without using a pretrained model). The task is to translate text from French to English. Your
implementation should include the following steps:

CODE:
import torch
import [Link] as nn
import [Link] as optim
import numpy as np
import math
import random
from [Link] import Dataset, DataLoader

# Sample parallel corpus (French to English)

french_sentences = [
"je suis étudiant",
"il est médecin",
"elle est enseignante",
"nous aimons apprendre",
"vous parlez anglais?",
"bonjour, comment ça va?",
"je m'appelle pierre",
"quelle heure est-il?"
]

english_sentences = [
"i am a student",
"he is a doctor",
"she is a teacher",
"we love learning",
"do you speak english?",
"hello, how are you?",
"my name is pierre",
"what time is it?"
]

# Add special tokens

SOS_TOKEN = "<sos>"
EOS_TOKEN = "<eos>"
PAD_TOKEN = "<pad>"

class Vocabulary:
def __init__(self):
self.word2index = {PAD_TOKEN: 0, SOS_TOKEN: 1, EOS_TOKEN: 2}
self.word2count = {}
self.index2word = {0: PAD_TOKEN, 1: SOS_TOKEN, 2: EOS_TOKEN}
 self.n_words = 3 # Count SOS, EOS, PAD

def add_sentence(self, sentence):

for word in [Link](' '):
self.add_word(word)

def add_word(self, word):

if word not in self.word2index:
self.word2index[word] = self.n_words
self.word2count[word] = 1
self.index2word[self.n_words] = word
self.n_words += 1
else:
self.word2count[word] += 1

# Create vocabularies
french_vocab = Vocabulary()
english_vocab = Vocabulary()

for sent in french_sentences:

french_vocab.add_sentence(sent)

for sent in english_sentences:

english_vocab.add_sentence(sent)

# Tokenize and numericalize sentences

def sentence_to_indices(sentence, vocab):
indices = [vocab.word2index[word] for word in [Link](' ')]
[Link](0, vocab.word2index[SOS_TOKEN])
[Link](vocab.word2index[EOS_TOKEN])
return indices

# Pad sequences to make them equal length

def pad_sequence(sequence, max_len, pad_idx):
sequence = sequence[:max_len]
padding_length = max_len - len(sequence)
return sequence + [pad_idx] * padding_length

# Create dataset
class TranslationDataset(Dataset):
def __init__(self, src_sentences, tgt_sentences, src_vocab, tgt_vocab,
max_len=20):
self.src_sentences = src_sentences
self.tgt_sentences = tgt_sentences
self.src_vocab = src_vocab
self.tgt_vocab = tgt_vocab
self.max_len = max_len

def __len__(self):
return len(self.src_sentences)

def __getitem__(self, idx):

src_sentence = self.src_sentences[idx]
tgt_sentence = self.tgt_sentences[idx]
 src_indices = sentence_to_indices(src_sentence, self.src_vocab)
tgt_indices = sentence_to_indices(tgt_sentence, self.tgt_vocab)

# Pad sequences
src_padded = pad_sequence(src_indices, self.max_len,
self.src_vocab.word2index[PAD_TOKEN])
tgt_padded = pad_sequence(tgt_indices, self.max_len,
self.tgt_vocab.word2index[PAD_TOKEN])

return [Link](src_padded, dtype=[Link]),

[Link](tgt_padded, dtype=[Link])

# Create dataset and dataloader

dataset = TranslationDataset(french_sentences, english_sentences, french_vocab,
english_vocab)
dataloader = DataLoader(dataset, batch_size=2, shuffle=True)
class PositionalEncoding([Link]):
def __init__(self, d_model, max_len=100):
super(PositionalEncoding, self).__init__()
pe = [Link](max_len, d_model)
position = [Link](0, max_len, dtype=[Link]).unsqueeze(1)
div_term = [Link]([Link](0, d_model, 2).float() * (-
[Link](10000.0) / d_model))
pe[:, 0::2] = [Link](position * div_term)
pe[:, 1::2] = [Link](position * div_term)
pe = [Link](0)
self.register_buffer('pe', pe)

def forward(self, x):

return x + [Link][:, :[Link](1)]

class MultiHeadAttention([Link]):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self).__init__()
assert d_model % num_heads == 0, "d_model must be divisible by num_heads"
self.d_model = d_model
self.num_heads = num_heads
self.d_k = d_model // num_heads

self.W_q = [Link](d_model, d_model)

self.W_k = [Link](d_model, d_model)
self.W_v = [Link](d_model, d_model)
self.W_o = [Link](d_model, d_model)

def scaled_dot_product_attention(self, Q, K, V, mask=None):

attn_scores = [Link](Q, [Link](-2, -1)) / [Link](self.d_k)
if mask is not None:
attn_scores = attn_scores.masked_fill(mask == 0, -1e9)
attn_probs = [Link](attn_scores, dim=-1)
output = [Link](attn_probs, V)
return output

def split_heads(self, x):

 batch_size, seq_length, d_model = [Link]()
return [Link](batch_size, seq_length, self.num_heads,
self.d_k).transpose(1, 2)

def combine_heads(self, x):

batch_size, _, seq_length, d_k = [Link]()
return [Link](1, 2).contiguous().view(batch_size, seq_length,
self.d_model)

def forward(self, Q, K, V, mask=None):

Q = self.split_heads(self.W_q(Q))
K = self.split_heads(self.W_k(K))
V = self.split_heads(self.W_v(V))

attn_output = self.scaled_dot_product_attention(Q, K, V, mask)

output = self.W_o(self.combine_heads(attn_output))
return output

class PositionWiseFeedForward([Link]):
def __init__(self, d_model, d_ff):
super(PositionWiseFeedForward, self).__init__()
self.fc1 = [Link](d_model, d_ff)
self.fc2 = [Link](d_ff, d_model)
[Link] = [Link]()

def forward(self, x):

return self.fc2([Link](self.fc1(x)))

class EncoderLayer([Link]):
def __init__(self, d_model, num_heads, d_ff, dropout):
super(EncoderLayer, self).__init__()
self.self_attn = MultiHeadAttention(d_model, num_heads)
self.feed_forward = PositionWiseFeedForward(d_model, d_ff)
self.norm1 = [Link](d_model)
self.norm2 = [Link](d_model)
[Link] = [Link](dropout)

def forward(self, x, mask):

attn_output = self.self_attn(x, x, x, mask)
x = self.norm1(x + [Link](attn_output))
ff_output = self.feed_forward(x)
x = self.norm2(x + [Link](ff_output))
return x

class DecoderLayer([Link]):
def __init__(self, d_model, num_heads, d_ff, dropout):
super(DecoderLayer, self).__init__()
self.self_attn = MultiHeadAttention(d_model, num_heads)
self.cross_attn = MultiHeadAttention(d_model, num_heads)
self.feed_forward = PositionWiseFeedForward(d_model, d_ff)
self.norm1 = [Link](d_model)
self.norm2 = [Link](d_model)
self.norm3 = [Link](d_model)
[Link] = [Link](dropout)
 def forward(self, x, enc_output, src_mask, tgt_mask):
attn_output = self.self_attn(x, x, x, tgt_mask)
x = self.norm1(x + [Link](attn_output))
attn_output = self.cross_attn(x, enc_output, enc_output, src_mask)
x = self.norm2(x + [Link](attn_output))
ff_output = self.feed_forward(x)
x = self.norm3(x + [Link](ff_output))
return x

class Transformer([Link]):
def __init__(self, src_vocab_size, tgt_vocab_size, d_model=256, num_heads=8,
num_layers=3, d_ff=512, dropout=0.1, max_len=100):
super(Transformer, self).__init__()
self.encoder_embedding = [Link](src_vocab_size, d_model)
self.decoder_embedding = [Link](tgt_vocab_size, d_model)
self.positional_encoding = PositionalEncoding(d_model, max_len)

self.encoder_layers = [Link]([
EncoderLayer(d_model, num_heads, d_ff, dropout) for _ in
range(num_layers)
])

self.decoder_layers = [Link]([
DecoderLayer(d_model, num_heads, d_ff, dropout) for _ in
range(num_layers)
])

[Link] = [Link](d_model, tgt_vocab_size)

[Link] = [Link](dropout)

def generate_mask(self, src, tgt):

src_mask = (src != 0).unsqueeze(1).unsqueeze(2)
tgt_mask = (tgt != 0).unsqueeze(1).unsqueeze(2)
seq_length = [Link](1)
nopeak_mask = (1 - [Link]([Link](1, seq_length, seq_length),
diagonal=1)).bool()
tgt_mask = tgt_mask & nopeak_mask
return src_mask, tgt_mask

def forward(self, src, tgt):

src_mask, tgt_mask = self.generate_mask(src, tgt)

# Encoder
src_embedded =
[Link](self.positional_encoding(self.encoder_embedding(src)))
enc_output = src_embedded
for layer in self.encoder_layers:
enc_output = layer(enc_output, src_mask)

# Decoder
tgt_embedded =
[Link](self.positional_encoding(self.decoder_embedding(tgt)))
dec_output = tgt_embedded
 for layer in self.decoder_layers:
dec_output = layer(dec_output, enc_output, src_mask, tgt_mask)

output = [Link](dec_output)
return output
# Initialize model
device = [Link]("cpu")
model = Transformer(
src_vocab_size=french_vocab.n_words,
tgt_vocab_size=english_vocab.n_words,
d_model=128,
num_heads=4,
num_layers=2,
d_ff=256,
dropout=0.1
).to(device)

# Loss and optimizer

criterion = [Link](ignore_index=0) # Ignore padding index
optimizer = [Link]([Link](), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)

# Training loop
def train(model, dataloader, criterion, optimizer, num_epochs=100):
[Link]()
for epoch in range(num_epochs):
total_loss = 0
for src, tgt in dataloader:
src = [Link](device)
tgt = [Link](device)

# Shift tgt for teacher forcing

tgt_input = tgt[:, :-1]
tgt_output = tgt[:, 1:]

optimizer.zero_grad()
output = model(src, tgt_input)

# Reshape for loss calculation

loss = criterion([Link](-1, [Link](-1)),
tgt_output.reshape(-1))
[Link]()
[Link]()

total_loss += [Link]()

if (epoch + 1) % 10 == 0:
print(f"Epoch [{epoch+1}/{num_epochs}], Loss: {total_loss /
len(dataloader):.4f}")

# Train the model

train(model, dataloader, criterion, optimizer, num_epochs=100)
def translate_sentence(model, sentence, src_vocab, tgt_vocab, max_len=20,
device="cpu"):
[Link]()
 # Tokenize and numericalize the source sentence
tokens = [Link](' ')
src_indices = [src_vocab.word2index[token] for token in tokens]
src_indices = [src_vocab.word2index[SOS_TOKEN]] + src_indices +
[src_vocab.word2index[EOS_TOKEN]]
src_tensor = [Link](src_indices,
dtype=[Link]).unsqueeze(0).to(device)

# Generate source mask

src_mask = (src_tensor != 0).unsqueeze(1).unsqueeze(2)

# Encoder forward pass

with torch.no_grad():
src_embedded = model.encoder_embedding(src_tensor)
src_embedded = model.positional_encoding(src_embedded)
enc_output = src_embedded
for layer in model.encoder_layers:
enc_output = layer(enc_output, src_mask)

# Initialize target with SOS token

tgt_indices = [tgt_vocab.word2index[SOS_TOKEN]]

for i in range(max_len):
tgt_tensor = [Link](tgt_indices,
dtype=[Link]).unsqueeze(0).to(device)

# Generate target mask

tgt_mask = (tgt_tensor != 0).unsqueeze(1).unsqueeze(2)
seq_length = tgt_tensor.size(1)
nopeak_mask = (1 - [Link]([Link](1, seq_length, seq_length),
diagonal=1)).bool().to(device)
tgt_mask = tgt_mask & nopeak_mask

# Decoder forward pass

with torch.no_grad():
tgt_embedded = model.decoder_embedding(tgt_tensor)
tgt_embedded = model.positional_encoding(tgt_embedded)
dec_output = tgt_embedded
for layer in model.decoder_layers:
dec_output = layer(dec_output, enc_output, src_mask, tgt_mask)

output = [Link](dec_output)
next_token = [Link](2)[:, -1].item()
#decoded word
decoded_word = tgt_vocab.index2word[next_token]
print(f"Step {i+1}: Decoded word -> '{decoded_word}'")

# Stop if EOS token is generated

if next_token == tgt_vocab.word2index[EOS_TOKEN]:
break

tgt_indices.append(next_token)
 # Convert indices to words
translated_sentence = [tgt_vocab.index2word[idx] for idx in tgt_indices[1:]]
# Remove SOS

print("=" * 50)
print(f"Complete Translation:")
print(f"'{sentence}' -> '{' '.join(translated_sentence)}'")
print("=" * 50)

return ' '.join(translated_sentence)

# Test translation
test_sentence = "bonjour, comment ça va?"
translation = translate_sentence(model, test_sentence, french_vocab,
english_vocab, device=device)
print(f"French: {test_sentence}")
print(f"English: {translation}")

OUTPUT: