NLP Project: Text-to-Narration
Lab instructor: DR. S. KALYANI
Department: SCOPE
1. Introduction
• Text-to-Speech (TTS) is a technology that converts written text into spoken
words using artificial intelligence and speech synthesis techniques.
• It enables machines to read aloud any given text, making digital content more
accessible and interactive.
2. Objective
We aim to develop a reliable and expressive TTS model that can convert stories, novels,
and any form of writing into a profound narration with intonations and natural
expressiveness.
3. Use Cases
TTS technology is widely applicable in various fields, including:
Audiobooks & Storytelling – Creating immersive narrations for books and literature.
Assistive Technology – Enabling visually impaired users to consume written content
audibly.
AI Assistants – Powering voice-based applications like Siri, Alexa, and Google Assistant.
Language Learning – Assisting learners in pronunciation and comprehension.
Entertainment & Gaming – Enhancing gaming experiences with dynamic voiceovers.
4. Existing TTS Systems and Their Limitations
Current TTS systems, including Google TTS, Tacotron 2, and Amazon Polly, have made
significant strides in generating human-like speech. However, they still exhibit limitations
that hinder their ability to fully replicate natural speech.
7. Comparative Analysis: WhisperSpeech vs. Other TTS Models
Feature Google TTS Tacotron 2 Amazon Polly
Cloud-Based Yes Yes Yes
End-to-End No Yes No
Training
Emotion No Yes Limited
Integration
Real-Time Yes No Yes
Synthesis
[Link]
import pandas as pd
import torch
from [Link] import Dataset, DataLoader
from transformers import BertTokenizer, BertForSequenceClassification
from [Link] import MultiLabelBinarizer
from [Link] import classification_report, roc_auc_score,
multilabel_confusion_matrix
from [Link].class_weight import compute_class_weight
import [Link] as plt
import numpy as np
import time, json, pathlib, seaborn as sns
data_dir = "/kaggle/input/goemotions/data"
train_path = f"{data_dir}/[Link]"
dev_path = f"{data_dir}/[Link]"
test_path = f"{data_dir}/[Link]"
emotions_path = f"{data_dir}/[Link]"
with open(emotions_path) as f:
emotions = [[Link]() for ln in f]
train_df = pd.read_csv(train_path, sep="\t", header=None, names=["text",
"labels", "id"])
dev_df = pd.read_csv(dev_path, sep="\t", header=None, names=["text",
"labels", "id"])
test_df = pd.read_csv(test_path, sep="\t", header=None, names=["text",
"labels", "id"])
label_map = {str(i): emo for i, emo in enumerate(emotions)}
for df in (train_df, dev_df, test_df):
df["labels"] = df["labels"].astype(str).map(
lambda s: [label_map[lbl] for lbl in [Link](",")]
)
mlb = MultiLabelBinarizer(classes=emotions)
y_train = mlb.fit_transform(train_df["labels"])
y_dev = [Link](dev_df["labels"])
y_test = [Link](test_df["labels"])
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
class EmotionDataset(Dataset):
def __init__(self, texts, labels, tok, max_len=128):
[Link], [Link], [Link], self.max_len = texts, labels,
tok, max_len
def __len__(self): return len([Link])
def __getitem__(self, idx):
enc = [Link](
[Link][idx], truncation=True, padding='max_length',
max_length=self.max_len, return_tensors='pt'
)
return {
"input_ids": enc["input_ids"].squeeze(),
"attention_mask": enc["attention_mask"].squeeze(),
"labels": [Link]([Link][idx],
dtype=[Link]),
}
train_dl = DataLoader(EmotionDataset(train_df["text"], y_train,
tokenizer), batch_size=16, shuffle=True)
dev_dl =
DataLoader(EmotionDataset(dev_df["text"], y_dev, tokenizer),
batch_size=16)
test_dl =
DataLoader(EmotionDataset(test_df["text"], y_test, tokenizer),
batch_size=16)
device = [Link]("cuda" if [Link].is_available() else "cpu")
model = BertForSequenceClassification.from_pretrained(
"bert-base-uncased", num_labels=len(emotions)
)
model = [Link](model).to(device)
all_lbls = [l for sub in train_df["labels"] for l in sub]
class_wts = compute_class_weight("balanced", classes=emotions,
y=all_lbls)
class_wts_t = [Link](class_wts, dtype=[Link]).to(device)
criterion = [Link](weight=class_wts_t)
def evaluate(net, loader, thr=0.30):
[Link]()
logits, labels = [], []
with torch.no_grad():
for batch in loader:
ids = batch["input_ids"].to(device)
msk = batch["attention_mask"].to(device)
y = batch["labels"].to(device)
[Link](net(ids, attention_mask=msk).[Link]())
[Link]([Link]())
y_true = [Link](labels).numpy()
y_prob = [Link]([Link](logits)).numpy()
y_pred = (y_prob > thr).astype(int)
prf = classification_report(y_true, y_pred, target_names=emotions,
output_dict=True, zero_division=0)
return [Link](prf).T, roc_auc_score(y_true, y_prob,
average="micro"), roc_auc_score(y_true, y_prob, average="macro")
def train(net, tr_dl, dv_dl, epochs=5, lr=2e-5, thr=0.30, wd=1e-2):
opt = [Link]([Link](), lr=lr, weight_decay=wd)
train_losses, val_f1 = [], []
for ep in range(epochs):
[Link](); ep_loss, t0 = 0, [Link]()
for step, batch in enumerate(tr_dl):
opt.zero_grad()
ids = batch["input_ids"].to(device); msk =
batch["attention_mask"].to(device); y = batch["labels"].to(device)
loss = criterion(net(ids, attention_mask=msk).logits, y);
[Link](); [Link]()
ep_loss += [Link]()
if step % 100 == 0:
print(f"E{ep+1}/{epochs} step {step}/{len(tr_dl)} loss
{[Link]():.4f}")
avg = ep_loss/len(tr_dl); train_losses.append(avg)
print(f"Epoch {ep+1} done in {[Link]()-t0:.1f}s avg loss
{avg:.4f}")
prf, auc_micro, auc_macro = evaluate(net, dv_dl, thr)[0:3]
val_f1.append([Link]["macro avg","f1-score"])
print(f" → Dev macro-F1 {val_f1[-1]:.4f} micro-AUC
{auc_micro:.4f} macro-AUC {auc_macro:.4f}")
[Link](train_losses); [Link]("Train loss"); [Link]("epoch");
[Link]("loss"); plt.tight_layout(); [Link]("loss_curve.png");
[Link]()
[Link](val_f1); [Link]("Dev macro-F1");
[Link]("epoch"); [Link]("F1"); plt.tight_layout();
[Link]("f1_curve.png"); [Link]()
train(model, train_dl, dev_dl, epochs=5, lr=2e-5)
def predict(text, thr=0.30):
enc = tokenizer(text, return_tensors="pt", truncation=True,
padding="max_length", max_length=128).to(device)
with torch.no_grad():
probs = [Link](model(**enc).logits).cpu().numpy()[0]
idxs = [Link](probs > thr)[0]
return [emotions[i] for i in idxs] or ["neutral"]
if __name__ == "__main__":
sample = "Ugh, why do people never clean up after themselves? It’s so
frustrating!"
print("Predicted emotions:", predict(sample))
References
1. Google Cloud Text-to-Speech. [Link]
2. NVIDIA Tacotron 2. [Link]
3. Amazon Polly Overview. [Link]