25

APPENDIX – 1 CODING

Text Data
Importing the Lib

import os
import pandas as pd
import numpy as np
import torch
import pickle
from [Link] import to_categorical
from [Link] import Sequential
from [Link] import Dense,
Dropout,BatchNormalization
from sklearn.model_selection import train_test_split
from [Link] import LabelEncoder
from sklearn.feature_extraction.text import TfidfVectorizer,
CountVectorizer
from [Link] import classification_report
from transformers import AutoTokenizer, AutoModel
import [Link] as naw
from [Link] import indic_tokenize
from [Link].indic_normalize import
IndicNormalizerFactory
import optuna
import nltk
[Link]('punkt')
device = [Link]("cuda" if [Link].is_available() else
"cpu")
Load Tamil dataset
def load_dataset(base_dir='/content', lang='tamil'):
dataset = []
 26

text_file = [Link](base_dir, lang, "text", "[Link]")

text_df = pd.read_excel(text_file)
for _, row in text_df.iterrows():
metadata = {"class_label": row["Class Label Short"], "gender":
"Unknown"}
[Link]({
"transcript": row["Transcript"],
"class_label": metadata["class_label"]
})
return [Link](dataset)
dataset_df = load_dataset()
Preprocessing Tamil text
def preprocess_tamil_text(text):
normalizer_factory = IndicNormalizerFactory()
normalizer = normalizer_factory.get_normalizer("ta")
text = [Link](text)
tokens = list(indic_tokenize.trivial_tokenize(text, lang="ta"))
return ' '.join(tokens)
dataset_df['cleaned_transcript'] =
dataset_df['transcript'].apply(preprocess_tamil_text)

Text augmentation using nlpaug

def augment_text(text, num_augments=2):
aug = [Link](aug_src='wordnet', lang='tam')
augmented_texts = []
try:
for _ in range(num_augments):
augmented_texts.append([Link](text))
except Exception:
augmented_texts.append(text)
return augmented_texts
 27

augmented_data = []
for _, row in dataset_df.iterrows():
augmented_transcripts = augment_text(row['cleaned_transcript'],
num_augments=2)
for aug_text in augmented_transcripts:
augmented_data.append({
"transcript": aug_text,
"class_label": row['class_label']
})
augmented_df = [Link](augmented_data)
full_dataset_df = [Link]([dataset_df, augmented_df], ignore_index=True)

Encode class labels

label_encoder = LabelEncoder()
full_dataset_df['encoded_label'] =
label_encoder.fit_transform(full_dataset_df['class_label'])
label_encoder_path = "tamil_label_encoder.pkl"
with open(label_encoder_path, "wb") as f:
[Link](label_encoder, f)
Split data
X_train, X_test, y_train, y_test =
train_test_split( full_dataset_df['cleaned_transcript'],
full_dataset_df['encoded_label'], test_size=0.2, random_state=42)

def extract_embeddings(model_name, texts):

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name).to(device)
[Link]()
embeddings = []
batch_size = 16
 28

with torch.no_grad():
for i in range(0, len(texts), batch_size):
batch_texts = texts[i:i+batch_size]
encoded_inputs = tokenizer(batch_texts, padding=True, truncation=True,
max_length=128, return_tensors="pt")
encoded_inputs = {key: [Link](device) for key, tensor in
encoded_inputs.items()}
outputs = model(**encoded_inputs)
batch_embeddings =
outputs.last_hidden_state.mean(dim=1).cpu().numpy()
[Link](batch_embeddings)
return [Link](embeddings)

Extract features using different methods

vectorizers = {
"TF-IDF": TfidfVectorizer(),
"Count": CountVectorizer()}
embedding_models = {
"BERT-base-multilingual-cased": "bert-base-multilingual-cased",
"XLM-RoBERTa Base": "xlm-roberta-base",
"XLM-RoBERTa Large": "xlm-roberta-large"}
feature_sets = {}
Apply vectorization
for name, vectorizer in [Link]():
X_train_vec = vectorizer.fit_transform(X_train)
X_test_vec = [Link](X_test)
feature_sets[name] = (X_train_vec, X_test_vec)

Apply transformer models

for name, model_name in embedding_models.items():
feature_sets[name] = (
 29

extract_embeddings(model_name, X_train.tolist()),
extract_embeddings(model_name, X_test.tolist())
)

Hyperparameter tuning for XLM-RoBERTa Large

def objective(trial):
num_units = trial.suggest_int('num_units', 128, 512)
dropout_rate = trial.suggest_float('dropout_rate', 0.3, 0.6)

model = Sequential([
Dense(num_units, input_dim=feature_sets['XLM-RoBERTa Large']
[0].shape[1], activation='relu'),
BatchNormalization(),
Dropout(dropout_rate),
Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
history = [Link](feature_sets['XLM-RoBERTa Large'][0],
to_categorical(y_train), epochs=10, batch_size=32, verbose=0)
loss, accuracy = [Link](feature_sets['XLM-RoBERTa Large'][1],
to_categorical(y_test), verbose=0)
return accuracy
study = optuna.create_study(direction='maximize')
[Link](objective, n_trials=5)
print(f"Best Hyperparameters: {study.best_params}")
Evaluate each method
for method, (X_train_feat, X_test_feat) in feature_sets.items():
model = Sequential([
Dense(256, input_dim=X_train_feat.shape[1], activation='relu'),
BatchNormalization(),
 30

Dropout(0.5),
Dense(128, activation='relu'),
BatchNormalization(),
Dropout(0.5),
Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
[Link](X_train_feat, to_categorical(y_train), epochs=25, batch_size=32,
verbose=0)
y_pred = [Link](X_test_feat)
y_pred_labels = [Link](y_pred, axis=1)
print(f"Classification Report for {method}:")
print(classification_report(y_test, y_pred_labels,
target_names=label_encoder.classes_))
Load Telugu dataset
def load_dataset(base_dir='/content', lang='tel'):
dataset = []
text_file = [Link](base_dir, lang, "text", " [Link]")
text_df = pd.read_excel(text_file)
for _, row in text_df.iterrows():
metadata = {"class_label": row["Class Label Short"], "gender":
"Unknown"}
[Link]({
"transcript": row["Transcript"],
"class_label": metadata["class_label"]
})
return [Link](dataset)
dataset_df = load_dataset()
Preprocessing Tel text
def preprocess_tamil_text(text):
 31

normalizer_factory = IndicNormalizerFactory()
normalizer = normalizer_factory.get_normalizer("te")
text = [Link](text)
tokens = list(indic_tokenize.trivial_tokenize(text, lang="te"))
return ' '.join(tokens)
dataset_df['cleaned_transcript'] =
dataset_df['transcript'].apply(preprocess_tel_text)

Text augmentation using nlpaug

def augment_text(text, num_augments=2):
aug = [Link](aug_src='wordnet', lang='tam')
augmented_texts = []
try:
for _ in range(num_augments):
augmented_texts.append([Link](text))
except Exception:
augmented_texts.append(text)
return augmented_texts

augmented_data = []
for _, row in dataset_df.iterrows():
augmented_transcripts = augment_text(row['cleaned_transcript'],
num_augments=2)
for aug_text in augmented_transcripts:
augmented_data.append({
"transcript": aug_text,
"class_label": row['class_label']
})
augmented_df = [Link](augmented_data)
full_dataset_df = [Link]([dataset_df, augmented_df], ignore_index=True)
 32

Encode class labels

label_encoder = LabelEncoder()
full_dataset_df['encoded_label'] =
label_encoder.fit_transform(full_dataset_df['class_label'])
label_encoder_path = "tamil_label_encoder.pkl"
with open(label_encoder_path, "wb") as f:
[Link](label_encoder, f)
Split data
X_train, X_test, y_train, y_test =
train_test_split( full_dataset_df['cleaned_transcript'],
full_dataset_df['encoded_label'], test_size=0.2, random_state=42)
def extract_embeddings(model_name, texts):
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name).to(device)
[Link]()
embeddings = []
batch_size = 16
with torch.no_grad():
for i in range(0, len(texts), batch_size):
batch_texts = texts[i:i+batch_size]
encoded_inputs = tokenizer(batch_texts, padding=True, truncation=True,
max_length=128, return_tensors="pt")
encoded_inputs = {key: [Link](device) for key, tensor in
encoded_inputs.items()}
outputs = model(**encoded_inputs)
batch_embeddings =
outputs.last_hidden_state.mean(dim=1).cpu().numpy()
[Link](batch_embeddings)
return [Link](embeddings)

Extract features using different methods

vectorizers = {
 33

"TF-IDF": TfidfVectorizer(),
"Count": CountVectorizer()}
embedding_models = {
"BERT-base-multilingual-cased": "bert-base-multilingual-cased",
"XLM-RoBERTa Base": "xlm-roberta-base",
"XLM-RoBERTa Large": "xlm-roberta-large"}
feature_sets = {}
Apply vectorization
for name, vectorizer in [Link]():
X_train_vec = vectorizer.fit_transform(X_train)
X_test_vec = [Link](X_test)
feature_sets[name] = (X_train_vec, X_test_vec)

Apply transformer models

for name, model_name in embedding_models.items():
feature_sets[name] = (
extract_embeddings(model_name, X_train.tolist()),
extract_embeddings(model_name, X_test.tolist())
)

Hyperparameter tuning for XLM-RoBERTa Large

def objective(trial):
num_units = trial.suggest_int('num_units', 128, 512)
dropout_rate = trial.suggest_float('dropout_rate', 0.3, 0.6)

model = Sequential([
Dense(num_units, input_dim=feature_sets['XLM-RoBERTa Large']
[0].shape[1], activation='relu'),
BatchNormalization(),
Dropout(dropout_rate),
 34

Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
history = [Link](feature_sets['XLM-RoBERTa Large'][0],
to_categorical(y_train), epochs=10, batch_size=32, verbose=0)
loss, accuracy = [Link](feature_sets['XLM-RoBERTa Large'][1],
to_categorical(y_test), verbose=0)
return accuracy

study = optuna.create_study(direction='maximize')
[Link](objective, n_trials=5)
print(f"Best Hyperparameters: {study.best_params}")
Evaluate each method
for method, (X_train_feat, X_test_feat) in feature_sets.items():
model = Sequential([
Dense(256, input_dim=X_train_feat.shape[1], activation='relu'),
BatchNormalization(),
Dropout(0.5),
Dense(128, activation='relu'),
BatchNormalization(),
Dropout(0.5),
Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
[Link](X_train_feat, to_categorical(y_train), epochs=25, batch_size=32,
verbose=0)
y_pred = [Link](X_test_feat)
y_pred_labels = [Link](y_pred, axis=1)
print(f"Classification Report for {method}:")
print(classification_report(y_test, y_pred_labels,
 35

target_names=label_encoder.classes_))

Load Malayalam dataset

def load_dataset(base_dir='/content', lang='mal'):
dataset = []
text_file = [Link](base_dir, lang, "text", "[Link]")
text_df = pd.read_excel(text_file)
for _, row in text_df.iterrows():
metadata = {"class_label": row["Class Label Short"], "gender":
"Unknown"}
[Link]({
"transcript": row["Transcript"],
"class_label": metadata["class_label"]
})
return [Link](dataset)
dataset_df = load_dataset()
stopwords = [
"അവൻ", "അവൾ", "അവർ", "ആ", "ആകാം", "ആകുന്നു", "ആകും", "ആകെയുള്ള",
"ആകെയുള്ളത്", "ആകെയുള്ളവ", "ആകെയുള്ളവർ",
"ആകെയുള്ളവൻ", "ആകെയുള്ളവൾ", "ആകെയുള്ളവൾക്ക്", "‌", "ഇത്",
"ഇതിൽ", "ഇതിന്റെ", "ഇതും", "ഇതെല്ലാം",
"ഇവ", "ഇവയിൽ", "ഇവയുടെ", "ഇവയും", "ഇവയെല്ലാം", "ഇവൻ", "ഇവൾ",
"ഇവർ", "ഇവരുടെ", "ഇവരിൽ", "ഇവരെയും", "ഇവരെയെല്ലാം",
"ഇവരോട്", "ഇവരോടും", "ഇവരോടുള്ള", "ഇവരോടുള്ളത്",
"ഇവരോടുള്ളവ", "ഇവരോടുള്ളവർ", "ഇവരോടുള്ളവൻ", "ഇവരോടുള്ളവൾ",
"ഇവരോടുള്ളവൾക്ക്", "‌"
]

Preprocessing Malayalam text

def preprocess_ Malayalam_text(text):
normalizer_factory = IndicNormalizerFactory()
normalizer = normalizer_factory.get_normalizer("ml")
 36

text = [Link](text)
tokens = list(indic_tokenize.trivial_tokenize(text, lang="ml"))
tokens = [token for token in tokens if token not in stopwords]
return ' '.join(tokens)
dataset_df['cleaned_transcript'] = dataset_df['transcript'].apply(preprocess_
Malayalam_text)

Text augmentation using nlpaug

def augment_text(text, num_augments=2):
aug = [Link](aug_src='wordnet', lang='tam')
augmented_texts = []
try:
for _ in range(num_augments):
augmented_texts.append([Link](text))
except Exception:
augmented_texts.append(text)
return augmented_texts

augmented_data = []
for _, row in dataset_df.iterrows():
augmented_transcripts = augment_text(row['cleaned_transcript'],
num_augments=2)
for aug_text in augmented_transcripts:
augmented_data.append({
"transcript": aug_text,
"class_label": row['class_label']
})
augmented_df = [Link](augmented_data)
full_dataset_df = [Link]([dataset_df, augmented_df], ignore_index=True)

Encode class labels

 37

label_encoder = LabelEncoder()
full_dataset_df['encoded_label'] =
label_encoder.fit_transform(full_dataset_df['class_label'])
label_encoder_path = "tamil_label_encoder.pkl"
with open(label_encoder_path, "wb") as f:
[Link](label_encoder, f)
Split data
X_train, X_test, y_train, y_test =
train_test_split( full_dataset_df['cleaned_transcript'],
full_dataset_df['encoded_label'], test_size=0.2, random_state=42)

def extract_embeddings(model_name, texts):

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name).to(device)
[Link]()
embeddings = []
batch_size = 16
with torch.no_grad():
for i in range(0, len(texts), batch_size):
batch_texts = texts[i:i+batch_size]
encoded_inputs = tokenizer(batch_texts, padding=True, truncation=True,
max_length=128, return_tensors="pt")
encoded_inputs = {key: [Link](device) for key, tensor in
encoded_inputs.items()}
outputs = model(**encoded_inputs)
batch_embeddings =
outputs.last_hidden_state.mean(dim=1).cpu().numpy()
[Link](batch_embeddings)
return [Link](embeddings)

Extract features using different methods

vectorizers = {
 38

"TF-IDF": TfidfVectorizer(),
"Count": CountVectorizer()}
embedding_models = {
"BERT-base-multilingual-cased": "bert-base-multilingual-cased",
"XLM-RoBERTa Base": "xlm-roberta-base",
"XLM-RoBERTa Large": "xlm-roberta-large"}
feature_sets = {}
Apply vectorization
for name, vectorizer in [Link]():
X_train_vec = vectorizer.fit_transform(X_train)
X_test_vec = [Link](X_test)
feature_sets[name] = (X_train_vec, X_test_vec)

Apply transformer models

for name, model_name in embedding_models.items():
feature_sets[name] = (
extract_embeddings(model_name, X_train.tolist()),
extract_embeddings(model_name, X_test.tolist())
)

Hyperparameter tuning for XLM-RoBERTa Large

def objective(trial):
num_units = trial.suggest_int('num_units', 128, 512)
dropout_rate = trial.suggest_float('dropout_rate', 0.3, 0.6)

model = Sequential([
Dense(num_units, input_dim=feature_sets['XLM-RoBERTa Large']
[0].shape[1], activation='relu'),
BatchNormalization(),
Dropout(dropout_rate),
 39

Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
history = [Link](feature_sets['XLM-RoBERTa Large'][0],
to_categorical(y_train), epochs=10, batch_size=32, verbose=0)
loss, accuracy = [Link](feature_sets['XLM-RoBERTa Large'][1],
to_categorical(y_test), verbose=0)
return accuracy
study = optuna.create_study(direction='maximize')
[Link](objective, n_trials=5)
print(f"Best Hyperparameters: {study.best_params}")
Evaluate each method
for method, (X_train_feat, X_test_feat) in feature_sets.items():
model = Sequential([
Dense(256, input_dim=X_train_feat.shape[1], activation='relu'),
BatchNormalization(),
Dropout(0.5),
Dense(128, activation='relu'),
BatchNormalization(),
Dropout(0.5),
Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
[Link](X_train_feat, to_categorical(y_train), epochs=25, batch_size=32,
verbose=0)
y_pred = [Link](X_test_feat)
y_pred_labels = [Link](y_pred, axis=1)
print(f"Classification Report for {method}:")
print(classification_report(y_test, y_pred_labels,
target_names=label_encoder.classes_))
 40

Audio Data
Import lib
import os
import pandas as pd
import numpy as np
import librosa
import tensorflow as tf
from sklearn.model_selection import train_test_split
from [Link] import LabelEncoder
from [Link] import Sequential
from [Link] import Conv2D, MaxPooling2D, Flatten, Dense,
Dropout
from [Link] import BatchNormalization
from [Link] import ReduceLROnPlateau, EarlyStopping
Load tamil dataset
def parse_filename(file_name):
parts = file_name.split('_')
metadata = {
"class_label": parts[3],
"gender": parts[4]
}
return metadata
def load_dataset(base_dir= '../train', lang='tamil'):
dataset = []
lang_dir = [Link](base_dir, lang)
audio_dir = [Link](lang_dir, "audio")
text_dir = [Link](lang_dir, "text")
text_file = [Link](text_dir, [file for file in [Link](text_dir) if
[Link](".xlsx")][0])
text_df = pd.read_excel(text_file)
 41

for file in text_df['File Name']:

if (file + ".wav") in [Link](audio_dir):
metadata = parse_filename(file)
audio_path = [Link](audio_dir, file + ".wav")
transcript_row = text_df.loc[text_df["File Name"] == file]
if not transcript_row.empty:
transcript = transcript_row.iloc[0]["Transcript"]
class_label_short = transcript_row.iloc[0]["Class Label Short"]
[Link]({
"audio_path": audio_path,
"transcript": transcript,
"class_label": class_label_short,
"gender": metadata["gender"]
})
else:
transcript_row = text_df.loc[text_df["File Name"] == file]
if not transcript_row.empty:
transcript = transcript_row.iloc[0]["Transcript"]
class_label_short = transcript_row.iloc[0]["Class Label Short"]
[Link]({
"audio_path": "Nil",
"transcript": transcript,
"class_label": class_label_short,
"gender": "Unknown"
})
return [Link](dataset)
dataset_df = load_dataset()
data = dataset_df[dataset_df['audio_path'] != 'Nil']
data = dataset_df[dataset_df['audio_path'].apply([Link])]
data['audio_path'] = data['audio_path'].[Link]('\\', '/', regex=False)
 42

SAMPLE_RATE = 22050
DURATION = 3
MFCC_FEATURES = 40
def extract_features(file_path, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
audio, sr = [Link](file_path, sr=sr, duration=duration)
mfccs = [Link](y=audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing {file_path}: {e}")
return None
data['features'] = data['audio_path'].apply(lambda x: extract_features(x))
data = [Link](subset=['features'])
label_encoder = LabelEncoder()
data['class_label_encoded'] = label_encoder.fit_transform(data['class_label'])
Normal model
X = [Link](data['features'].tolist())
y = [Link].to_categorical(data['class_label_encoded'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=42)
X_train = X_train[..., [Link]]
X_test = X_test[..., [Link]]
X_train = np.expand_dims(X_train, axis=-1)
X_test = np.expand_dims(X_test, axis=-1)
model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
 43

MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(128, activation='relu'),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
[Link]()
history = [Link](X_train, y_train, epochs=250, batch_size=32,
validation_split=0.2)
test_loss, test_accuracy = [Link](X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
from [Link] import classification_report

y_pred = [Link](X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)

report = classification_report(y_true, y_pred_classes,

target_names=label_encoder.classes_)
print(report)
Model With TTSAudioNormalizer and DynamicAudioNormalizer
def tts_audio_normalizer(audio, target_peak=-1.0, gain_dB=0.0):
target_amplitude = 10 ** (target_peak / 20)
current_peak = [Link]([Link](audio))
if current_peak > 0:
normalization_factor = target_amplitude / current_peak
else:
normalization_factor = 1.0
 44

normalized_audio = audio * normalization_factor

gain_amplitude = 10 ** (gain_dB / 20)
normalized_audio = normalized_audio * gain_amplitude
return normalized_audio
def dynamic_audio_normalizer(audio, sr):
peak = [Link]([Link](audio))
if peak > 0:
return audio / peak
return audio
def extract_features(file_path, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
audio, sr = [Link](file_path, sr=sr, duration=duration)
normalized_audio = tts_audio_normalizer(audio, target_peak=-1.0,
gain_dB=3.0)
normalized_audio = dynamic_audio_normalizer(normalized_audio, sr)
mfccs = [Link](y=normalized_audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing {file_path}: {e}")
return None
data['processed features'] = data['audio_path'].apply(lambda x:
extract_features(x))
data = [Link](subset=['processed features'])
X = [Link](data['processed features'].tolist())
label_encoder = LabelEncoder()
data['class_label_encoded'] = label_encoder.fit_transform(data['class_label'])
y = [Link].to_categorical(data['class_label_encoded'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=42)
X_train = X_train[..., [Link], [Link]]
 45

X_test = X_test[..., [Link], [Link]]

norm_model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(256, activation='relu'),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])

norm_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
norm_model.summary()
history = norm_model.fit(X_train, y_train, epochs=80, batch_size=32,
validation_split=0.15, verbose=1, shuffle=True)
test_loss, test_accuracy = norm_model.evaluate(X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
y_pred = norm_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)
 46

Model with Data Augmentation

def augment_audio(audio, sr):
noise = [Link](len(audio))
audio_noise = audio + 0.005 * noise
audio_stretch = [Link].time_stretch(audio, rate=0.8)
audio_shift = [Link].pitch_shift(audio, sr=sr, n_steps=4)
return [audio, audio_noise, audio_stretch, audio_shift]
augmented_data = []
def extract_features(audio, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
normalized_audio = tts_audio_normalizer(audio, target_peak=-1.0,
gain_dB=3.0)
normalized_audio = dynamic_audio_normalizer(normalized_audio, sr)
mfccs = [Link](y=normalized_audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing audio: {e}")
return None
for index, row in [Link]():
audio, sr = [Link](row['audio_path'], sr=SAMPLE_RATE)
augmented_audios = augment_audio(audio, sr)
for aug_audio in augmented_audios:
features = extract_features(aug_audio, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE)
if features is not None:
augmented_data.append({
'features': features,
'class_label': row['class_label']
})
 47

augmented_df = [Link](augmented_data)
augmented_df
label_encoder = LabelEncoder()
augmented_df['class_label_encoded'] =
label_encoder.fit_transform(augmented_df['class_label'])
X_augmented = [Link](augmented_df['features'].tolist())
y_augmented =
[Link].to_categorical(augmented_df['class_label_encoded'])
X_combined = [Link]((X, X_augmented), axis=0)
y_combined = [Link]((y, y_augmented), axis=0)
X_train, X_test, y_train, y_test = train_test_split(X_combined, y_combined,
test_size=0.2, random_state=42)
X_train = X_train[..., [Link], [Link]]
X_test = X_test[..., [Link], [Link]]
aug_model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(256, activation='relu', kernel_regularizer=[Link].l2(0.01)),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
aug_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
 48

aug_model.summary()
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5,
min_lr=0.001)
early_stopping = EarlyStopping(monitor='val_loss', patience=10,
restore_best_weights=True)
history = aug_model.fit(X_train, y_train, epochs=100, batch_size=32,
validation_split=0.2, callbacks=[reduce_lr, early_stopping])
test_loss, test_accuracy = aug_model.evaluate(X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
y_pred = aug_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)
Normal model with adding BatchNormalization
imp_aug_model = Sequential([
Conv2D(64, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(256, (3, 1), activation='relu'),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(512, activation='relu'),
 49

BatchNormalization(),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
imp_aug_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
imp_aug_model.summary()
from [Link] import ReduceLROnPlateau, EarlyStopping
from [Link] import ModelCheckpoint
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5,
min_lr=0.001)
early_stopping = EarlyStopping(monitor='val_loss', patience=10,
restore_best_weights=True)
checkpoint = ModelCheckpoint('../models/malayalam_best_audio_model.keras',
monitor='val_accuracy', save_best_only=True, mode='max')
history = imp_aug_model.fit(X_train, y_train, epochs=150, batch_size=32,
validation_split=0.2, callbacks=[reduce_lr, early_stopping, checkpoint])
best_model =
[Link].load_model('../models/malayalam_best_audio_model.keras')
test_loss, test_accuracy = best_model.evaluate(X_test, y_test)
print(f"Best Model Test Accuracy: {test_accuracy * 100:.2f}%")
y_pred = best_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)

Load Malayalam dataset

def parse_filename(file_name):
parts = file_name.split('_')
metadata = {
"class_label": parts[3],
 50

"gender": parts[4]
}
return metadata
def load_dataset(base_dir= '../train', lang='malayalam'):
dataset = []
lang_dir = [Link](base_dir, lang)
audio_dir = [Link](lang_dir, "audio")
text_dir = [Link](lang_dir, "text")
text_file = [Link](text_dir, [file for file in [Link](text_dir) if
[Link](".xlsx")][0])
text_df = pd.read_excel(text_file)
for file in text_df['File Name']:
if (file + ".wav") in [Link](audio_dir):
metadata = parse_filename(file)
audio_path = [Link](audio_dir, file + ".wav")
transcript_row = text_df.loc[text_df["File Name"] == file]
if not transcript_row.empty:
transcript = transcript_row.iloc[0]["Transcript"]
class_label_short = transcript_row.iloc[0]["Class Label Short"]
[Link]({
"audio_path": audio_path,
"transcript": transcript,
"class_label": class_label_short,
"gender": metadata["gender"]
})
else:
transcript_row = text_df.loc[text_df["File Name"] == file]
if not transcript_row.empty:
transcript = transcript_row.iloc[0]["Transcript"]
class_label_short = transcript_row.iloc[0]["Class Label Short"]
 51

[Link]({
"audio_path": "Nil",
"transcript": transcript,
"class_label": class_label_short,
"gender": "Unknown"
})
return [Link](dataset)
dataset_df = load_dataset()
data = dataset_df[dataset_df['audio_path'] != 'Nil']
data = dataset_df[dataset_df['audio_path'].apply([Link])]
data['audio_path'] = data['audio_path'].[Link]('\\', '/', regex=False)
SAMPLE_RATE = 22050
DURATION = 3
MFCC_FEATURES = 40
def extract_features(file_path, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
audio, sr = [Link](file_path, sr=sr, duration=duration)
mfccs = [Link](y=audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing {file_path}: {e}")
return None
data['features'] = data['audio_path'].apply(lambda x: extract_features(x))
data = [Link](subset=['features'])
label_encoder = LabelEncoder()
data['class_label_encoded'] = label_encoder.fit_transform(data['class_label'])
Normal model
X = [Link](data['features'].tolist())
y = [Link].to_categorical(data['class_label_encoded'])
 52

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,

random_state=42)
X_train = X_train[..., [Link]]
X_test = X_test[..., [Link]]
X_train = np.expand_dims(X_train, axis=-1)
X_test = np.expand_dims(X_test, axis=-1)
model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(128, activation='relu'),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
[Link]()
history = [Link](X_train, y_train, epochs=250, batch_size=32,
validation_split=0.2)
test_loss, test_accuracy = [Link](X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
from [Link] import classification_report

y_pred = [Link](X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
 53

report = classification_report(y_true, y_pred_classes,

target_names=label_encoder.classes_)
print(report)
Model With TTSAudioNormalizer and DynamicAudioNormalizer
def tts_audio_normalizer(audio, target_peak=-1.0, gain_dB=0.0):
target_amplitude = 10 ** (target_peak / 20)
current_peak = [Link]([Link](audio))
if current_peak > 0:
normalization_factor = target_amplitude / current_peak
else:
normalization_factor = 1.0
normalized_audio = audio * normalization_factor
gain_amplitude = 10 ** (gain_dB / 20)
normalized_audio = normalized_audio * gain_amplitude
return normalized_audio
def dynamic_audio_normalizer(audio, sr):
peak = [Link]([Link](audio))
if peak > 0:
return audio / peak
return audio
def extract_features(file_path, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
audio, sr = [Link](file_path, sr=sr, duration=duration)
normalized_audio = tts_audio_normalizer(audio, target_peak=-1.0,
gain_dB=3.0)
normalized_audio = dynamic_audio_normalizer(normalized_audio, sr)
mfccs = [Link](y=normalized_audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
 54

print(f"Error processing {file_path}: {e}")

return None
data['processed features'] = data['audio_path'].apply(lambda x:
extract_features(x))
data = [Link](subset=['processed features'])
X = [Link](data['processed features'].tolist())
label_encoder = LabelEncoder()
data['class_label_encoded'] = label_encoder.fit_transform(data['class_label'])
y = [Link].to_categorical(data['class_label_encoded'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=42)
X_train = X_train[..., [Link], [Link]]
X_test = X_test[..., [Link], [Link]]
norm_model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(256, activation='relu'),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])

norm_model.compile(optimizer='adam', loss='categorical_crossentropy',
 55

metrics=['accuracy'])
norm_model.summary()
history = norm_model.fit(X_train, y_train, epochs=80, batch_size=32,
validation_split=0.15, verbose=1, shuffle=True)
test_loss, test_accuracy = norm_model.evaluate(X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
y_pred = norm_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)
Model with Data Augmentation
def augment_audio(audio, sr):
noise = [Link](len(audio))
audio_noise = audio + 0.005 * noise
audio_stretch = [Link].time_stretch(audio, rate=0.8)
audio_shift = [Link].pitch_shift(audio, sr=sr, n_steps=4)
return [audio, audio_noise, audio_stretch, audio_shift]
augmented_data = []
def extract_features(audio, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
normalized_audio = tts_audio_normalizer(audio, target_peak=-1.0,
gain_dB=3.0)
normalized_audio = dynamic_audio_normalizer(normalized_audio, sr)
mfccs = [Link](y=normalized_audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing audio: {e}")
return None
 56

for index, row in [Link]():

audio, sr = [Link](row['audio_path'], sr=SAMPLE_RATE)
augmented_audios = augment_audio(audio, sr)
for aug_audio in augmented_audios:
features = extract_features(aug_audio, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE)
if features is not None:
augmented_data.append({
'features': features,
'class_label': row['class_label']
})

augmented_df = [Link](augmented_data)
augmented_df
label_encoder = LabelEncoder()
augmented_df['class_label_encoded'] =
label_encoder.fit_transform(augmented_df['class_label'])
X_augmented = [Link](augmented_df['features'].tolist())
y_augmented =
[Link].to_categorical(augmented_df['class_label_encoded'])
X_combined = [Link]((X, X_augmented), axis=0)
y_combined = [Link]((y, y_augmented), axis=0)
X_train, X_test, y_train, y_test = train_test_split(X_combined, y_combined,
test_size=0.2, random_state=42)
X_train = X_train[..., [Link], [Link]]
X_test = X_test[..., [Link], [Link]]
aug_model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
 57

MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(256, activation='relu', kernel_regularizer=[Link].l2(0.01)),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
aug_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
aug_model.summary()
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5,
min_lr=0.001)
early_stopping = EarlyStopping(monitor='val_loss', patience=10,
restore_best_weights=True)
history = aug_model.fit(X_train, y_train, epochs=100, batch_size=32,
validation_split=0.2, callbacks=[reduce_lr, early_stopping])
test_loss, test_accuracy = aug_model.evaluate(X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
y_pred = aug_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)
Normal model with adding BatchNormalization
imp_aug_model = Sequential([
Conv2D(64, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
BatchNormalization(),
 58

MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(256, (3, 1), activation='relu'),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(512, activation='relu'),
BatchNormalization(),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
imp_aug_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
imp_aug_model.summary()
from [Link] import ReduceLROnPlateau, EarlyStopping
from [Link] import ModelCheckpoint
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5,
min_lr=0.001)
early_stopping = EarlyStopping(monitor='val_loss', patience=10,
restore_best_weights=True)
checkpoint = ModelCheckpoint('../models/malayalam_best_audio_model.keras',
monitor='val_accuracy', save_best_only=True, mode='max')
history = imp_aug_model.fit(X_train, y_train, epochs=150, batch_size=32,
validation_split=0.2, callbacks=[reduce_lr, early_stopping, checkpoint])
best_model =
[Link].load_model('../models/malayalam_best_audio_model.keras')
test_loss, test_accuracy = best_model.evaluate(X_test, y_test)
 59

print(f"Best Model Test Accuracy: {test_accuracy * 100:.2f}%")

y_pred = best_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)

Load telegu dataset

def parse_filename(file_name):
parts = file_name.split('_')
metadata = {
"class_label": parts[3],
"gender": parts[4]
}
return metadata
def load_dataset(base_dir= '../train', lang='telegu'):
dataset = []
lang_dir = [Link](base_dir, lang)
audio_dir = [Link](lang_dir, "audio")
text_dir = [Link](lang_dir, "text")
text_file = [Link](text_dir, [file for file in [Link](text_dir) if
[Link](".xlsx")][0])
text_df = pd.read_excel(text_file)
for file in text_df['File Name']:
if (file + ".wav") in [Link](audio_dir):
metadata = parse_filename(file)
audio_path = [Link](audio_dir, file + ".wav")
transcript_row = text_df.loc[text_df["File Name"] == file]
if not transcript_row.empty:
transcript = transcript_row.iloc[0]["Transcript"]
 60

class_label_short = transcript_row.iloc[0]["Class Label Short"]

[Link]({
"audio_path": audio_path,
"transcript": transcript,
"class_label": class_label_short,
"gender": metadata["gender"]
})
else:
transcript_row = text_df.loc[text_df["File Name"] == file]
if not transcript_row.empty:
transcript = transcript_row.iloc[0]["Transcript"]
class_label_short = transcript_row.iloc[0]["Class Label Short"]
[Link]({
"audio_path": "Nil",
"transcript": transcript,
"class_label": class_label_short,
"gender": "Unknown"
})
return [Link](dataset)
dataset_df = load_dataset()
data = dataset_df[dataset_df['audio_path'] != 'Nil']
data = dataset_df[dataset_df['audio_path'].apply([Link])]
data['audio_path'] = data['audio_path'].[Link]('\\', '/', regex=False)
SAMPLE_RATE = 22050
DURATION = 3
MFCC_FEATURES = 40
def extract_features(file_path, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
audio, sr = [Link](file_path, sr=sr, duration=duration)
 61

mfccs = [Link](y=audio, sr=sr, n_mfcc=n_mfcc)

return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing {file_path}: {e}")
return None
data['features'] = data['audio_path'].apply(lambda x: extract_features(x))
data = [Link](subset=['features'])
label_encoder = LabelEncoder()
data['class_label_encoded'] = label_encoder.fit_transform(data['class_label'])
Normal model
X = [Link](data['features'].tolist())
y = [Link].to_categorical(data['class_label_encoded'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=42)
X_train = X_train[..., [Link]]
X_test = X_test[..., [Link]]
X_train = np.expand_dims(X_train, axis=-1)
X_test = np.expand_dims(X_test, axis=-1)
model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(128, activation='relu'),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
 62

[Link](optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
[Link]()
history = [Link](X_train, y_train, epochs=250, batch_size=32,
validation_split=0.2)
test_loss, test_accuracy = [Link](X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
from [Link] import classification_report

y_pred = [Link](X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)

report = classification_report(y_true, y_pred_classes,

target_names=label_encoder.classes_)
print(report)
Model With TTSAudioNormalizer and DynamicAudioNormalizer
def tts_audio_normalizer(audio, target_peak=-1.0, gain_dB=0.0):
target_amplitude = 10 ** (target_peak / 20)
current_peak = [Link]([Link](audio))
if current_peak > 0:
normalization_factor = target_amplitude / current_peak
else:
normalization_factor = 1.0
normalized_audio = audio * normalization_factor
gain_amplitude = 10 ** (gain_dB / 20)
normalized_audio = normalized_audio * gain_amplitude
return normalized_audio
def dynamic_audio_normalizer(audio, sr):
peak = [Link]([Link](audio))
if peak > 0:
 63

return audio / peak

return audio
def extract_features(file_path, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE):
try:
audio, sr = [Link](file_path, sr=sr, duration=duration)
normalized_audio = tts_audio_normalizer(audio, target_peak=-1.0,
gain_dB=3.0)
normalized_audio = dynamic_audio_normalizer(normalized_audio, sr)
mfccs = [Link](y=normalized_audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing {file_path}: {e}")
return None
data['processed features'] = data['audio_path'].apply(lambda x:
extract_features(x))
data = [Link](subset=['processed features'])
X = [Link](data['processed features'].tolist())
label_encoder = LabelEncoder()
data['class_label_encoded'] = label_encoder.fit_transform(data['class_label'])
y = [Link].to_categorical(data['class_label_encoded'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=42)
X_train = X_train[..., [Link], [Link]]
X_test = X_test[..., [Link], [Link]]
norm_model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
 64

Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(256, activation='relu'),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])

norm_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
norm_model.summary()
history = norm_model.fit(X_train, y_train, epochs=80, batch_size=32,
validation_split=0.15, verbose=1, shuffle=True)
test_loss, test_accuracy = norm_model.evaluate(X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
y_pred = norm_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)
Model with Data Augmentation
def augment_audio(audio, sr):
noise = [Link](len(audio))
audio_noise = audio + 0.005 * noise
audio_stretch = [Link].time_stretch(audio, rate=0.8)
audio_shift = [Link].pitch_shift(audio, sr=sr, n_steps=4)
return [audio, audio_noise, audio_stretch, audio_shift]
augmented_data = []
 65

def extract_features(audio, n_mfcc=MFCC_FEATURES,

duration=DURATION, sr=SAMPLE_RATE):
try:
normalized_audio = tts_audio_normalizer(audio, target_peak=-1.0,
gain_dB=3.0)
normalized_audio = dynamic_audio_normalizer(normalized_audio, sr)
mfccs = [Link](y=normalized_audio, sr=sr, n_mfcc=n_mfcc)
return [Link](mfccs.T, axis=0)
except Exception as e:
print(f"Error processing audio: {e}")
return None
for index, row in [Link]():
audio, sr = [Link](row['audio_path'], sr=SAMPLE_RATE)
augmented_audios = augment_audio(audio, sr)
for aug_audio in augmented_audios:
features = extract_features(aug_audio, n_mfcc=MFCC_FEATURES,
duration=DURATION, sr=SAMPLE_RATE)
if features is not None:
augmented_data.append({
'features': features,
'class_label': row['class_label']
})

augmented_df = [Link](augmented_data)
augmented_df
label_encoder = LabelEncoder()
augmented_df['class_label_encoded'] =
label_encoder.fit_transform(augmented_df['class_label'])
X_augmented = [Link](augmented_df['features'].tolist())
y_augmented =
[Link].to_categorical(augmented_df['class_label_encoded'])
X_combined = [Link]((X, X_augmented), axis=0)
 66

y_combined = [Link]((y, y_augmented), axis=0)

X_train, X_test, y_train, y_test = train_test_split(X_combined, y_combined,
test_size=0.2, random_state=42)
X_train = X_train[..., [Link], [Link]]
X_test = X_test[..., [Link], [Link]]
aug_model = Sequential([
Conv2D(32, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(64, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(256, activation='relu', kernel_regularizer=[Link].l2(0.01)),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
aug_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
aug_model.summary()
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5,
min_lr=0.001)
early_stopping = EarlyStopping(monitor='val_loss', patience=10,
restore_best_weights=True)
history = aug_model.fit(X_train, y_train, epochs=100, batch_size=32,
validation_split=0.2, callbacks=[reduce_lr, early_stopping])
test_loss, test_accuracy = aug_model.evaluate(X_test, y_test)
print(f"Test Accuracy: {test_accuracy * 100:.2f}%")
 67

y_pred = aug_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)
Normal model with adding BatchNormalization
imp_aug_model = Sequential([
Conv2D(64, (3, 1), activation='relu', input_shape=(MFCC_FEATURES, 1,
1)),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(128, (3, 1), activation='relu'),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Conv2D(256, (3, 1), activation='relu'),
BatchNormalization(),
MaxPooling2D((2, 1)),
Dropout(0.3),
Flatten(),
Dense(512, activation='relu'),
BatchNormalization(),
Dropout(0.4),
Dense(len(label_encoder.classes_), activation='softmax')
])
imp_aug_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
imp_aug_model.summary()
from [Link] import ReduceLROnPlateau, EarlyStopping
 68

from [Link] import ModelCheckpoint

reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5,
min_lr=0.001)
early_stopping = EarlyStopping(monitor='val_loss', patience=10,
restore_best_weights=True)
checkpoint = ModelCheckpoint('../models/malayalam_best_audio_model.keras',
monitor='val_accuracy', save_best_only=True, mode='max')
history = imp_aug_model.fit(X_train, y_train, epochs=150, batch_size=32,
validation_split=0.2, callbacks=[reduce_lr, early_stopping, checkpoint])
best_model =
[Link].load_model('../models/malayalam_best_audio_model.keras')
test_loss, test_accuracy = best_model.evaluate(X_test, y_test)
print(f"Best Model Test Accuracy: {test_accuracy * 100:.2f}%")
y_pred = best_model.predict(X_test)
y_pred_classes = [Link](y_pred, axis=1)
y_true = [Link](y_test, axis=1)
report = classification_report(y_true, y_pred_classes,
target_names=label_encoder.classes_)
print(report)

APPENDIX – 2 SCREENSHOTS
 69

Fig A2.1: Classification report of bert-base-multilingual-cased in

tamil

Fig A2.2 Classification report of bert-base-multilingual-cased in

Telugu

Fig A2.3: Classification report of bert-base-multilingual-cased in

Malayalam
 70

Fig A2.4: Classification report of TFIDF Vectorizer in Tamil

Fig A2.5: Classification report of TFIDF Vectorizer in Telugu

 71

Fig A2.6: Classification report of TFIDF Vectorizer in Malayalam

Fig A2.7: Classification report of COUNT Vectorizer in Tamil

Fig A2.8: Classification report of COUNT Vectorizer in

Telugu
 72

Fig A2.9: Classification report of COUNT Vectorizer in Malayalam

Fig A2.10: Classification report of xlm-roberta base in Tamil

Fig A2.11: Classification report of xlm-roberta-base in Telugu

Fig A2.12: Classification report of xlm roberta-base in Malayalam

 73

Fig A2.13: Classification report of xlm-roberta-large in Tamil

Fig A2.14: Classification report of xlm-roberta-large in Telugu

Fig A2.15: Classification report of xlm-roberta-large in Malayalam

 74

Fig A2.16: Classification report of Without Preprocessing in Tamil

Fig A2.17: Classification report Without Preprocessing in Telugu

Fig A2.18: Classification report of Without Preprocessing in

Malayalam
 75

Fig A2.19: Classification report of With TTSAudioNormalizer and

Dynamic AudioNormalizer in Tamil

Fig A2.20: Classification report of With TTSAudioNormalizer and

Dynamic AudioNormalizer in Telugu

Fig A2.21: Classification report of With TTSAudioNormalizer and

 76

Dynamic AudioNormalizer in Malayalam

Fig A2.22: Classification report of with Data Augmentation in Tamil

Fig A2.23: Classification report with Data Augmentation in Telugu

Fig A2.24: Classification report of with Data Augmentation in

Malayalam
 77

Fig A2.25: Classification report of with Data Augmentation and

BatchNormalization in Tamil

Fig A2.26: Classification report with Data Augmentation and

BatchNormalization in Telugu

Fig A2.27: Classification report of with Data Augmentation and

 78

BatchNormalization in Malayalam

REFERENCES

[1]. Alkaff, M., Miqdad, M. A., Fachrurrazi, M., Abdi, M. N., Abidin, A. Z., & Amalia,
R. (2023). Hate Speech Detection for Banjarese Languages on Instagram Using
Machine Learning Methods. MATRIK: Jurnal Manajemen, Teknik Informatika dan
Rekayasa Komputer, 22(3), 495-504.

[2]. An, J., Lee, W., Jeon, Y., Ok, J., Kim, Y., & Lee, G. G. (2024). An investigation into
explainable audio hate speech detection. arXiv preprint arXiv:2408.06065.

[3]. Ava, L. T., Karim, A., Hassan, M. M., Faisal, F., Azam, S., Al Haque, A. F., &
Zaman, S. (2023). Intelligent Identification of Hate Speeches to address the
increased rate of Individual Mental Degeneration. Procedia Computer Science, 219,
1527-1537.

[4]. Barman, S., & Das, M. (2023, September). hate-alert@ dravidianlangtech:

Multimodal abusive language detection and sentiment analysis in Dravidian
languages. In Proceedings of the Third Workshop on Speech and Language
Technologies for Dravidian Languages (pp. 217-224).

[5]. Dwivedy, V., & Roy, P. K. (2023). Deep feature fusion for hate speech detection: a
transfer learning approach. Multimedia Tools and Applications, 82(23), 36279-
36301.

[6]. Hee, M. S., Sharma, S., Cao, R., Nandi, P., Chakraborty, T., & Lee, R. K. W.
(2024). Recent advances in hate speech moderation: Multimodality and the role of
large models. arXiv preprint arXiv:2401.16727.

[7]. Imbwaga, J. L., Chittaragi, N. B., & Koolagudi, S. G. (2024). Automatic hate
speech detection in audio using machine learning algorithms. International Journal
of Speech Technology, 27(2), 447-469.
 79

[8]. Koreddi, V., Manisha, N., Kaif, S. M., & Kumar, Y. T. S. (2025). Multilingual AI
system for detecting offensive content across text, audio, and visual media.
Engineering Research Express, 7(1), 015216.

[9]. Manasa, G., Akshay, A., Raj, B. S. G. S., Sripriya, C., & Tejaswini, D. (2024).
AUDIO BASED HATESPEECH CLASSIFICATION FROM ONLINE SHORT-
FORM VIDEOS. International Journal of Information Technology and Computer
Engineering, 12(1), 514-521.

[10]. Narula, R., & Chaudhary, P. (2024). A comprehensive review on detection of hate
speech for multi-lingual data. Social Network Analysis and Mining, 14(1), 1-35.

[11]. Premjith, B., Chakravarthi, B. R., Kumaresan, P. K., Rajiakodi, S., Karnati, S.,
Mangamuru, S., & Janakiram, C. (2024, March). Findings of the Shared Task on
Hate and Offensive Language Detection in Telugu Codemixed Text (HOLD-
Telugu)@ DravidianLangTech 2024. In Proceedings of the Fourth Workshop on
Speech, Vision, and Language Technologies for Dravidian Languages (pp. 49-55).

[12]. Premjith, B., Jyothish, G., Sowmya, V., Chakravarthi, B. R., Nandhini, K.,
Natarajan, R., ... & Reddy, M. (2024, March). Findings of the Shared Task on
Multimodal Social Media Data Analysis in Dravidian Languages (MSMDA-
DL)@ DravidianLangTech 2024. In Proceedings of the Fourth Workshop on
Speech, Vision, and Language Technologies for Dravidian Languages (pp. 56-61).

[13]. Premjith, B., Sowmya, V., Chakravarthi, B. R., Natarajan, R., Nandhini, K.,
Murugappan, A., ... & Sn, P. (2023, September). Findings of the shared task on
multimodal abusive language detection and sentiment analysis in Tamil and
Malayalam. In Proceedings of the Third Workshop on Speech and Language
Technologies for Dravidian Languages (pp. 72-79).

[14]. Rana, A., & Jha, S. (2022). Emotion based hate speech detection using
multimodal learning. arXiv preprint arXiv:2202.06218.
 80

[15]. Sahu, G., Cohen, R., & Vechtomova, O. (2021). Towards a multi-agent system for
online hate speech detection. arXiv preprint arXiv:2105.01129.

[16]. Shimi, G., Mahibha, C. J., & Thenmozhi, D. (2024). An Empirical Analysis of
Language Detection in Dravidian Languages. Indian Journal of Science and
Technology, 17(15), 1515-1526.

[17]. Sreelakshmi, K., Premjith, B., Chakravarthi, B. R., & Soman, K. P. (2024).
Detection of hate speech and offensive language codemix text in dravidian
languages using cost-sensitive learning approach. IEEE Access, 12, 20064-20090.

[18]. Tembe, L. A., & Anand Kumar, M. (2023, December). Hate Speech Detection
Using Audio in Portuguese Language. In International Conference on Speech and
Language Technologies for Low-resource Languages (pp. 359-367). Cham:
Springer Nature Switzerland.

[19]. Thapa, S., Shah, A., Jafri, F. A., Naseem, U., & Razzak, I. (2022). A multi-modal
dataset for hate speech detection on social media: Case-study of Russia-Ukraine
conflict. In CASE 2022-5th Workshop on Challenges and Applications of
Automated Extraction of Socio-Political Events from Text, Proceedings of the
Workshop. Association for Computational Linguistics.

[20]. Vrysis, L., Vryzas, N., Kotsakis, R., Saridou, T., Matsiola, M., Veglis, A., ... &
Dimoulas, C. (2021). A web interface for analyzing hate speech. Future Internet,
13(3), 80.
81