conda create -n pytorch_env python=3.
11
conda activate pytorch_env
pip install torch torchvision torchaudio
pip install ipykernel
python -m ipykernel install --user --name pytorch_env --display-name "Python PyTorch"
Aim
To implement a custom NLP Dataset and DataLoader in PyTorch for tokenizing text, building a
vocabulary, converting text into numerical sequences, and generating dynamically padded batches.
Algorithm
1. Initialize the text dataset and tokenize each sentence into lowercase words.
2. Build a vocabulary by counting word frequencies and assigning a unique index to each word,
including special tokens (<PAD> and <UNK>).
3. Convert tokenized sentences into numerical index sequences using the constructed
vocabulary.
4. Create a custom Dataset class by inheriting from [Link] and implementing
the __len__() and __getitem__() methods.
5. Define a custom collate_fn() to dynamically pad all sequences in a batch to the length of the
longest sequence using the <PAD> token.
6. Create a DataLoader with the custom dataset, specified batch size, shuffling, and the custom
padding function.
7. Iterate through the DataLoader to generate padded mini-batches and display the batch
contents and their dimensions.
import torch
from [Link] import Dataset, DataLoader
from collections import Counter
# -----------------------------
# Step 1: Raw Text Data
# -----------------------------
texts = [
"I love natural language processing",
"PyTorch is used for deep learning",
"NLP models understand human language",
"Deep learning models are powerful"
# -----------------------------
# Step 2: Tokenization Function
# -----------------------------
def tokenize(text):
return [Link]().split()
# Tokenize all sentences
tokenized_texts = [tokenize(sentence) for sentence in texts]
print("Tokenized Text:")
print(tokenized_texts)
# -----------------------------
# Step 3: Build Vocabulary
# -----------------------------
counter = Counter()
for sentence in tokenized_texts:
[Link](sentence)
# Add special tokens
vocab = {
"<PAD>": 0,
"<UNK>": 1
for word in counter:
vocab[word] = len(vocab)
print("\nVocabulary:")
print(vocab)
# -----------------------------
# Step 4: Convert Words to Numbers
# -----------------------------
def text_to_indices(tokens):
return [
[Link](word, vocab["<UNK>"])
for word in tokens
numerical_texts = [
text_to_indices(sentence)
for sentence in tokenized_texts
print("\nNumerical Representation:")
print(numerical_texts)
# -----------------------------
# Step 5: Create Custom Dataset
# -----------------------------
class TextDataset(Dataset):
def __init__(self, data):
[Link] = data
def __len__(self):
return len([Link])
def __getitem__(self, index):
return [Link]([Link][index])
dataset = TextDataset(numerical_texts)
# -----------------------------
# Step 6: Dynamic Padding Function
# -----------------------------
def collate_fn(batch):
# Find maximum sentence length in batch
max_length = max(len(sequence) for sequence in batch)
padded_batch = []
for sequence in batch:
padding_length = max_length - len(sequence)
padded_sequence = [Link](
sequence,
[Link](
padding_length,
dtype=[Link]
padded_batch.append(padded_sequence)
return [Link](padded_batch)
# -----------------------------
# Step 7: Create DataLoader
# -----------------------------
loader = DataLoader(
dataset,
batch_size=2,
shuffle=True,
collate_fn=collate_fn
# -----------------------------
# Step 8: Display Batches
# -----------------------------
for batch in loader:
print("\nBatch:")
print(batch)
print("Batch Shape:")
print([Link])