Principal Component Analysis (PCA)
Implementation from Scratch
import numpy as np
import pandas as pd
import [Link] as plt
# Set the display precision to 2 decimal places
np.set_printoptions(precision=3)
What is PCA?
PCA is a dimensionality reduction technique that transforms a dataset into a new coordinate
system where the greatest variance lies on the first principal component (PC), the second
greatest variance on the second PC, and so on.
PCA Steps
1. Standardize the Data: Ensure all features have a mean of 0 and a standard deviation of 1.
2. Compute the Covariance Matrix: Understand the relationship between features.
3. Calculate Eigenvalues and Eigenvectors: Identify the principal components.
4. Project the Data: Transform the dataset to the new coordinate system.
# Generate Dataset
# Generate Sample Data
def generate_data():
[Link](42)
X = [Link](100, 6) # 100 samples, 3 features
X[:, 2] = X[:, 0] + X[:, 1] # Add correlation
X[:, 1] = X[:, 3] + X[:, 5] # Add correlation
return [Link](X, columns=['Feature1', 'Feature2',
'Feature3', 'Feature4', 'Feature5', 'Feature6'])
Load Dataset
data = generate_data()
# Implement PCA from Scratch using numpy
[ Step1 ]: Standardize Dataset
data_mean = [Link](data, axis=0)
data_mean
Feature1 0.506084
Feature2 0.965149
Feature3 1.070996
Feature4 0.505380
Feature5 0.476977
Feature6 0.459768
dtype: float64
data_std = [Link](data, axis=0)
data_std
Feature1 0.297080
Feature2 0.431274
Feature3 0.422078
Feature4 0.284933
Feature5 0.305046
Feature6 0.296602
dtype: float64
data_standardized = (data - data_mean) / data_std
data_standardized
{"summary":"{\n \"name\": \"data_standardized\",\n \"rows\": 100,\n
\"fields\": [\n {\n \"column\": \"Feature1\",\n
\"properties\": {\n \"dtype\": \"number\",\n \"std\":
1.005037815259212,\n \"min\": -1.684941063799225,\n
\"max\": 1.630609792950837,\n \"num_unique_values\": 100,\n
\"samples\": [\n 1.576380804779218,\n -
1.1769464100255942,\n 0.9610047551915758\n ],\n
\"semantic_type\": \"\",\n \"description\": \"\"\n }\
n },\n {\n \"column\": \"Feature2\",\n \"properties\":
{\n \"dtype\": \"number\",\n \"std\":
1.005037815259212,\n \"min\": -2.034820491977912,\n
\"max\": 2.077271372832893,\n \"num_unique_values\": 100,\n
\"samples\": [\n 0.8921073976401365,\n
0.06812666887483403,\n 0.18271589348088974\n ],\n
\"semantic_type\": \"\",\n \"description\": \"\"\n }\
n },\n {\n \"column\": \"Feature3\",\n \"properties\":
{\n \"dtype\": \"number\",\n \"std\":
1.005037815259212,\n \"min\": -1.8741745417579108,\n
\"max\": 2.107690972053611,\n \"num_unique_values\": 100,\n
\"samples\": [\n 2.107690972053611,\n -
1.5739184998766362,\n 1.208784729406462\n ],\n
\"semantic_type\": \"\",\n \"description\": \"\"\n }\
n },\n {\n \"column\": \"Feature4\",\n \"properties\":
{\n \"dtype\": \"number\",\n \"std\":
1.005037815259212,\n \"min\": -1.7154660857325439,\n
\"max\": 1.7349282305550595,\n \"num_unique_values\": 100,\n
\"samples\": [\n 0.10780203602201602,\n
0.7342644084591154,\n -0.038464346835154366\n ],\n
\"semantic_type\": \"\",\n \"description\": \"\"\n }\
n },\n {\n \"column\": \"Feature5\",\n \"properties\":
{\n \"dtype\": \"number\",\n \"std\":
1.005037815259212,\n \"min\": -1.547033995260417,\n
\"max\": 1.703546676509282,\n \"num_unique_values\": 100,\n
\"samples\": [\n -0.5489338540571447,\n
0.6006312865834622,\n -1.3749378224015223\n ],\n
\"semantic_type\": \"\",\n \"description\": \"\"\n }\
n },\n {\n \"column\": \"Feature6\",\n \"properties\":
{\n \"dtype\": \"number\",\n \"std\":
1.005037815259212,\n \"min\": -1.5135825140945722,\n
\"max\": 1.7540844275442191,\n \"num_unique_values\": 100,\n
\"samples\": [\n 1.1936100054433552,\n -
0.6063172325401689,\n 0.3026295641391838\n ],\n
\"semantic_type\": \"\",\n \"description\": \"\"\n }\
n }\n ]\
n}","type":"dataframe","variable_name":"data_standardized"}
[ Step 2 ]: Compute Covariance Matrix
cov_matrix = [Link](data_standardized, rowvar=False)
print(cov_matrix)
[[ 1.01 0.05 0.741 -0.01 -0.165 0.083]
[ 0.05 1.01 -0.029 0.737 0.106 0.761]
[ 0.741 -0.029 1.01 -0.077 -0.155 0.032]
[-0.01 0.737 -0.077 1.01 0.114 0.101]
[-0.165 0.106 -0.155 0.114 1.01 0.045]
[ 0.083 0.761 0.032 0.101 0.045 1.01 ]]
[ Step 3 ]: Calculate Eigenvalues and Eigenvectors
# Calculate Eigenvalues and Eigenvectors
eigenvalues, eigenvectors = [Link](cov_matrix)
print("Eigenvalues: \n")
print(eigenvalues)
print("Eigenvectors: \n")
print(eigenvectors)
Eigenvalues:
[ 1.827e+00 2.145e+00 2.648e-01 9.377e-01 8.859e-01 -6.802e-16]
Eigenvectors:
[[-6.787e-01 9.285e-03 7.081e-01 1.946e-01 -6.053e-03 -5.038e-16]
[-4.566e-02 6.824e-01 -3.120e-02 -7.942e-02 -3.535e-02 -7.237e-01]
[-6.733e-01 -5.042e-02 -7.043e-01 2.175e-01 2.942e-02 4.004e-16]
[ 5.292e-02 5.050e-01 -3.362e-02 2.622e-01 -6.661e-01 4.781e-01]
[ 2.595e-01 1.396e-01 2.018e-02 8.396e-01 4.559e-01 -1.067e-17]
[-1.172e-01 5.071e-01 -1.307e-02 -3.673e-01 5.885e-01 4.977e-01]]
[ Step 4 ]: Select Principal Components (2 PCs)
# To sort eigenvalues and eigenvectors in ascending order
sorted_indices = [Link](eigenvalues)
# But we need to sort eigenvalues and eigenvectors in descending order
sorted_indices = [Link](eigenvalues)[::-1]
# Apply the sort
eigenvalues = eigenvalues[sorted_indices]
eigenvectors = eigenvectors[:, sorted_indices]
print("Eigenvalues: \n")
print(eigenvalues)
print("Eigenvectors: \n")
print(eigenvectors)
Eigenvalues:
[ 2.145e+00 1.827e+00 9.377e-01 8.859e-01 2.648e-01 -6.802e-16]
Eigenvectors:
[[ 9.285e-03 -6.787e-01 1.946e-01 -6.053e-03 7.081e-01 -5.038e-16]
[ 6.824e-01 -4.566e-02 -7.942e-02 -3.535e-02 -3.120e-02 -7.237e-01]
[-5.042e-02 -6.733e-01 2.175e-01 2.942e-02 -7.043e-01 4.004e-16]
[ 5.050e-01 5.292e-02 2.622e-01 -6.661e-01 -3.362e-02 4.781e-01]
[ 1.396e-01 2.595e-01 8.396e-01 4.559e-01 2.018e-02 -1.067e-17]
[ 5.071e-01 -1.172e-01 -3.673e-01 5.885e-01 -1.307e-02 4.977e-01]]
n_components = 2
eigenvectors_reduced = eigenvectors[:, :n_components]
eigenvectors_reduced
array([[ 0.009, -0.679],
[ 0.682, -0.046],
[-0.05 , -0.673],
[ 0.505, 0.053],
[ 0.14 , 0.26 ],
[ 0.507, -0.117]])
explained_variance_ratio = eigenvalues[:n_components] /
[Link](eigenvalues)
explained_variance_ratio
array([0.354, 0.301])
[ Step 5 ]: Project Data
projected_data = [Link](data_standardized, eigenvectors_reduced)
pca_df = [Link](projected_data, columns=[f'PC{i+1}' for i in
range(n_components)])
## Apply PCA using SKlearn
from [Link] import StandardScaler
from [Link] import PCA
[ step 1]: Load and Standardize Data
data = generate_data()
scaler = StandardScaler()
data_scaled = scaler.fit_transform(data)
[ Step 2 ]: Apply PCA
pca = PCA(n_components=2) # Reduce to 2 principal components
principal_components = pca.fit_transform(data_scaled)
pca_df = [Link](principal_components, columns=['PC1', 'PC2'])
pca_df
# Explained Variance
explained_variance = pca.explained_variance_ratio_