MACHINE LEARNING – MODULE 3

Complete Teacher-Style Mind Map / Cheat Sheet

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

SIMILARITY-BASED LEARNING

Definition

Similarity-based learning predicts output using similarity between data points.

Main Idea:

✔ Similar data → Similar output

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

INSTANCE-BASED vs MODEL-BASED LEARNING

Instance-Based Learning Model-Based Learning

Lazy learner Eager learner

Stores training data Builds model

Slow testing Fast testing

No generalization initially Learns generalized model

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

K-NEAREST NEIGHBOR (KNN)

Definition

KNN classifies data using nearest neighboring data points.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

KNN Steps

1. Select value of K

2. Calculate distance

3. Sort distances
 4. Select nearest K neighbors

5. Majority voting

6. Predict class

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

DISTANCE METRICS

Euclidean Distance

Used for continuous data.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Hamming Distance

Used for categorical/binary attributes.

Rule:

✔ Same value → 0
✔ Different value → 1

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

FACTORS AFFECTING KNN

Choice of K

Small K

✔ Low bias
High variance
Overfitting

Large K

✔ Low variance
High bias
Underfitting

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━
 Feature Scaling

Why Important?

Large values dominate distance calculation.

Solution:

✔ Normalization
✔ Standardization

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Curse of Dimensionality

Problem:

High-dimensional data reduces KNN performance.

Effects:

Slow computation
Poor accuracy

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Outliers & Noise

Problem:

Nearby outliers may misclassify data.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Large Dataset Size

Advantage:

✔ Better representation

Disadvantage:

Slow prediction

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

WEIGHTED K-NEAREST NEIGHBOR

Improved KNN where closer neighbors get higher importance.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Weighted KNN Process

1. Compute distances

2. Select nearest K neighbors

3. Compute inverse distances

4. Compute weights

5. Weighted voting

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Weight Formula

Closer neighbors → Larger weights

w_i=\frac{1/d_i}{\sum (1/d_i)}

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

NEAREST CENTROID CLASSIFIER

Definition

Classifies based on nearest class centroid (mean).

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Steps

1. Compute centroid of each class

2. Compute distance from test instance

3. Select nearest centroid

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Centroid Formula
Centroid=\left(\frac{\sum x}{n},\frac{\sum y}{n}\right)

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

LOCALLY WEIGHTED REGRESSION (LWR)

Definition

A regression model giving higher importance to nearby points.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Features

✔ Non-parametric
✔ Memory-based
✔ Local approximation

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Linear Model

h_\beta(x)=\beta_0+\beta_1x

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Cost Function

J(\beta)=\frac{1}{2}\sum w_i(h_\beta(x_i)-y_i)^2

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Gaussian Weight Function

w_i=e^{-\frac{(x_i-x)^2}{2\tau^2}}

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

REGRESSION ANALYSIS

Definition

Regression predicts continuous values.

✔ Price prediction
✔ Revenue prediction
✔ Crop yield prediction

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

PURPOSE OF REGRESSION

✔ Relationship analysis
✔ Strength of relationship
✔ Linear/non-linear analysis
✔ Attribute contribution

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

CORRELATION vs CAUSATION

Correlation

Variables move together.

Causation

One variable directly affects another.

Important:

Correlation ≠ Causation

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

LINEAR & NON-LINEAR RELATIONSHIPS

Linear Relationship

Straight line relation.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Non-Linear Relationship

Examples:
✔ Exponential
✔ Polynomial
✔ Power functions

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

TYPES OF REGRESSION

Linear Regression

Single independent variable.

Multiple Linear Regression

Multiple independent variables.

Polynomial Regression

Handles curved relationships.

Logistic Regression

Binary classification.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

LIMITATIONS OF REGRESSION

Outliers affect model

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

LINEAR REGRESSION

Equation

y=a_0+a_1x+e

Terms:

✔ a₀ → Intercept
✔ a₁ → Slope
✔ e → Error
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

ORDINARY LEAST SQUARES (OLS)

Goal

Find line minimizing prediction error.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Error Formula

e_i=y_i-(a_0+a_1x_i)

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Cost Function

J(a_0,a_1)=\sum_{i=1}^{n}[y_i-(a_0+a_1x_i)]^2

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Regression Coefficients

Slope

a_1=\frac{\overline{xy}-(\overline{x})(\overline{y})}{\overline{x^2}-(\overline{x})^2}

Intercept

a_0=\overline{y}-a_1\overline{x}

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

LINEAR REGRESSION IN MATRIX FORM

Matrix Equation

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

MULTIPLE LINEAR REGRESSION

Equation

y=a_0+a_1x_1+a_2x_2+\cdots+a_nx_n+\epsilon
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Assumptions

✔ No multicollinearity
✔ Normally distributed residuals

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

POLYNOMIAL REGRESSION

Purpose

Handles non-linear relationships.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Quadratic Equation

y=a_0+a_1x+a_2x^2

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Cubic Equation

y=a_0+a_1x+a_2x^2+a_3x^3

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Problems with Higher Degree

Overfitting
Strange curves

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

LOGISTIC REGRESSION

Purpose

Binary classification.

Examples:
✔ Spam detection
✔ Pass/Fail prediction
✔ Admission prediction

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

LOGIT / SIGMOID FUNCTION

Sigmoid Function

p(x)=\frac{1}{1+e^{-x}}

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Output Range

✔ 0 to 1 probability

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

ODDS & LOG-ODDS

Odds Formula

Odds=\frac{p}{1-p}

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

THRESHOLD FUNCTION

Classification Rule

y=\begin{cases}1,&p(x)\ge0.5\0,&otherwise\end{cases}

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

MAXIMUM LIKELIHOOD ESTIMATION (MLE)

Purpose

Find best regression coefficients.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━
Features

✔ Minimizes prediction error

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

DECISION TREE LEARNING

Definition

Supervised learning model using tree-like decisions.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

STRUCTURE OF DECISION TREE

Components

✔ Root Node
✔ Internal Node
✔ Branch
✔ Leaf Node

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

DECISION TREE PROCESS

Building the Tree

1. Select best attribute

2. Split dataset

3. Repeat recursively

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Classification

1. Start at root

2. Evaluate condition

3. Traverse branches
 4. Reach leaf node

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

ADVANTAGES OF DECISION TREES

✔ Easy to understand
✔ Handles non-linearity
✔ Fast training
✔ Supports continuous & discrete data

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

DISADVANTAGES OF DECISION TREES

Overfitting
Sensitive to noisy data
Complex with continuous data
NP-complete optimization

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

ENTROPY

Definition

Measures uncertainty/randomness.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Important Rule

✔ Higher entropy → Higher uncertainty

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Entropy Formula

Entropy(D)=-\sum_{i=1}^{n}P_i\log_2(P_i)

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━
 INFORMATION GAIN

Purpose

Select best splitting attribute.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Rule

✔ Maximum gain → Best attribute

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

STOPPING CRITERIA

✔ Entropy = 0
✔ Minimum samples reached
✔ Maximum depth reached

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

DECISION TREE ALGORITHMS

Major Algorithms

✔ ID3
✔ C4.5
✔ CART
✔ CHAID
✔ QUEST

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

ID3 ALGORITHM

Features

✔ Uses Information Gain

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━
ID3 Steps

1. Compute entropy

2. Compute information gain

3. Select best attribute

4. Split dataset

5. Repeat recursively

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

C4.5 ALGORITHM

Improvements over ID3

✔ Handles continuous data

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Splitting Criterion

✔ Gain Ratio

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Gain Ratio Formula

Gain\ Ratio(A)=\frac{Information\ Gain(A)}{Split\ Info(A)}

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

CART ALGORITHM

Full Form

Classification And Regression Trees

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━
Features

✔ Binary tree
✔ Handles classification & regression
✔ Uses GINI Index

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

MOST IMPORTANT EXAM TOPICS

KNN Algorithm
Weighted KNN
Nearest Centroid Classifier
Locally Weighted Regression
Linear Regression
Polynomial Regression
Logistic Regression
Entropy & Information Gain
ID3 Algorithm
C4.5 Algorithm
CART Algorithm

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

FAST REVISION PLAN

First 2 Hours

Learn:

• KNN

• Weighted KNN

• Distance Metrics

• Nearest Centroid

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Next 2 Hours

Learn:
 • Linear Regression

• Multiple Regression

• Polynomial Regression

• Logistic Regression

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━

Final 2 Hours

Learn:

• Decision Trees

• Entropy

• Information Gain

• ID3

• C4.5

• CART

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
━