PHASE 03

Advanced Analytics
9 – 18 Months 9 – 18 Months

01 APIs & Scraping requests, BeautifulSoup, rate limiting

02 Statsmodels OLS, ARIMA, forecasting, ACF/PACF

03 PySpark SparkSession, DataFrame, parquet at scale

04 Automation cron, Airflow, argparse, scheduling

05 Code Quality pytest, type hints, black, modular code

06 Cloud & Deployment AWS S3, BigQuery, Docker, Streamlit

07 Performance vectorization, Numba, Dask, profiling

08 Feature Engineering encoding, binning, interaction terms

Python Data Analytics · Advanced Level Page 1

 01 —

APIs & Web Scraping

— REST API WITH REQUESTS —

REST API

import requests, time

# GET request
url = '[Link]
headers = {'Authorization': 'Bearer YOUR_TOKEN',
'Content-Type': 'application/json'}
response = [Link](url, headers=headers, timeout=10)
response.raise_for_status() # raise on 4xx/5xx
data = [Link]()

# POST request
payload = {'start_date': '2024-01-01', 'limit': 100}
r = [Link](url, json=payload, headers=headers)

# Pagination loop
all_records = []
page = 1
while True:
r = [Link](url, params={'page': page, 'per_page': 100})
records = [Link]()['data']
if not records: break
all_records.extend(records)
page += 1
[Link](0.5) # respect rate limits
df = [Link](all_records)

— WEB SCRAPING WITH BEAUTIFULSOUP —

Python Data Analytics · Advanced Level Page 2

 BeautifulSoup

from bs4 import BeautifulSoup

import requests, pandas as pd

url = '[Link]
r = [Link](url, headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup([Link], '[Link]')

# Find elements
title = [Link]('h1').[Link]()
prices = [[Link] for p in soup.find_all('span', class_='price')]
links = [a['href'] for a in soup.find_all('a', href=True)]

# Scrape a table
table = [Link]('table', id='data-table')
rows = table.find_all('tr')
data = [[[Link]() for td in row.find_all('td')] for row in rows[1:]]
headers= [[Link]() for th in rows[0].find_all('th')]
df = [Link](data, columns=headers)

TIP Always check [Link] before scraping. Use [Link]() between requests.

Python Data Analytics · Advanced Level Page 3

 02 —

Statsmodels & Forecasting

— OLS REGRESSION —

OLS

import [Link] as sm
import [Link] as smf

# Formula API (R-style)

model = [Link]('sales ~ price + advertising + C(region)', data=df)
result = [Link]()
print([Link]())

# Coefficients and p-values

print([Link]) # coefficients
print([Link]) # p-values
print(result.rsquared_adj) # adjusted R-squared

# Predictions with confidence intervals

pred = result.get_prediction(new_data)
ci = pred.conf_int(alpha=0.05)

— TIME SERIES — ARIMA —

ARIMA

from [Link] import ARIMA

from [Link] import seasonal_decompose
from [Link] import plot_acf, plot_pacf

# Decompose trend/seasonal/residual
result = seasonal_decompose(df['sales'], model='additive', period=12)
[Link]()

# Check stationarity
from [Link] import adfuller
adf_test = adfuller(df['sales'])
print(f'ADF p-value: {adf_test[1]:.4f}') # < 0.05 = stationary

# Fit ARIMA(p, d, q)
model = ARIMA(df['sales'], order=(2, 1, 2))
result = [Link]()

# Forecast next 12 periods

forecast = [Link](steps=12)
conf_int = result.get_forecast(12).conf_int()

Python Data Analytics · Advanced Level Page 4

 — SARIMA (SEASONAL) —

SARIMA

from [Link] import SARIMAX

# SARIMA(p,d,q)(P,D,Q,s) — s=12 for monthly data

model = SARIMAX(df['sales'],
order=(1, 1, 1),
seasonal_order=(1, 1, 1, 12))
result = [Link](disp=False)
print([Link]())
forecast = [Link](steps=24)

Python Data Analytics · Advanced Level Page 5

 03 —

PySpark (Big Data)

— SESSION & READ —

Setup & Read

from [Link] import SparkSession

from [Link] import functions as F
from [Link] import *

# Create session
spark = [Link] \
.appName('Analytics') \
.config('[Link]', '200') \
.getOrCreate()

# Read data
df = [Link]('data/*.csv', header=True, inferSchema=True)
df = [Link]('data/sales/')
df = [Link]('data/events/')

# Basic inspection
[Link]()
[Link](5)
[Link]()

— TRANSFORMATIONS —

Python Data Analytics · Advanced Level Page 6

 Transforms

# Select & filter

[Link]('name', 'sales', 'city')
[Link]([Link]('age') > 30)
[Link](([Link]('city') == 'Karachi') & ([Link]('sales') > 1000))

# Add columns
df = [Link]('sales_k', [Link]('sales') / 1000)
df = [Link]('category', [Link]([Link]('score') > 80, 'A').otherwise('B'))

# GroupBy
[Link]('city').agg(
[Link]('sales').alias('total_sales'),
[Link]('age').alias('avg_age'),
[Link]('*').alias('records')
).orderBy('total_sales', ascending=False).show()

# Join
joined = [Link](customers, on='customer_id', how='inner')

# Write output
[Link]('output/', mode='overwrite')

Python Data Analytics · Advanced Level Page 7

 04 —

Automation & Pipelines

— CLI WITH ARGPARSE —

argparse

import argparse

def main():
parser = [Link](description='Run analytics pipeline')
parser.add_argument('--input', required=True, help='Input CSV path')
parser.add_argument('--output', required=True, help='Output path')
parser.add_argument('--year', type=int, default=2024)
parser.add_argument('--verbose',action='store_true')
args = parser.parse_args()

if [Link]:
print(f'Processing {[Link]} for year {[Link]}')

df = pd.read_csv([Link])
# ... process ...
df.to_csv([Link], index=False)

if __name__ == '__main__':
main()

# Run: python [Link] --input [Link] --output [Link] --year 2024

— SCHEDULE WITH SCHEDULE LIBRARY —

Python Data Analytics · Advanced Level Page 8

 Scheduling

import schedule, time

def daily_report():
df = pd.read_sql('SELECT * FROM sales', engine)
df.to_csv(f'reports/sales_{today}.csv')
print('Daily report generated')

def weekly_summary():
# ... run weekly analysis ...
pass

# Schedule jobs
[Link]().[Link]('08:00').do(daily_report)
[Link]().[Link]('09:00').do(weekly_summary)
[Link](30).[Link](check_data_quality)

# Run loop
while True:
schedule.run_pending()
[Link](60)

Python Data Analytics · Advanced Level Page 9

 05 —

Code Quality & Testing

— TYPE HINTS —

Type Hints

from typing import Optional, List, Dict, Tuple

import pandas as pd

def clean_dataframe(
df: [Link],
columns: List[str],
fillvalue: Optional[float] = 0.0
) -> [Link]:
'''Clean specified columns in a DataFrame.

Args:
df: Input DataFrame
columns: Columns to clean
fillvalue: Value to fill NaN with

Returns:
Cleaned DataFrame
'''
return df[columns].fillna(fillvalue)

— TESTING WITH PYTEST —

Python Data Analytics · Advanced Level Page 10

 pytest

# test_analytics.py
import pytest, pandas as pd
from my_module import clean_dataframe, calc_revenue

# Fixture
@[Link]
def sample_df():
return [Link]({'a': [1,2,None], 'b': [4,None,6]})

# Test functions
def test_clean_fills_nan(sample_df):
result = clean_dataframe(sample_df, ['a','b'], fillvalue=0)
assert [Link]().sum().sum() == 0

def test_clean_returns_dataframe(sample_df):
result = clean_dataframe(sample_df, ['a'])
assert isinstance(result, [Link])

def test_revenue_positive():
assert calc_revenue(100, 0.1) == 110.0

# Run: pytest test_analytics.py -v

Python Data Analytics · Advanced Level Page 11

 06 —

Cloud & Deployment

— AWS S3 WITH BOTO3 —

AWS S3

import boto3, pandas as pd

from io import StringIO, BytesIO

s3 = [Link]('s3',
aws_access_key_id='KEY',
aws_secret_access_key='SECRET',
region_name='ap-south-1'
)

# Read CSV from S3

obj = s3.get_object(Bucket='my-bucket', Key='data/[Link]')
df = pd.read_csv(obj['Body'])

# Write DataFrame to S3
buf = BytesIO()
df.to_parquet(buf, index=False)
[Link](0)
s3.put_object(Bucket='my-bucket', Key='output/[Link]', Body=buf)

# List files
files = s3.list_objects_v2(Bucket='my-bucket', Prefix='data/')
keys = [f['Key'] for f in files['Contents']]

— STREAMLIT DASHBOARD —

Python Data Analytics · Advanced Level Page 12

 Streamlit

import streamlit as st
import pandas as pd, [Link] as px

st.set_page_config(page_title='Sales Dashboard', layout='wide')

[Link]('Sales Analytics Dashboard')

# Sidebar filters
city = [Link]('City', [Link]())
year = [Link]('Year', 2020, 2024, 2024)

# Filter data
filtered = df[([Link] == city) & ([Link] == year)]

# KPI metrics
col1, col2, col3 = [Link](3)
[Link]('Total Sales', f'${[Link]():,.0f}')
[Link]('Avg Order', f'${[Link]():,.0f}')
[Link]('Total Orders', len(filtered))

# Chart
fig = [Link](filtered, x='date', y='sales', title='Sales Trend')
st.plotly_chart(fig, use_container_width=True)

# Data table
[Link]([Link](50), use_container_width=True)

# Run: streamlit run [Link]

Python Data Analytics · Advanced Level Page 13

 07 —

Performance Optimization

— VECTORIZATION VS LOOPS —

Vectorization

# SLOW: Python loop

for i in range(len(df)):
[Link][i, 'tax'] = [Link][i, 'price'] * 0.18

# FASTER: apply()
df['tax'] = df['price'].apply(lambda x: x * 0.18)

# FASTEST: vectorized NumPy

df['tax'] = df['price'].values * 0.18

# Use [Link] instead of apply for conditions

df['level'] = [Link](df['score'] > 80, 'High', 'Low')

# Use [Link] for complex expressions

[Link]('profit = revenue - cost - tax', inplace=True)

— DASK (PARALLEL PANDAS) —

Dask

import [Link] as dd

# Read large CSV (lazy loading)

ddf = dd.read_csv('large_data/*.csv')

# Same API as pandas — operations are lazy

result = [Link]('city')['sales'].sum()

# Trigger computation
result = [Link]() # returns pandas DataFrame

# Repartition for better parallelism

ddf = [Link](npartitions=10)

# Write to parquet (parallel)

ddf.to_parquet('output/', write_index=False)

Python Data Analytics · Advanced Level Page 14

 08 —

Feature Engineering

— ENCODING CATEGORICALS —

Encoding

# One-hot encoding
df = pd.get_dummies(df, columns=['city', 'gender'], drop_first=True)

# Label encoding (ordinal)

from [Link] import OrdinalEncoder
enc = OrdinalEncoder(categories=[['Low','Medium','High']])
df[['level_enc']] = enc.fit_transform(df[['level']])

# Target encoding (mean of target per category)

means = [Link]('city')['sales'].mean()
df['city_encoded'] = df['city'].map(means)

# Frequency encoding
freq = df['city'].value_counts(normalize=True)
df['city_freq'] = df['city'].map(freq)

— NUMERIC FEATURES —

Numeric Features

# Log transform (fix right skew)

df['log_salary'] = np.log1p(df['salary']) # log(x+1)

# Binning
df['age_group'] = [Link](df['age'],
bins=[0, 18, 35, 55, 100],
labels=['Teen', 'Young', 'Mid', 'Senior'])

# Interaction terms
df['price_x_qty'] = df['price'] * df['quantity']
df['price_sq'] = df['price'] ** 2

# Rolling features
df['sales_ma7'] = df['sales'].rolling(7).mean()
df['sales_lag1'] = df['sales'].shift(1)
df['sales_change'] = df['sales'].pct_change()

# Normalization
from [Link] import MinMaxScaler
scaler = MinMaxScaler()
df[['price_norm']] = scaler.fit_transform(df[['price']])

Python Data Analytics · Advanced Level Page 15

 FEATURE SELECTION DIMENSIONALITY
Remove Noise PCA

Use correlation > 0.95 to drop redundant from [Link] import PCA.
features. Use RandomForest Reduce 100+ features to top N components
.feature_importances_ to rank them. explaining 95% variance.

DATE FEATURES VALIDATION

Extract From Dt Cross-Val

[Link], .month, .dayofweek, Never fit transformers on test data. Use

.quarter, .is_weekend — all powerful Pipeline + cross_val_score to avoid data
predictors. leakage.

Python Data Analytics · Advanced Level Page 16