1.
Program on data wrangling: Combining and merging datasets, Reshaping and
Pivoting
import pandas as pd
# ----------------------------------------------------
# PART 1: COMBINING AND MERGING DATASETS
# ----------------------------------------------------
print("Combining and Merging Datasets")
print("-------------------------------")
# Create first sample dataset
dataset1 = [Link]({
'Name': ['John', 'Mary', 'David'],
'Age': [25, 31, 42]
})
# Create second sample dataset
dataset2 = [Link]({
'Name': ['Emily', 'Michael', 'Sarah'],
'Age': [28, 35, 38]
})
# Concatenating the two datasets (stacking them vertically)
combined_dataset = [Link]([dataset1, dataset2])
print("Concatenated Dataset:")
print(combined_dataset)
# ----------------------------------------------------
# PART 2: MERGING DATASETS USING A COMMON COLUMN
# ----------------------------------------------------
# Create first dataset with ID and Name
dataset1 = [Link]({
'ID': [1, 2, 3],
'Name': ['John', 'Mary', 'David']
})
# Create second dataset with ID and Age
dataset2 = [Link]({
'ID': [1, 2, 3],
'Age': [25, 31, 42]
})
# Merge the two datasets using the ID column
merged_dataset = [Link](dataset1, dataset2, on='ID')
print("\nMerged Dataset:")
print(merged_dataset)
# ----------------------------------------------------
# PART 3: RESHAPING AND PIVOTING
# ----------------------------------------------------
print("\nReshaping and Pivoting")
print("-------------------------------")
# Create a sample dataset with repeated IDs and Years
dataset = [Link]({
'ID': [1, 1, 2, 2],
'Year': [2018, 2019, 2018, 2019],
'Sales': [100, 120, 80, 90]
})
# Reshape using pivot_table: makes Years into columns
reshaped_dataset = pd.pivot_table(dataset, values='Sales', index='ID', columns='Year')
print("Reshaped Dataset:")
print(reshaped_dataset)
# ----------------------------------------------------
# PART 4: PIVOTING USING MELT (Long Format Conversion)
# ----------------------------------------------------
dataset = [Link]({
'ID': [1, 2],
'2018': [100, 80],
'2019': [120, 90]
})
# Melt converts wide data to long data format
pivoted_dataset = [Link](
dataset,
id_vars='ID', # Keep ID as it is
value_vars=['2018', '2019'],# Columns to unpivot
var_name='Year', # New column name for years
value_name='Sales' # New column name for sales values
)
print("\nPivoted Dataset:")
print(pivoted_dataset)
2. Program on Data Transformation: String Manipulation, Regular Expressions
import pandas as pd
import re
#Creating dataset
dataset = [Link]({
'Name': ['John Smith', 'Mary Johnson', 'David Lee'],
'Address': ['123 Main St', '456 Elm St', '789 Oak St']
})
dataset
#Convert to lowercase
dataset['Name'] = dataset['Name'].[Link]()
dataset['Address'] = dataset['Address'].[Link]()
dataset
#Convert to uppercase
dataset['Name'] = dataset['Name'].[Link]()
dataset['Address'] = dataset['Address'].[Link]()
dataset
#Strip whitespace(removes extra spaces at the beginning and end)
dataset['Name'] = dataset['Name'].[Link]()
dataset['Address'] = dataset['Address'].[Link]()
dataset
#Replace Strings(finds a word and replaces it)
dataset['Name'] = dataset['Name'].[Link]('JOHN', 'JONATHAN')
dataset['Address'] = dataset['Address'].[Link]('ST', 'STREET')
dataset
#Creating another dataset(to apply Regular expression)
dataset = [Link]({
'Email': ['john@[Link]', 'mary@[Link]', 'david@[Link]'],
'Phone': ['123-456-7890', '098-765-4321', '555-123-4567']
})
dataset
dataset['Domain'] = dataset['Email'].[Link](r'@(.*)', expand=False)
dataset
dataset['Valid Phone'] = dataset['Phone'].[Link](
r'^\d{3}-\d{3}-\d{4}$', regex=True)
dataset
#Extract phone number components
dataset[['AreaCode','Prefix','LineNumber']]=dataset['Phone'].[Link](r'(\d{3})-(\d{3})-(
\d{4})')
Dataset
4. Program to measure central tendency and measures of dispersion: Mean, Median,
Mode, Standard Deviation, Variance, Mean deviation and Quartile deviation for a
frequency distribution/data.
import pandas as pd
import numpy as np
#load the dataset
# Direct data (no CSV file)
data = [Link]({
'Values': [10, 20, 20, 30, 40, 50, 50, 50, 60, 70]
})
data
print("Central Tendency Measures")
print("----------------------------")
#mean calculation
mean = [Link](data['Values'])
print("Mean:", mean)
#Median calucation
median = [Link](data['Values'])
print("Median:", median)
#Mode Calculation
mode = data['Values'].mode().iloc[0]
print("Mode:", mode)
#Variance
variance = [Link](data['Values'])
print("Variance:", variance)
#Mean Deviation
mean_dev = [Link]([Link](data['Values'] - mean))
print("Mean Deviation:", mean_dev)
#Quartile Deviation
q1 = [Link](data['Values'], 25)
q3 = [Link](data['Values'], 75)
quartile_dev = (q3 - q1) / 2
print("Quartile Deviation:", quartile_dev)
print("\nSummary Statistics:")
print(data['Values'].describe())
7. Program to implement one sample, two sample and paired sample t-tests for a sample
data and analyse the results.
import pandas as pd
import numpy as np
from scipy import stats
# Load sample data
data = pd.read_csv('Documents/[Link]')
print(data)
# One Sample T-Test
print("One Sample T-Test")
print("------------------")
# Define the null hypothesis mean
null_mean = 0
# Perform one sample t-test
t_stat, p_val = stats.ttest_1samp(data['Values'], null_mean)
print("T-Statistic:", t_stat)
print("P-Value:", p_val)
# Interpret the results
if p_val < 0.05:
print("Reject the null hypothesis. The sample mean is significantly different from the null
mean.")
else:
print("Fail to reject the null hypothesis. The sample mean is not significantly different from
the null mean.")
# Sample data
data1 = [10, 12, 14, 11]
# Perform two-sample t-test
t_stat, p_val = stats.ttest_ind(data['Values'], data1)
print("T-Statistic:", t_stat)
print("P-Value:", p_val)
# Interpretation
if p_val < 0.05:
print("Reject the null hypothesis. The two sample means are significantly different.")
else:
print("Fail to reject the null hypothesis. The two sample means are not significantly
different.")
# Before and after data
after = [70, 68, 60, 66]
# Perform Paired Sample T-Test
t_stat, p_val = stats.ttest_rel(data['Values'], after)
print("T-Statistic:", t_stat)
print("P-Value:", p_val)
# Interpret the results
if p_val < 0.05:
print("Reject the null hypothesis. There is a significant difference between before and after.")
else:
print("Fail to reject the null hypothesis. There is no significant difference between before and
after.")
# Analyze the results
print("\nAnalysis of Results")
print("---------------------")
# Calculate the mean and standard deviation of each sample
mean1 = [Link](data['Values'])
std_dev1 = [Link](data['Values'])
print("Sample 1: Mean =", mean1, ", Standard Deviation =", std_dev1)
# Analyze the results
print("\nAnalysis of Results")
print("---------------------")
# Calculate the mean and standard deviation of each sample
mean1 = [Link](data1)
std_dev1 = [Link](data1)
print("Sample 1: Mean =", mean1, ", Standard Deviation =", std_dev1)
# Analyze the results
print("\nAnalysis of Results")
print("---------------------")
# Calculate the mean and standard deviation of each sample
mean1 = [Link](after)
std_dev1 = [Link](after)
print("Sample 1: Mean =", mean1, ", Standard Deviation =", std_dev1)
8. Program to implement One-way and Two-way ANOVA tests and analyze the results
import pandas as pd
import numpy as np
from scipy import stats
#load the sample data
data = [Link]({
"Group": ["A", "B", "C", "A"],
"Values": [10, 18, 20, 15]
})
print(data)
print("One-way ANOVA")
print("-------------")
Groups= data['Group']
Groups
groupA = data[data['Group'] == 'A']['Values']
groupB = data[data['Group'] == 'B']['Values']
groupC = data[data['Group'] == 'C']['Values']
f_stat, p_val = stats.f_oneway(groupA, groupB, groupC)
print("F-Statistic:", f_stat)
print("P-Value:", p_val)
# Interpret the results
if p_val < 0.05:
print("Reject the null hypothesis. The means of the groups are significantly different.")
else:
print("Fail to reject the null hypothesis. The means of the groups are not significantly
different.")
data = [Link]({
"Group1": ["A", "B", "C", "A"],
"Group2":["F","M","M","F"],
"Values": [10, 18, 20, 15]
})
print(data)
Groups1= data['Group1']
Groups1
Groups2= data['Group2']
Groups2
import pandas as pd
import [Link] as sm
from [Link] import ols
# Fit two-way ANOVA model
model = ols('Values ~ C(Group1) + C(Group2) + C(Group1):C(Group2)', data=data).fit()
anova_table = [Link].anova_lm(model, typ=2)
print(anova_table)
# Print F-stat and p-value
print("Group1:")
print(" F-Statistic:", anova_table.loc["C(Group1)", "F"])
print(" P-Value :", anova_table.loc["C(Group1)", "PR(>F)"])
print("\nGroup2:")
print(" F-Statistic:", anova_table.loc["C(Group2)", "F"])
print(" P-Value :", anova_table.loc["C(Group2)", "PR(>F)"])
# Extract p-values
p_val1 = anova_table.loc["C(Group1)", "PR(>F)"]
p_val2 = anova_table.loc["C(Group2)", "PR(>F)"]
# Interpret the results
if p_val1 < 0.05 and p_val2 < 0.05:
print("Reject the null hypothesis. The means of the groups are significantly different.")
else:
print("Fail to reject the null hypothesis. The means of the groups are not significantly
different.")
print("Tukey's HSD Test")
print("-----------------")
from [Link] import pairwise_tukeyhsd
tukey = pairwise_tukeyhsd(
endog=data["Values"],
groups=data["Group1"],
alpha=0.05
)
print(tukey)
print()
12. Program to Implement multiple linear regression using iris dataset, visualize and
analyze the results.
import pandas as pd
import numpy as np
import [Link] as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from [Link] import mean_squared_error, r2_score
from [Link] import StandardScaler
#Loading the Iris Dataset
from [Link] import load_iris
iris = load_iris()
data = [Link](data=[Link], columns=iris.feature_names)
data['target'] = [Link]
data
#Defining Features and Target
X = [Link]('target', axis=1) # all features
y = data['target'] # target variable
X
# Split the dataset into training and testing sets (80% train, 20% test)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# Standardize (scale) the feature data
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = [Link](X_test)
# Create and train the Multiple Linear Regression model
model = LinearRegression()
[Link](X_train_scaled, y_train)
# Predict target values for the test set
y_pred = [Link](X_test_scaled)
# Evaluate model performance
mse = mean_squared_error(y_test, y_pred) # Mean Squared Error
r2 = r2_score(y_test, y_pred)
print("Mean Squared Error:", mse)
print("R-squared Value:", r2)
# Visualization: Actual vs Predicted values
[Link](figsize=(10, 8))
[Link](y_test, y_pred)
[Link]([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--') # Perfect line
[Link]("Multiple Linear Regression on Iris Dataset")
[Link]("Actual Values")
[Link]("Predicted Values")
[Link]()
# Analysis of the Results
print("\nAnalysis of Results")
print("---------------------")
print("Coefficients:", model.coef_) # Model weights
print("Intercept:", model.intercept_) # Bias value
print("Feature Importances: Not available for LinearRegression model")
print("Mean Squared Error:", mse)
print("R-squared Value:", r2)
print("Training Data Shape:", X_train_scaled.shape)
print("Testing Data Shape:", X_test_scaled.shape)