import numpy as np
import pandas as pd
import seaborn as sns
import [Link] as plt
In [2]:
df_train = pd.read_csv('/kaggle/input/tabular-dataset-ready-for-malicious-
url-detection/train_dataset.csv',)
df_test = pd.read_csv('/kaggle/input/tabular-dataset-ready-for-malicious-url-
detection/test_dataset.csv',)
In [3]:
df_train.head()
Out[3]:
5 rows × 60 columns
In [4]:
df_test.head()
Out[4]:
5 rows × 60 columns
In [5]:
df_train.dtypes.value_counts()
Out[5]:
int64 48
float64 9
object 3
Name: count, dtype: int64
In [6]:
df_test.dtypes.value_counts()
Out[6]:
int64 48
float64 9
object 3
Name: count, dtype: int64
In [7]:
df_train['label'] = df_train['label'].astype('int')
df_test['label'] = df_test['label'].astype('int')
In [8]:
df_train['label_text'] = ['Malicious' if i else 'Benign' for i in
df_train.label]
df_test['label_text'] = ['Malicious' if i else 'Benign' for i in
df_test.label]
In [9]:
df_train.label_text.value_counts()
Out[9]:
label_text
Benign 5283175
Malicious 1445673
Name: count, dtype: int64
In [10]:
df_test.label_text.value_counts()
Out[10]:
label_text
Benign 1320794
Malicious 361419
Name: count, dtype: int64
In [11]:
# Initialise the list that will store the analysed features
analysed_features = []
Boxplot for some numeric Features
In [12]:
features_boxplot_nolog = ['url_entropy', 'url_nunique_chars_ratio',]
features_boxplot_log = [c for c in df_train.columns if 'len' in c]
features_boxplot = features_boxplot_log + features_boxplot_nolog
analysed_features.append(features_boxplot)
In [13]:
for f in features_boxplot:
fig, axes = [Link](1, 2, figsize=(15, 5), sharey=True)
# [Link](figsize=(12, 8))
[Link](data=df_train, x=f, y="label_text", hue="label_text",
ax=axes[0])
axes[0].set_title('Training data')
[Link](data=df_test, x=f, y="label_text", hue="label_text",
ax=axes[1])
axes[1].set_title('Test data')
if f in features_boxplot_log:
axes[0].set_xscale('log')
axes[1].set_xscale('log')
Barplot for Has features
In [14]:
has_features = [c for c in df_train.columns if 'has_' in c] + ['label_text']
analysed_features.append([c for c in df_train.columns if 'has_' in c])
df_train[has_features].head()
Out[14]:
In [15]:
df_train[has_features].groupby('label_text').mean()
Out[15]:
In [16]:
df_test[has_features].groupby('label_text').mean()
Out[16]:
In [17]:
for title, df in {'Train data': df_train, 'Test data': df_test}.items():
mean_has_df = df[has_features].groupby('label_text').mean()
df_mean_barplot =
mean_has_df.[Link]().reset_index().rename(columns={'level_0': 'has_feature',
0:
'mean_val'})
[Link](figsize=(16, 8))
g = [Link](x=df_mean_barplot['has_feature'],
y=df_mean_barplot['mean_val'],
hue=df_mean_barplot['label_text'])
g.set_xticklabels(g.get_xticklabels(), rotation=30,
horizontalalignment='right')
g.set_title(title)
Count Features Correlation
In [18]:
count_features = [c for c in df_train.columns if 'count' in c and not 'has_'
in c]
analysed_features.append([c for c in df_train.columns if 'count' in c])
In [19]:
corr = df_train[count_features].corr()
# Generate a mask for the upper triangle
mask = [Link](np.ones_like(corr, dtype=np.bool_))
# Set up the matplotlib figure
f, ax = [Link](figsize=(25, 20))
# Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)
# Draw the heatmap with the mask and correct aspect ratio
[Link](corr, mask=mask, cmap=cmap, vmax=.3, center=0,
square=True, linewidths=.5, cbar_kws={"shrink": .5}, annot=True)
Out[19]:
<Axes: >
Correlation Matrix
In [20]:
exclude_corr = ['url', 'source', 'label_text', 'tld'] + [c for c in
df_train.columns if 'has_' in c]
corr = df_train[[i for i in df_train.columns if i not in
exclude_corr]].corr()
In [21]:
linkcode
# Generate a mask for the upper triangle
mask = [Link](np.ones_like(corr, dtype=np.bool_))
# Set up the matplotlib figure
f, ax = [Link](figsize=(25, 20))
# Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)
# Draw the heatmap with the mask and correct aspect ratio
[Link](corr, mask=mask, cmap=cmap, vmax=.3, center=0,
square=True, linewidths=.5, cbar_kws={"shrink": .5}, annot=True)
Out[21]:
<Axes: >
In [ ]: