RMS Titanic (Simple EDA)

# Linear algebra and data manipulation imports
import pandas as pd

# Visualization imports
import matplotlib.pyplot as plt
import seaborn as sns

# sns.set_style('whitegrid')
# plt.style.use('dark_background')
%matplotlib inline

DATA_DIR='../data/'

# Import train data and test data
train = pd.read_csv(DATA_DIR + 'raw_titanic_train.csv')
test = pd.read_csv(DATA_DIR + 'raw_titanic_test.csv')

# Check info of train dataset
train.info()

# Check first 10 rows of train data set
train.head(10)

# Check shape of train dataset
train.shape

# Show description of train dataset for numerical columns
# (may include some categorical columns as well)
train.describe()

# Note that 'Survived' and 'Pclass' shown are categorical variables and not numerical

# Check info of test dataset
test.info()

# Check first 10 rows of test data set
test.head(10)

# Has same columns as the train data except the label, 'Survived' is out

# Check the dimensions (row count) of the test data
test.shape

# Show the description of the test data, excluding non-numeric columns
test.describe()

# Note that Pclass' shown is a categorical variable

# Visualize null values in train data

sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')

# Visualize null values in test data

sns.heatmap(test.isnull(),yticklabels=False,cbar=False,cmap='viridis')

# Visualizing classification label 'Survived'

sns.countplot(train['Survived'])
plt.title('Count of Survived and Not Survived')

# Visualizing classification label 'Survived' for 'Sex' feature

sns.countplot(x='Survived',data=train,hue='Sex',palette='RdBu_r')
plt.title('Count of Survived distributed against Sex')

# Visualizing classification label 'Survived' for 'Pclass' feature

sns.countplot(x='Survived',data=train,hue='Pclass',palette='viridis')
plt.title('Count of Survived distributed against Pclass')

train['Age'].dropna().plot.hist(bins=40)
plt.title('Age distribution plot')

sns.countplot(x='SibSp',data=train)
plt.title('Count of SibSp')

sns.countplot(x='Survived', data=train, hue='Embarked')
plt.title('Count of Survived distributed against Embarked')
# it looks like many passenger embarked from Southampton.
# Nonetheless, this distribution does not clearly show any relevant correlation between embarkation and survival.

sns.distplot(train['Fare'],kde=False,bins=40)
plt.title('Fare distribution plot')

plt.figure(figsize=(10,7))
sns.boxplot(x='Pclass',y='Age',data=train)
plt.title('Boxplot of Pclass against Age')

import re

# feature engineering on Name feature. It looks like the user title could have an effect on Survival
def eng_name(data):
    """
    helper method to engineer new Title feature from Name feature
    :param data: train or test data
    :return: None
    """
    data['Title'] = data['Name'].apply(lambda name: re.split('[,.]+ *', name)[1])

def impute_age(data):
    """
    helper method to perform imputation on Age
    :param data: train or test data
    :return: None
    """
    Pclass1Mean = data['Age'][data['Pclass'] == 1].mean()
    Pclass2Mean = data['Age'][data['Pclass'] == 2].mean()
    Pclass3Mean = data['Age'][data['Pclass'] == 3].mean()

    def _impute(cols):
        Age = cols[0]
        Pclass = cols[1]

        if (pd.isnull(Age)):
            if Pclass == 1:
                return Pclass1Mean
            elif Pclass == 2:
                return Pclass2Mean
            else:
                return Pclass3Mean
        else:
            return Age

    data['Age'] = data[['Age', 'Pclass']].apply(_impute,axis=1)

# perform imputation on train data
impute_age(train)

# Visualize null values in train data
sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')

eng_name(train)
train.head()[['PassengerId', 'Name', 'Title']]

# Cabin has to many missing values, could make it categorical as Cabin Available vs Cabin Inavailable
train.drop('Cabin',axis=1,inplace=True)

# Impute (one) missing value for Fare column. The culprit is a male and third class passenger
train['Embarked'] = train['Embarked'].fillna(train['Embarked'].mode()[0])

# Visualize null values in train data
sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')

plt.figure(figsize=(10,7))
sns.boxplot(x='Pclass',y='Age',data=test)

# Impute missing age values in test dataset
impute_age(test)

eng_name(test)
test.head()[['PassengerId', 'Name', 'Title']]

# Cabin has to many missing values, could make it categorical as Cabin Available vs Cabin Inavailable
test.drop('Cabin',axis=1,inplace=True)

# Impute (one) missing value for Fare column. The culprit is a male and third class passenger
test['Fare'] = test['Fare'].fillna(test['Fare'][(test['Pclass'] == 3) & (test['Sex'] == 'male')].mean())

# Visualize null values in test data¡
sns.heatmap(test.isnull(),yticklabels=False,cbar=False,cmap='viridis')

# save clean test data to csv file
train.to_csv(DATA_DIR + 'clean_titanic_train.csv',index=False)
test.to_csv(DATA_DIR + 'clean_titanic_test.csv',index=False)