import os import numpy as np import pandas as pd import matplotlib.pyplot as plt import seaborn as sns
from google.colab import files
from sklearn.preprocessing import StandardScaler, LabelEncoder, Imputer, MinMaxScaler from sklearn.impute import SimpleImputer from sklearn.linear_model import Ridge from sklearn.neighbors import KNeighborsRegressor from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor from sklearn.model_selection import ShuffleSplit, GridSearchCV, train_test_split from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error, explained_variance_score, mean_squared_log_error from sklearn.decomposition import PCA from sklearn.pipeline import Pipeline from sklearn.feature_selection import SelectKBest, SelectFromModel
from xgboost import XGBRegressor
from mlxtend.regressor import StackingRegressor
from time import clock
seed = 20
"""# Loading In Datasets"""
from google.colab import drive drive.mount('/content/My-Drive')
os.chdir('/content/My-Drive/')
df = pd.read_csv('My Drive/Datathon/BUDSA_Datathon_2019.csv',low_memory=False)
df.info()
"""# Preprocessing
Missing Values
"""
Total NA values per Category
total_NA = df.isnull().sum().sort_values(ascending=False)
Percent Missing for Each Category
percent_NA = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)
Removing the missing data
data_to_delete = pd.concat([total_NA, percent_NA], axis=1, keys=["Total", "Percent"]) print(data_to_delete)
"""##String Cleaning"""
df['doctor'] = df['idaction_url'].replace('caredash.com/doctors/',' ',regex=True,inplace=True)
print(df['idaction_url'].head())
SCRAP
df['idaction_name'].replace(' | CareDash','',regex=True,inplace=True) df['idaction_name'].replace('|CareDash',' ',regex=True,inplace=True) print(df['idaction_name'].head())
"""## Encoding"""
labels_enc = LabelEncoder() x_num = df.loc[:, df.dtypes != np.object].copy() col_nam = pd.DataFrame(x_num).columns imputer = SimpleImputer(missing_values =np.nan, strategy = 'median') x_num = imputer.fit_transform(x_num)
scaler = StandardScaler() scaler1 = MinMaxScaler()
x_num = scaler1.fit_transform(x_num)
x_num = pd.DataFrame(x_num, columns=col_nam)
print(x_num.describe())
labels_enc = LabelEncoder()
x_cat = df.loc[:, df.dtypes == np.object].copy()
x_cat = x_cat.astype(str)
cat_imputer = SimpleImputer(missing_values=np.nan, strategy = 'most_frequent')
dummy = x_cat.apply(labels_enc.fit_transform)
dummy = cat_imputer.fit_transform(dummy)
print(dummy)
dummy_col = pd.DataFrame(x_cat).columns print(dummy_col) dummy = pd.DataFrame(dummy, columns = dummy_col)
df1 = pd.concat([x_num,dummy], axis=1)
print(df1.info())
"""# Feature Enginering
Custom Int Features
"""
df['visit_events_by_actions'] = df['visit_total_events'] / df['visit_total_actions'] df['visit_events_by_time'] = df['visit_total_events'] / df['visit_total_time'] df['visit_actions_by_time'] = df['visit_total_actions'] / df['visit_total_time'] df['review_by_events'] = df['review_count'] / df['visit_total_events'] df['events_minus_actions'] = df['visit_total_events'] - df['visit_total_actions']
df1['visit_events_by_actions'] = df1['visit_total_events'] / df1['visit_total_actions'] df1['visit_events_by_time'] = df1['visit_total_events'] / df1['visit_total_time'] df1['visit_actions_by_time'] = df1['visit_total_actions'] / df1['visit_total_time'] df1['review_by_events'] = df1['review_count'] / df1['visit_total_events'] df1['events_minus_actions'] = df1['visit_total_events'] - df1['visit_total_actions']
"""##Custom Time Features"""
df['server_time'] = pd.to_datetime(df['server_time'], infer_datetime_format=True) df1['server_time'] = pd.to_datetime(df1['server_time'], infer_datetime_format=True)
print(df['server_time'].dtype)
df['visit_first_action_time'] = pd.to_datetime(df['visit_first_action_time']) df1['visit_first_action_time'] = pd.to_datetime(df1['visit_first_action_time'])
df['visit_last_action_time'] = pd.to_datetime(df['visit_last_action_time']) df1['visit_last_action_time'] = pd.to_datetime(df1['visit_last_action_time'])
df['visit_time_diff'] = df['visit_last_action_time'] - df['visit_first_action_time'] df1['visit_time_diff'] = df1['visit_last_action_time'] - df1['visit_first_action_time']
df1.info()
"""# Exploratory Data Analysis
Histograms
Checking Distributions across feats """
df2 = df.loc[:, df.dtypes == np.object]
for col in df2: if len(df2[col].unique()) > 20: df2 = df2.drop(columns=[col])
Histograms of all the Features
mx_cols = pd.DataFrame(df2).columns x_cols = pd.DataFrame(df2, columns=mx_cols)
print(x_cols)
for fig,col in enumerate(x_cols.columns):
plt.figure(fig)
sns.countplot(x=col, data=x_cols)
sns.plt.title(col +' count plot')
plt.show(fig)
"""## Correlation Matrix"""
plt.figure() sns.heatmap(df1.corr(), fmt=".2f") plt.show()
"""## Breakdown Features in Cats | Correlation"""
actions = pd.DataFrame() actions['idaction_event_action'], actions['idaction_url'], actions['idaction_event_category'], actions['server_time'], actions['idaction_name'] = df1['idaction_event_action'], df1['idaction_url'], df1['idaction_event_category'], df1['server_time'], df1['idaction_name']
print(actions)
acquisition = pd.DataFrame() acquisition['device'], acquisition['channel'], acquisition['medium'], acquisition['source'], acquisition['organic_search_engine'], acquisition['browser'] = df1['device'], df1['channel'], df1['medium'], df1['source'], df1['organic_search_engine'], df1['browser']
profile_char = pd.DataFrame() profile_char['location'], profile_char['profile_claimed'], profile_char['page_type'], profile_char['specialty'], profile_char['review_count'] = df1['location'], df1['profile_claimed'], df1['page_type'], df1['specialty'], df1['review_count']
visit_char = pd.DataFrame() visit_char['idvisit'], visit_char['visit_total_time'], visit_char['visit_total_events'], visit_char['visit_total_actions'], visit_char['visit_last_action_time'], visit_char['visit_first_action_time'] = df1['idvisit'], df1['visit_total_time'], df1['visit_total_events'], df1['visit_total_actions'], df1['visit_last_action_time'], df1['visit_first_action_time']
visitor_char = pd.DataFrame() visitor_char['location_city'], visitor_char['location_region'], visitor_char['location_country'], visitor_char['idvisitor'], visitor_char['visitor_count_visits'] = df1['location_city'], df1['location_region'], df1['location_country'], df1['idvisitor'], df1['visitor_count_visits']
Actions
plt.figure() sns.heatmap(actions.corr(), fmt=".2f") plt.show()
Acquisition
plt.figure() sns.heatmap(acquisition.corr(), fmt=".2f") plt.show()
profile_char
plt.figure() sns.heatmap(profile_char.corr(), fmt=".2f") plt.show()
visit_char
plt.figure() sns.heatmap(visit_char.corr(), fmt=".2f") plt.show()
visitor_char
plt.figure() sns.heatmap(visitor_char.corr(), fmt=".2f") plt.show()
"""## KDE"""
for col in df1: plt.figure() sns.kdeplot(df1[col], shade=True) plt.title(col +' KDE') plt.show()
"""# Time Series Analysis
Indexing
"""
print(df['visit_first_action_time'].dtype)
df = df.set_index('visit_first_action_time') df1 = df1.set_index('visit_first_action_time')
df['weekday'] = df.index.weekday_name df['date'] = df.index.date df['hour'] = df.index.hour df['minute'] = df.index.minute
df1['weekday'] = df1.index.weekday_name df1['date'] = df1.index.date df1['hour'] = df1.index.hour df1['minute'] = df1.index.minute
df['weekday'].value_counts()
df['hour'].value_counts()
"""Time Series with Last Action and Total Time"""
plt.figure() df.plot(x='visit_last_action_time', y='visit_total_time') plt.show()
"""##Total Time"""
df.reset_index() df = df.set_index('index')
df1.reset_index() df1 = df1.set_index('index')
hour - Total Time
df.plot(x='hour', y='visit_total_time')
minute - Total Time
plt.figure() plt.scatter(x=df['minute'], y=df['visit_total_time']) plt.show()
"""## Total Action"""
Weekday - Total Actions
df.plot(x='weekday', y='visit_total_actions')
Date - Total Actions
plt.figure() plt.scatter(x=df['date'], y=df['visit_total_actions']) plt.show()
Hour - Total Actions
Minute - Total Actions
plt.figure() plt.scatter(x=df['hour'], y=df['visit_total_actions']) plt.show()
Minute - Total Actions
plt.figure() plt.scatter(x=df['minute'], y=df['visit_total_actions']) plt.show()
"""## Total Events"""
Date - Total Events
df.plot(x='date', y='visit_total_events')
Weekday - Total Events
plt.figure() plt.scatter(x=df['weekday'], y=df['visit_total_events']) plt.show()
plt.figure() plt.scatter(x=df['minute'], y=df['visit_total_events']) plt.show()
plt.figure() plt.scatter(x=df['hour'], y=df['visit_total_events']) plt.show()
"""# Value Counts
Event Category """
plt.figure() sns.countplot(x=df['idaction_event_category'], data=df) plt.show()
"""Action Name"""
df['idaction_name'].value_counts() / df['idaction_name'].value_counts().sum() *100
"""Event Action"""
df['idaction_event_action'].value_counts() / df['idaction_event_action'].value_counts().sum() *100
plt.figure() sns.countplot(x=df['idaction_event_action'], data=df) plt.show()
df['location'].value_counts()
df['location_region'].value_counts()
plt.figure() sns.countplot(x=df['idaction_name'], data=df) plt.show()
SCRAP
Pie Charts for Event Category
x_graph = df1['idaction_event_category'].unique() labels = ['none','profileView','profile_page','portal_sign_up','search_flow']
plt.figure() plt.title('Event Category') plt.pie(x_graph, autopct='%1.1f%%', labels=labels) plt.axis('equal') plt.show()
"""#Breakdown with Total Time
Profile characteristics
"""
Profile Characteristics Using Total Visit Time
By Location
df.groupby('location', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
By Page Type
df.groupby('page_type', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
By Review_count
df.groupby('review_count', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
By Specialty
df.groupby('specialty', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
By Profile Claimed
df.groupby('profile_claimed', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
"""##Actions"""
Idaction URL
WORK
df.groupby('idaction_url', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
Idaction Name
WORK
df.groupby('idaction_name', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
Idaction Event Category
df.groupby('idaction_event_category', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
Idaction event action
df.groupby('idaction_event_action', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
"""##Acquisition"""
device
df.groupby('device', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
browser
df.groupby('browser', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
organic search engine
df.groupby('organic_search_engine', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
source
df.groupby('source', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
medium
df.groupby('medium', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
channel
df.groupby('channel', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
"""##Visitor Characteristics"""
vistor count visits
df.groupby('visitor_count_visits', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
location country
df.groupby('location_country', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
location region
df.groupby('location_region', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
location city
df.groupby('location_city', as_index=True)['visit_total_time'].mean().sort_values(axis=0, ascending=False)
"""# Breakdown with Total Events
Profile Characteristics
"""
location
df.groupby('location', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
page type
df.groupby('page_type', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
review count
df.groupby('review_count', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
specialty
df.groupby('specialty', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
profile claimed
df.groupby('profile_claimed', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
"""## Actions"""
idaction url
df.groupby('idaction_url', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
idaction name
df.groupby('idaction_name', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
idaction event category
df.groupby('idaction_event_category', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
idaction event action
df.groupby('idaction_event_action', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
"""## Acquisition"""
device
df.groupby('device', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
browser
df.groupby('browser', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
organic search engine
df.groupby('organic_search_engine', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
source
df.groupby('source', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
medium
df.groupby('medium', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
channel
df.groupby('channel', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
"""## Visitor Characteristics"""
vistor count visits
df.groupby('visitor_count_visits', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
location country
df.groupby('location_country', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
location region
df.groupby('location_region', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
location city
df.groupby('location_city', as_index=True)['visit_total_events'].mean().sort_values(axis=0, ascending=False)
"""# Cross-Validation
Variable Selection
"""
df[visit_time_diff]
ATTEMPT TO GET TOTAL SECONDS
df['visit_time_diff'] = df['visit_time_diff'].dt.total_seconds() df['visit_time_diff'].dtype
df1 = df1.reset_index() df1 = df1.dropna() x = df1.drop(columns=['index','visit_total_actions','visit_first_action_time','visit_last_action_time','server_time','idvisitor'], axis=1) y = df1['visit_total_actions']
x
y.shape
"""## Train Test Split"""
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3, random_state=seed)
print(x.shape) print(x_train.shape) print(x_test.shape)
cross = ShuffleSplit(n_splits=5, test_size=0.3, random_state=seed)
"""# Plumbing"""
n_trees = 100 m_features = 12 scaler = StandardScaler()
"""## Ridge"""
clf_ridge = Pipeline([('stan2', scaler), ('pca', PCA(n_components=m_features)), ('ridge', GridSearchCV(Ridge(), param_grid={'alpha': [1, .1, .01 ],'fit_intercept': [True, False], 'normalize':[True,False], 'max_iter':[25,50,100]}, cv=cross, refit=True)) ])
"""## KNN"""
n_neigh = 6 clf_knn = Pipeline([('stan3', scaler), ('feats', PCA(n_components=m_features)), ('knn', GridSearchCV(KNeighborsRegressor(n_neighbors=n_neigh, metric='euclidean'), param_grid={'leaf_size': [3,4,6], 'n_neighbors': [3,4,6],'n_jobs': [1], 'weights': ['uniform','distance']}, cv=cross, refit=True)) ])
"""## Random Forest"""
clf_rand_forest = Pipeline([('stan1', scaler), ('feats', SelectFromModel(RandomForestRegressor(n_estimators=n_trees, max_features=m_features, n_jobs=-1))), ('rf', GridSearchCV(RandomForestRegressor(n_jobs=-1), param_grid={'n_estimators': [4,6,10], 'max_features': [3,4,5,7,10,12],'max_depth': [8,12,20], 'min_samples_split': [6,8,10], 'min_samples_leaf': [2,3,4], 'bootstrap': [True]}, cv=cross, refit=True)) ])
"""## XGB"""
clf_xgb = Pipeline([('stan5', scaler), ('feats',SelectFromModel(XGBRegressor(seed=seed, n_estimators=n_trees, max_depth=7, subsample=.8, learning_rate=.05))), ('xgb',GridSearchCV(XGBRegressor(seed=seed, n_estimators=n_trees, max_depth=7, subsample=.8, learning_rate=.05), param_grid = {'nthread':[4],'learning_rate': [.05, .07, .075, .08 ],'max_depth': [5,6,8],'min_child_weight': [4,5,6],'silent': [1],'subsample': [0.7],'colsample_bytree': [0.7],'n_estimators': [50, 75,100]}, cv=cross, refit=True)) ])
"""## Gradient Boost"""
clf_grad = Pipeline([('stan6', scaler),
('feats',SelectFromModel(GradientBoostingRegressor(random_state=seed, n_estimators=n_trees, max_features=m_features, max_depth=6, learning_rate=.05,subsample=.8))),
('grad', GridSearchCV(GradientBoostingRegressor(random_state=seed, n_estimators=n_trees, max_features=m_features, max_depth=6, learning_rate=.05,subsample=.8),
param_grid = {'learning_rate': [.05, .07,.08], 'n_estimators': [10,25,50,100], 'max_depth': [5,6,7],'min_samples_split': [.2,.3,.6,.7,.8], 'min_samples_leaf': [.1,.2,.3], 'subsample': [0.7], 'max_features':['auto']},
cv=cross,
refit=True))
])
"""# Testing
Keys and Dicts
"""
pipeline_key = {0:'Ridge',1:'KNN',2:'Random Forest Classifier',3:"XGBRegressor",4:'GradientBoosting'} pipelines = [clf_ridge,clf_knn,clf_rand_forest,clf_xgb,clf_grad] grid = {0:'ridge',1:'knn',2:'rf',3:'xgb',4:'grad'}
"""##Fit Loops"""
for idx, clf in enumerate(pipelines):
start_time = clock()
clf.fit(x_train, y_train)
y_pred = clf.predict(x_test)
print(pipeline_key[idx],'R2 Score')
print(r2_score(y_test, y_pred))
print(pipeline_key[idx],'MAE')
print(mean_absolute_error(y_test, y_pred))
print(pipeline_key[idx],'Explained Variance')
print(explained_variance_score(y_test,y_pred))
print(pipeline_key[idx],'Mean Squared Log Error')
print(mean_squared_log_error(y_test, y_pred))
print("pipeline test error = "'{}'.format((((abs(y_pred-y_test)/y_pred) / len(y_pred))*100).mean())+" percent")
#print("Tuned rf best params: {}".format(clf.get_params()))
print( clock() - start_time, "seconds")
print('______________________________________________________________________________________________________________')
for cols in df1: if np.all(np.isfinite(df1[col])): print(col)
"""##Accuracy Loops"""
for idx, val in enumerate(pipelines): print('%s pipeline test accuracy: %.3f' % (pipeline_key[idx], val.score(x_test, y_test)))
PRIZES - Most Technical, Best for CareDash
