bbb_jack_cvtesting.py

# -*- coding: utf-8 -*-
"""bbb_jack_CVTesting.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1x1UdinRIa6aO3KyeMv0Z65UimkyI7XyW
"""

from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, make_scorer
from sklearn.model_selection import cross_val_score, StratifiedKFold, KFold, LeaveOneOut, train_test_split
from sklearn.ensemble import StackingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import BaggingClassifier, ExtraTreesClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import RidgeClassifier, Perceptron
import pandas as pd
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
# from catboost import CatBoostClassifier
import numpy as np

train=pd.read_csv('/content/train_data77up.csv')
test=pd.read_csv('/content/test_data23up.csv')
df = pd.concat([train,test], axis=0)
df=df.reset_index(drop=True)

# y_train=train['target']
# X_train=train.drop(['class'],axis=1)

# y_test=test['class']
# X_test=test.drop(['class'],axis=1)

y=df['target']
X=df.drop(['target'],axis=1)

from sklearn.preprocessing import StandardScaler
#dataset = pd.read_csv('df.csv', sep=',')
dataset = df
X1 = dataset.drop(['target'],axis=1)
Y1 =dataset['target']
X1 = X1.to_numpy()
Y1 = Y1.to_numpy()
std_scale = StandardScaler().fit(X1)
X1 = std_scale.transform(X1)
X1 = np.nan_to_num(X1.astype('float32'))

X=X1
y=Y1

from sklearn.model_selection import LeaveOneOut
from sklearn.metrics import accuracy_score, matthews_corrcoef, recall_score, confusion_matrix
from xgboost import XGBClassifier
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from lightgbm import LGBMClassifier

# Define the classifiers
classifiers = {
    'XGB': XGBClassifier(),
    'RF': RandomForestClassifier(),
    'ET': ExtraTreesClassifier(),
    'LGBM': LGBMClassifier()
}

# Define the evaluation metrics
metrics = {
    'Accuracy': accuracy_score,
    'MCC': matthews_corrcoef,
    'Recall': recall_score,
    'Specificity': lambda y_true, y_pred: recall_score(y_true, y_pred, pos_label=0)
}


# Perform LOOCV and evaluate metrics for each classifier
for clf_name, clf in classifiers.items():
    print(f"Classifier: {clf_name}")
    loo = LeaveOneOut()
    y_true, y_pred = [], []
    for train_index, test_index in loo.split(X):
        X_train, X_test = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]

        clf.fit(X_train, y_train)
        y_pred.extend(clf.predict(X_test))
        y_true.extend(y_test)

    for metric_name, metric_func in metrics.items():
        metric_value = metric_func(y_true, y_pred)
        print(f"{metric_name}: {metric_value}")
    print('---')



"""**Stacking**"""

import numpy as np
from sklearn.model_selection import LeaveOneOut
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from lightgbm import LGBMClassifier
from sklearn.linear_model import LogisticRegression
from xgboost import XGBClassifier
from sklearn.metrics import matthews_corrcoef



# Define the base classifiers
clfs_st = [
    ExtraTreesClassifier(),
     XGBClassifier(),
]

# Initialize the arrays to store the predictions
stack_train = np.zeros((len(X), len(clfs_st)))  # Number of training data x Number of classifiers
stack_test = np.zeros((len(X), len(clfs_st)))  # Number of testing data x Number of classifiers

# Perform Leave-One-Out cross-validation
loo = LeaveOneOut()

for train_index, test_index in loo.split(X):
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]

    for j, clf in enumerate(clfs_st):
        clf.fit(X_train, y_train)
        stack_train[test_index, j] = clf.predict(X_test)
        stack_test[:, j] += clf.predict(X)

# Define the meta-classifier
meta_clf = LogisticRegression()

# Train the meta-classifier on the stacked features
meta_clf.fit(stack_train, y)

# Make predictions using the meta-classifier
pred = meta_clf.predict(stack_test)

# Compute the accuracy and other metrics
accuracy = accuracy_score(y, pred)
mcc = matthews_corrcoef(y, pred)

conf_mat = confusion_matrix(y, pred)
sensitivity = conf_mat[0, 0] / (conf_mat[0, 0] + conf_mat[0, 1])
specificity = conf_mat[1, 1] / (conf_mat[1, 0] + conf_mat[1, 1])

# Print the results
print("Accuracy:", accuracy)
print("Sensitivity:", sensitivity)
print("Specificity:", specificity)
print("MCC:", mcc)

"""**Blending**"""

import numpy as np
import pandas as pd
from sklearn.model_selection import LeaveOneOut
from sklearn.metrics import accuracy_score, confusion_matrix, roc_curve, auc
from sklearn.ensemble import RandomForestClassifier
from lightgbm import LGBMClassifier
from sklearn.linear_model import LogisticRegression

# Define the classifiers
clfs_bl = [
    RandomForestClassifier(),
    LGBMClassifier(),
]

# Initialize the arrays to store the predictions
blend_train = np.zeros((len(X), len(clfs_bl)))  # Number of training data x Number of classifiers
blend_test = np.zeros((len(X), len(clfs_bl)))  # Number of testing data x Number of classifiers

# Perform Leave-One-Out cross-validation
loo = LeaveOneOut()

for train_index, test_index in loo.split(X):
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]

    for j, clf in enumerate(clfs_bl):
        clf.fit(X_train, y_train)
        blend_train[test_index, j] = clf.predict(X_test)
        blend_test[test_index, j] += clf.predict(X_test)

# Take the majority vote for each sample
pred = np.argmax(blend_test, axis=1)

# Compute the accuracy and other metrics
accuracy = accuracy_score(y, pred)
mcc = matthews_corrcoef(y, pred)
conf_mat = confusion_matrix(y, pred)
sensitivity = conf_mat[0, 0] / (conf_mat[0, 0] + conf_mat[0, 1])
specificity = conf_mat[1, 1] / (conf_mat[1, 0] + conf_mat[1, 1])

# Compute the false positive rate, true positive rate, and thresholds for the ROC curve
fpr_bl, tpr_bl, thresholds = roc_curve(y, blend_test[:, 1])
roc_aucbl = auc(fpr_bl, tpr_bl)

# # Save TPR and FPR to CSV file
# data = {'fpr': fpr_bl, 'tpr': tpr_bl}
# df = pd.DataFrame(data)
# df.to_csv('blroc_curve.csv', index=False)

# Print the results
print("Accuracy:", accuracy)
print("Sensitivity:", sensitivity)
print("Specificity:", specificity)
print("MCC:", mcc)

import numpy as np
from sklearn.ensemble import ExtraTreesClassifier, StackingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_curve, auc,roc_auc_score
from sklearn.model_selection import StratifiedKFold, train_test_split
from xgboost import XGBClassifier

xgb=XGBClassifier(n_estimators=400, max_depth=9, learning_rate=0.1),
et=ExtraTreesClassifier(n_estimators=100, max_depth=None, min_samples_split=2),
lgbm=LGBMClassifier(n_estimators=200, max_depth=5, learning_rate=0.1)
rf=RandomForestClassifier()

# Assuming you have already preprocessed your data and split it into features (X) and labels (y)
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Define the base learners
base_learners = [
    ('ET', ExtraTreesClassifier(n_estimators=100, max_depth=None, min_samples_split=2)),
    ('XGB', XGBClassifier(n_estimators=400, max_depth=9, learning_rate=0.1))
]

# Define the meta-learner
meta_learner = LogisticRegression()

# Define the stacking classifier
stacking_clf = StackingClassifier(estimators=base_learners, final_estimator=meta_learner)

from sklearn.model_selection import LeaveOneOut, cross_val_predict
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from lightgbm import LGBMClassifier
from xgboost import XGBClassifier

lgbm = cross_val_predict(LGBMClassifier(), X, y, cv=LeaveOneOut(), method='predict_proba')
et = cross_val_predict(ExtraTreesClassifier(), X, y, cv=LeaveOneOut(), method='predict_proba')
rf = cross_val_predict(RandomForestClassifier(), X, y, cv=LeaveOneOut(), method='predict_proba')
xgb = cross_val_predict(XGBClassifier(), X, y, cv=LeaveOneOut(), method='predict_proba')
st=cross_val_predict(stacking_clf, X, y,cv=LeaveOneOut(),method='predict_proba')

ada_fpr, ada_tpr, thresholds = roc_curve(y, xgb[:, 1])
ada_auc = auc(ada_fpr, ada_tpr)


lgbm_fpr, lgbm_tpr, thresholds = roc_curve(y, lgbm[:, 1])
lgbm_auc = auc(lgbm_fpr, lgbm_tpr)

rf_fpr, rf_tpr, thresholds = roc_curve(y, rf[:, 1])
rf_auc = auc(rf_fpr, rf_tpr)

# svc_fpr, svc_tpr, thresholds = roc_curve(y, svc[:, 1])
# svc_auc = auc(svc_fpr, svc_tpr)

et_fpr, et_tpr, thresholds = roc_curve(y, et[:, 1])
et_auc = auc(et_fpr, et_tpr)

st_fpr, st_tpr, thresholds = roc_curve(y, st[:, 1])
st_auc = auc(st_fpr, st_tpr)

# replace X1 with X_test and Y1 with y_test
from sklearn.metrics import roc_curve, roc_auc_score
import matplotlib.pyplot as plt


plt.figure(figsize=(20, 10), dpi=600)
plt.plot([0, 1], [0, 1], linestyle="--", lw=2,  label="Chance", alpha=0.8)
plt.plot(rf_fpr, rf_tpr, marker='.', label='RF (auc = %0.3f)' % rf_auc)
plt.plot(et_fpr, et_tpr, marker='.', label='ET (auc = %0.3f)' % et_auc)
# plt.plot(svc_fpr, svc_tpr, linestyle='-', label='SVC (auc = %0.3f)' % svc_auc)
plt.plot(st_fpr, st_tpr, linestyle='-', label='Stacking (ET,XGB) (auc = %0.3f)' % st_auc)
plt.plot(fpr_bl, tpr_bl, linestyle='-', label='Blending (RF,LGBM) (auc = %0.3f)' % roc_aucbl)
plt.plot(ada_fpr, ada_tpr, linestyle='-', label='XGB (auc = %0.3f)' % ada_auc)
plt.plot(lgbm_fpr, lgbm_tpr, linestyle='-',color='black', label='LGBM (auc = %0.3f)' % lgbm_auc)


# plt.xlabel('False Positive Rate -->')
# plt.ylabel('True Positive Rate -->')

plt.legend(loc="lower right", fontsize=20, ncol=1)

plt.show()