About

import json import joblib from datasets import Dataset from matplotlib import pyplot as plt from sklearn.calibration import CalibratedClassifierCV, calibration_curve from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics import classification_report, roc_auc_score, confusion_matrix, precision_recall_curve, auc import numpy as np import pandas as pd from sklearn.linear_model import LogisticRegression, LinearRegression from sklearn.ensemble import GradientBoostingClassifier from sklearn.model_selection import train_test_split from modeling.ClasicModels.performance_utils import plot_roc_curve, plot_pr_curve, plt_show, get_model_performance, \ get_threshold, evaluate_new, print_statistics # Function to conditionally merge dataframes based on flags def conditional_merge(base_df, merge_df, merge_flag, merge_col='PATIENT_UNIQUE_NUMBER'): if merge_flag: base_df = pd.merge(base_df, merge_df, left_on='id', right_on=merge_col, how='left') base_df.drop(columns=[merge_col], inplace=True) return base_df def read_observation_data_conditionally(file_tuple, path, flag): if flag: data = pd.read_csv(path + file_tuple[0], low_memory=False) data[file_tuple[1]] = data[file_tuple[1]].astype(str) return data return None # Flags to enable/disable merging of specific dataframes observation_data_flag = True pharmacy_observations_flag = True radiology_observations_flag = True vital_observations_flag = True speciality_and_facilitiy_observations_flag = True visit_observations_flag = True lab_observations_flag = True prediction_mode = True train_mode = False model_id = 'd' # Define file and column name tuples based on train_mode flag if train_mode: observation_path = r"C:/Users/dev1/Documents/CHI_Projects/data_v4/" patient_data = ('p_data_d_v2.csv', 'PATIENT_UNIQUE_NUMBER') observation_data = ('basic_observations.csv', 'PATIENT_UNIQUE_NUMBER') pharmacy_observations = ('Pharmacy_observations.csv', 'PATIENT_UNIQUE_NUMBER') radiology_observations = ('Radiology_observations.csv', 'PATIENT_UNIQUE_NUMBER') vital_observations = ('Vital_observations.csv', 'patient_unique_number') speciality_and_facility_observations = ('speciality_and_facility_codes_observations.csv', 'patient_unique_number') visit_observations = ('reason_for_visit_observations.csv', 'patient_unique_number') lab_observations = ('LAB_INFO_observations.csv', 'patient_unique_number') else: observation_path = r"C:/Users/dev1/Documents/CHI_Projects/data_2023/" patient_data = ('p_data_pred_d.csv', 'PATIENT_UNIQUE_NUMBER') observation_data = ('basic_observations_2023.csv', 'PATIENT_UNIQUE_NUMBER') pharmacy_observations = ('pharmacy_observations_2023.csv', 'patient_id') radiology_observations = ('radiology_observations_2023.csv', 'patient_id') vital_observations = ('vital_observations_2023.csv', 'patient_id') speciality_and_facility_observations = ('facility_speciality_observations_2023.csv', 'patient_id') visit_observations = ('visit_reason_observations_2023.csv', 'patient_id') lab_observations = ('lab_observations_2023.csv', 'patient_id') # Read the main dataframe df = pd.read_csv(observation_path + patient_data[0], low_memory=False) df['PATIENT_UNIQUE_NUMBER'] = df[patient_data[1]].astype(str) df.columns = ['id', 'label'] if len(df.columns) == 2 else ['id'] # Function to read observation data observation_data_df = read_observation_data_conditionally(observation_data, observation_path, observation_data_flag) pharmacy_observations_df = read_observation_data_conditionally(pharmacy_observations, observation_path, pharmacy_observations_flag) radiology_observations_df = read_observation_data_conditionally(radiology_observations, observation_path, radiology_observations_flag) vital_observations_df = read_observation_data_conditionally(vital_observations, observation_path, vital_observations_flag) speciality_and_facility_observations_df = read_observation_data_conditionally(speciality_and_facility_observations, observation_path, speciality_and_facilitiy_observations_flag) visit_observations_df = read_observation_data_conditionally(visit_observations, observation_path, visit_observations_flag) lab_observations_df = read_observation_data_conditionally(lab_observations, observation_path, lab_observations_flag) if train_mode: train_df, test_df = train_test_split(df, test_size=0.3, random_state=42) else: test_df = df train_df = pd.DataFrame(columns=df.columns) train_df['id'] = train_df['id'].str.strip() test_df['id'] = test_df['id'].str.strip() take_sample = False if take_sample: true_rows = train_df[train_df['label'] == 1] false_rows = train_df[train_df['label'] == 0] sample_size = min(len(true_rows), len(false_rows)) false_sample = false_rows.sample(n=sample_size, random_state=42) train_df = pd.concat([true_rows, false_sample]) # test_df = test_df.sample(frac = 0.1) print('train_df.shape', train_df.shape) train_df = conditional_merge(train_df, observation_data_df, observation_data_flag, merge_col=observation_data[1]) train_df = conditional_merge(train_df, pharmacy_observations_df, pharmacy_observations_flag, merge_col=pharmacy_observations[1]) train_df = conditional_merge(train_df, radiology_observations_df, radiology_observations_flag, merge_col=radiology_observations[1]) train_df = conditional_merge(train_df, vital_observations_df, vital_observations_flag, merge_col=vital_observations[1]) train_df = conditional_merge(train_df, speciality_and_facility_observations_df, speciality_and_facilitiy_observations_flag, merge_col=speciality_and_facility_observations[1]) train_df = conditional_merge(train_df, visit_observations_df, visit_observations_flag, merge_col=visit_observations[1]) train_df = conditional_merge(train_df, lab_observations_df, lab_observations_flag, merge_col=lab_observations[1]) print('train_df.shape', train_df.shape) test_df = conditional_merge(test_df, observation_data_df, observation_data_flag, merge_col=observation_data[1]) test_df = conditional_merge(test_df, pharmacy_observations_df, pharmacy_observations_flag, merge_col=pharmacy_observations[1]) test_df = conditional_merge(test_df, radiology_observations_df, radiology_observations_flag, merge_col=radiology_observations[1]) test_df = conditional_merge(test_df, vital_observations_df, vital_observations_flag, merge_col=vital_observations[1]) test_df = conditional_merge(test_df, speciality_and_facility_observations_df, speciality_and_facilitiy_observations_flag, merge_col=speciality_and_facility_observations[1]) test_df = conditional_merge(test_df, visit_observations_df, visit_observations_flag, merge_col=visit_observations[1]) test_df = conditional_merge(test_df, lab_observations_df, lab_observations_flag, merge_col=lab_observations[1]) print('statistics for train') print_statistics(train_df) print('statistics for test') print_statistics(test_df) train_data = Dataset.from_pandas(train_df) eval_data = Dataset.from_pandas(test_df) def preprocess_function(batch): # Extract the list of ids observation_data_columns = ['PRINICIPAL_DIAGNOSIS', 'SECONDARY_DIAGNOSIS', 'AGE_CATEGORY', 'MARITAL_STATUS', 'GENDER', 'BLOOD_GROUP'] text_parts = [] if observation_data_flag: text_parts.extend([batch[col] for col in observation_data_columns]) if pharmacy_observations_flag: text_parts.append(batch['pharamacy_observations']) if radiology_observations_flag: text_parts.append(batch['radiology_observations']) if speciality_and_facilitiy_observations_flag: text_parts.append(batch['facility_and_specialty_codes_observations']) if vital_observations_flag: text_parts.append(batch['vital_observations']) if visit_observations_flag: text_parts.append(batch['reason_for_visit']) if lab_observations_flag: text_parts.append(batch['lab_info_ovservations']) batch['text'] = ['\n'.join([str(part) if part else "" for part in parts]) for parts in zip(*text_parts)] return batch train_data = train_data.map(preprocess_function, batched=True, batch_size=1000) eval_data = eval_data.map(preprocess_function, batched=True, batch_size=1000) if train_mode: vectorizer = TfidfVectorizer() patient_history_train = vectorizer.fit_transform(train_data['text']) joblib.dump(vectorizer, f'vectorizer_{model_id}.pkl') else: vectorizer = joblib.load(f'vectorizer_{model_id}.pkl') patient_history_test = vectorizer.transform(eval_data['text']) if train_mode: print('start training') model = LogisticRegression(class_weight='balanced', C=10) model.fit(patient_history_train, train_df['label']) print('end training') joblib.dump(model, f'model_{model_id}.pkl') else: model = joblib.load(f'model_{model_id}.pkl') predictions = model.predict_proba(patient_history_test) print('end predictions') predictions = predictions[:, 1] ################################# if prediction_mode: df_eval = pd.DataFrame({'patient_id': test_df['id'], 'y_pred': predictions}) df_eval.to_csv(f'predictions_{model_id}.csv') if train_mode: cm = confusion_matrix(test_df['label'], np.round(predictions)) print("Confusion Matrix:") print(cm) ############################################### df_eval = pd.DataFrame({'patient_id': test_df['id'], 'y_truth': test_df['label'], 'y_pred': predictions}) auprc, auroc = evaluate_new(df_eval, y_truth_column_name='y_truth', y_pred_column_name='y_pred') print('auprc : ', auprc) print('auroc : ', auroc) ############ CalibratedClassifierCV ##################### print('CalibratedClassifier .. ') calibrated_clf = CalibratedClassifierCV(estimator=model, method='isotonic', cv= 'prefit') calibrated_clf.fit(patient_history_train, train_df['label']) # Predictions and calibration curve print('Predictions and calibration curve') y_pred = calibrated_clf.predict_proba(patient_history_test)[:, 1] fraction_of_positives, mean_predicted_value = calibration_curve(test_df['label'], y_pred, n_bins=10) df_eval['calibrated_pred'] = y_pred auprc, auroc = evaluate_new(df_eval, y_truth_column_name='y_truth', y_pred_column_name='calibrated_pred') print('auprc : ', auprc) print('auroc : ', auroc) # Reshape for linear regression mean_predicted_value = mean_predicted_value.reshape(-1, 1) reg = LinearRegression().fit(mean_predicted_value, fraction_of_positives) slope = reg.coef_[0] intercept = reg.intercept_ print(f"Calibration Slope: {slope}") print(f"Calibration Intercept: {intercept}") df_eval.to_csv(f'valid_predictions_{model_id}.csv')

Language

Javascript

Rating

Voted: 0 by 0 user(s)

How to Setup Snippet

import json import joblib from datasets import Dataset from matplotlib import pyplot as plt from sklearn.calibration import CalibratedClassifierCV, calibration_curve from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics import classification_report, roc_auc_score, confusion_matrix, precision_recall_curve, auc import numpy as np import pandas as pd from sklearn.linear_model import LogisticRegression, LinearRegression from sklearn.ensemble import GradientBoostingClassifier from sklearn.model_selection import train_test_split from modeling.ClasicModels.performance_utils import plot_roc_curve, plot_pr_curve, plt_show, get_model_performance, \ get_threshold, evaluate_new, print_statistics # Function to conditionally merge dataframes based on flags def conditional_merge(base_df, merge_df, merge_flag, merge_col='PATIENT_UNIQUE_NUMBER'): if merge_flag: base_df = pd.merge(base_df, merge_df, left_on='id', right_on=merge_col, how='left') base_df.drop(columns=[merge_col], inplace=True) return base_df def read_observation_data_conditionally(file_tuple, path, flag): if flag: data = pd.read_csv(path + file_tuple[0], low_memory=False) data[file_tuple[1]] = data[file_tuple[1]].astype(str) return data return None # Flags to enable/disable merging of specific dataframes observation_data_flag = True pharmacy_observations_flag = True radiology_observations_flag = True vital_observations_flag = True speciality_and_facilitiy_observations_flag = True visit_observations_flag = True lab_observations_flag = True prediction_mode = True train_mode = False model_id = 'd' # Define file and column name tuples based on train_mode flag if train_mode: observation_path = r"C:/Users/dev1/Documents/CHI_Projects/data_v4/" patient_data = ('p_data_d_v2.csv', 'PATIENT_UNIQUE_NUMBER') observation_data = ('basic_observations.csv', 'PATIENT_UNIQUE_NUMBER') pharmacy_observations = ('Pharmacy_observations.csv', 'PATIENT_UNIQUE_NUMBER') radiology_observations = ('Radiology_observations.csv', 'PATIENT_UNIQUE_NUMBER') vital_observations = ('Vital_observations.csv', 'patient_unique_number') speciality_and_facility_observations = ('speciality_and_facility_codes_observations.csv', 'patient_unique_number') visit_observations = ('reason_for_visit_observations.csv', 'patient_unique_number') lab_observations = ('LAB_INFO_observations.csv', 'patient_unique_number') else: observation_path = r"C:/Users/dev1/Documents/CHI_Projects/data_2023/" patient_data = ('p_data_pred_d.csv', 'PATIENT_UNIQUE_NUMBER') observation_data = ('basic_observations_2023.csv', 'PATIENT_UNIQUE_NUMBER') pharmacy_observations = ('pharmacy_observations_2023.csv', 'patient_id') radiology_observations = ('radiology_observations_2023.csv', 'patient_id') vital_observations = ('vital_observations_2023.csv', 'patient_id') speciality_and_facility_observations = ('facility_speciality_observations_2023.csv', 'patient_id') visit_observations = ('visit_reason_observations_2023.csv', 'patient_id') lab_observations = ('lab_observations_2023.csv', 'patient_id') # Read the main dataframe df = pd.read_csv(observation_path + patient_data[0], low_memory=False) df['PATIENT_UNIQUE_NUMBER'] = df[patient_data[1]].astype(str) df.columns = ['id', 'label'] if len(df.columns) == 2 else ['id'] # Function to read observation data observation_data_df = read_observation_data_conditionally(observation_data, observation_path, observation_data_flag) pharmacy_observations_df = read_observation_data_conditionally(pharmacy_observations, observation_path, pharmacy_observations_flag) radiology_observations_df = read_observation_data_conditionally(radiology_observations, observation_path, radiology_observations_flag) vital_observations_df = read_observation_data_conditionally(vital_observations, observation_path, vital_observations_flag) speciality_and_facility_observations_df = read_observation_data_conditionally(speciality_and_facility_observations, observation_path, speciality_and_facilitiy_observations_flag) visit_observations_df = read_observation_data_conditionally(visit_observations, observation_path, visit_observations_flag) lab_observations_df = read_observation_data_conditionally(lab_observations, observation_path, lab_observations_flag) if train_mode: train_df, test_df = train_test_split(df, test_size=0.3, random_state=42) else: test_df = df train_df = pd.DataFrame(columns=df.columns) train_df['id'] = train_df['id'].str.strip() test_df['id'] = test_df['id'].str.strip() take_sample = False if take_sample: true_rows = train_df[train_df['label'] == 1] false_rows = train_df[train_df['label'] == 0] sample_size = min(len(true_rows), len(false_rows)) false_sample = false_rows.sample(n=sample_size, random_state=42) train_df = pd.concat([true_rows, false_sample]) # test_df = test_df.sample(frac = 0.1) print('train_df.shape', train_df.shape) train_df = conditional_merge(train_df, observation_data_df, observation_data_flag, merge_col=observation_data[1]) train_df = conditional_merge(train_df, pharmacy_observations_df, pharmacy_observations_flag, merge_col=pharmacy_observations[1]) train_df = conditional_merge(train_df, radiology_observations_df, radiology_observations_flag, merge_col=radiology_observations[1]) train_df = conditional_merge(train_df, vital_observations_df, vital_observations_flag, merge_col=vital_observations[1]) train_df = conditional_merge(train_df, speciality_and_facility_observations_df, speciality_and_facilitiy_observations_flag, merge_col=speciality_and_facility_observations[1]) train_df = conditional_merge(train_df, visit_observations_df, visit_observations_flag, merge_col=visit_observations[1]) train_df = conditional_merge(train_df, lab_observations_df, lab_observations_flag, merge_col=lab_observations[1]) print('train_df.shape', train_df.shape) test_df = conditional_merge(test_df, observation_data_df, observation_data_flag, merge_col=observation_data[1]) test_df = conditional_merge(test_df, pharmacy_observations_df, pharmacy_observations_flag, merge_col=pharmacy_observations[1]) test_df = conditional_merge(test_df, radiology_observations_df, radiology_observations_flag, merge_col=radiology_observations[1]) test_df = conditional_merge(test_df, vital_observations_df, vital_observations_flag, merge_col=vital_observations[1]) test_df = conditional_merge(test_df, speciality_and_facility_observations_df, speciality_and_facilitiy_observations_flag, merge_col=speciality_and_facility_observations[1]) test_df = conditional_merge(test_df, visit_observations_df, visit_observations_flag, merge_col=visit_observations[1]) test_df = conditional_merge(test_df, lab_observations_df, lab_observations_flag, merge_col=lab_observations[1]) print('statistics for train') print_statistics(train_df) print('statistics for test') print_statistics(test_df) train_data = Dataset.from_pandas(train_df) eval_data = Dataset.from_pandas(test_df) def preprocess_function(batch): # Extract the list of ids observation_data_columns = ['PRINICIPAL_DIAGNOSIS', 'SECONDARY_DIAGNOSIS', 'AGE_CATEGORY', 'MARITAL_STATUS', 'GENDER', 'BLOOD_GROUP'] text_parts = [] if observation_data_flag: text_parts.extend([batch[col] for col in observation_data_columns]) if pharmacy_observations_flag: text_parts.append(batch['pharamacy_observations']) if radiology_observations_flag: text_parts.append(batch['radiology_observations']) if speciality_and_facilitiy_observations_flag: text_parts.append(batch['facility_and_specialty_codes_observations']) if vital_observations_flag: text_parts.append(batch['vital_observations']) if visit_observations_flag: text_parts.append(batch['reason_for_visit']) if lab_observations_flag: text_parts.append(batch['lab_info_ovservations']) batch['text'] = ['\n'.join([str(part) if part else "" for part in parts]) for parts in zip(*text_parts)] return batch train_data = train_data.map(preprocess_function, batched=True, batch_size=1000) eval_data = eval_data.map(preprocess_function, batched=True, batch_size=1000) if train_mode: vectorizer = TfidfVectorizer() patient_history_train = vectorizer.fit_transform(train_data['text']) joblib.dump(vectorizer, f'vectorizer_{model_id}.pkl') else: vectorizer = joblib.load(f'vectorizer_{model_id}.pkl') patient_history_test = vectorizer.transform(eval_data['text']) if train_mode: print('start training') model = LogisticRegression(class_weight='balanced', C=10) model.fit(patient_history_train, train_df['label']) print('end training') joblib.dump(model, f'model_{model_id}.pkl') else: model = joblib.load(f'model_{model_id}.pkl') predictions = model.predict_proba(patient_history_test) print('end predictions') predictions = predictions[:, 1] ################################# if prediction_mode: df_eval = pd.DataFrame({'patient_id': test_df['id'], 'y_pred': predictions}) df_eval.to_csv(f'predictions_{model_id}.csv') if train_mode: cm = confusion_matrix(test_df['label'], np.round(predictions)) print("Confusion Matrix:") print(cm) ############################################### df_eval = pd.DataFrame({'patient_id': test_df['id'], 'y_truth': test_df['label'], 'y_pred': predictions}) auprc, auroc = evaluate_new(df_eval, y_truth_column_name='y_truth', y_pred_column_name='y_pred') print('auprc : ', auprc) print('auroc : ', auroc) ############ CalibratedClassifierCV ##################### print('CalibratedClassifier .. ') calibrated_clf = CalibratedClassifierCV(estimator=model, method='isotonic', cv= 'prefit') calibrated_clf.fit(patient_history_train, train_df['label']) # Predictions and calibration curve print('Predictions and calibration curve') y_pred = calibrated_clf.predict_proba(patient_history_test)[:, 1] fraction_of_positives, mean_predicted_value = calibration_curve(test_df['label'], y_pred, n_bins=10) df_eval['calibrated_pred'] = y_pred auprc, auroc = evaluate_new(df_eval, y_truth_column_name='y_truth', y_pred_column_name='calibrated_pred') print('auprc : ', auprc) print('auroc : ', auroc) # Reshape for linear regression mean_predicted_value = mean_predicted_value.reshape(-1, 1) reg = LinearRegression().fit(mean_predicted_value, fraction_of_positives) slope = reg.coef_[0] intercept = reg.intercept_ print(f"Calibration Slope: {slope}") print(f"Calibration Intercept: {intercept}") df_eval.to_csv(f'valid_predictions_{model_id}.csv')

Codevault

Scroll down to see more snippets from this codevault.

Wordpress Compatability

The author has indicated that this snippet is compatable up to wordpress version: 6.4

Code Snippet Plugin Sync

Free & Pro

Download this snippet by clicking the download button, then head over to the Code Snippet Plugin settings in your wordpress admin dashboard, select the import menu then upload this file to import into your wordpress site.

Pro Only (Coming Soon)

You will be able to click a button and sync this snippet to your wordpress site automatically and from your dashboard manage all code snippets across all your wordpress sites that have the Code Snippets Pro plugin installed.

History

Last modified:

25/06/2024