MLP.py

# -*- coding: utf-8 -*-
"""HW1_Q4.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1U1CfuviLLJsDEuQIl7zFaJ6cLIP-OopJ

## Section I

### Download Data
"""

!pip install --upgrade --no-cache-dir gdown
!gdown 17KnUeG8_I9vgGU5mqym2hfFjnJPSEH-X

# from google.colab import files
# uploaded = files.upload()

"""### Read .csv & Call .info"""

import pandas as pd
df = pd.read_csv("/content/CarPrice_Assignment.csv")
df.info()

"""## Section II"""

df.isnull().sum()

for i in df.columns:
  print('Number of NaN in',i,'=',df.isna().sum().sum())

"""## Section II"""

import pandas as pd
import difflib

# list of valid car company names
valid_names = ['alfa-romero', 'audi', 'bmw', 'chevrolet', 'dodge', 'honda', 'isuzu', 'jaguar', 'mazda', 'buick', 'mercury', 'mitsubishi', 'nissan', 'peugeot', 'plymouth', 'porsche', 'renault', 'saab', 'subaru', 'toyota', 'volkswagen', 'vw', 'volvo']

# read the original csv file into a pandas dataframe
df = pd.read_csv('/content/CarPrice_Assignment.csv')

# extract the first word of each CarName into a new CompanyName column
df['CompanyName'] = df['CarName'].str.split().str[0]

# replace any invalid names with the nearest match from the list
df['CompanyName'] = df['CompanyName'].apply(lambda x: difflib.get_close_matches(x, valid_names, n=1)[0])

# Place CompanyName column at first position
# # reorder the columns so that CompanyName is the first column
# df = df[['CompanyName', *df.columns[:-4]]]
cols = df.columns.tolist()
cols = cols[-1:] + cols[:-1]
df = df[cols]

# drop the CarName, car_ID, and symbolling columns
df = df.drop(['CarName', 'car_ID', 'symboling'], axis=1)

# save the modified dataframe to a new csv file
df.to_csv('CarPrice_Assignment1.csv', index=False)

"""## Section III

### Type1: Numeric (1,2,...)
"""

import pandas as pd

# read the csv file
df = pd.read_csv('/content/CarPrice_Assignment1.csv')

# iterate over each column in the dataframe
for col in df.columns:
    # check if the column has a non-numeric data type
    if df[col].dtype == 'object':
        # use the pandas factorize() method to encode the values as integers
        df[col] = pd.factorize(df[col])[0]

# save the updated dataframe to a new csv file
df.to_csv('CarPrice_Assignment2.csv', index=False)

import pandas as pd

# load the csv file
df = pd.read_csv('/content/CarPrice_Assignment2.csv')

# normalize each column by its own maximum value
normalized_df = df.apply(lambda x: x / x.max(), axis=0)

# save the normalized dataframe to a new csv file
normalized_df.to_csv('CarPrice_Assignment3.csv', index=False)

"""### Type1: One-hot (Multi-Column)"""

import pandas as pd

# Load CSV file
df = pd.read_csv('/content/CarPrice_Assignment1.csv')

# Loop through each column in the dataframe
for col in df.columns:
    # Check if the column contains non-numeric values (i.e. objects)
    if df[col].dtype == 'object':
        # Convert non-numeric values to one-hot encoded columns
        df = pd.concat([df, pd.get_dummies(df[col], prefix=col)], axis=1)
        # Drop the original column
        df.drop(columns=[col], inplace=True)

# Save result to a new CSV file
df.to_csv('CarPrice_Assignment4.csv', index=False)

"""### Type1: One-hot (Single-Column)"""

import pandas as pd

# Load CSV file
df = pd.read_csv('/content/CarPrice_Assignment3.csv')

# Group columns by the characters before the first underscore in their headers
groups = {}
for col in df.columns:
    prefix = col.split('_')[0]
    if prefix not in groups:
        groups[prefix] = []
    groups[prefix].append(col)

# Merge columns within each group and remove original columns
for prefix, cols in groups.items():
    if len(cols) > 1:
        new_col = '_'.join([prefix] + [''.join(c.split(prefix+'_')[1:]) for c in cols])
        df[new_col] = df[cols].apply(lambda x: ' '.join(x.astype(str)), axis=1)
        df.drop(columns=cols, inplace=True)

# Save result to a new CSV file
df.to_csv('CarPrice_Assignment5.csv', index=False)

"""## Section IV"""

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Read CSV file into DataFrame
df = pd.read_csv('/content/CarPrice_Assignment2.csv')

# Calculate correlation matrix
corr_matrix = df.corr()

# Create heatmap using seaborn
plt.figure(figsize=(25,25))
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', linewidths=0.5, annot_kws={"size": 8}, fmt='.3f', yticklabels=corr_matrix.columns)

# Adjust font size of annotations
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)

# Adjust margins of PDF file
plt.savefig('PIcS1.pdf', bbox_inches='tight')

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Read CSV file into DataFrame
df = pd.read_csv('/content/CarPrice_Assignment2.csv')

# Select columns to include in correlation matrix
cols = df.columns.tolist()
cols.remove('price')

# Calculate correlation matrix
corr_matrix = df[cols].corrwith(df['price']).sort_values(ascending=False)

# Create heatmap using seaborn
plt.figure(figsize=(2,10))
sns.heatmap(corr_matrix.to_frame(), annot=True, cmap='coolwarm', linewidths=0.5, annot_kws={"size": 12}, fmt='.3f', cbar=False)

# Rotate x-axis tick labels to be horizontal
plt.xticks(rotation=0)

# Adjust font size of annotations
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)

# Adjust margins of PDF file
plt.tight_layout()
plt.savefig('priceCM1.pdf', bbox_inches='tight')

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Read CSV file into DataFrame
df = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Calculate correlation matrix
corr_matrix = df.corr()

# Create heatmap using seaborn
plt.figure(figsize=(25,25))
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', linewidths=0.5, annot_kws={"size": 4}, fmt='.2f', yticklabels=corr_matrix.columns)

# Adjust font size of annotations
plt.xticks(fontsize=8)
plt.yticks(fontsize=8, rotation=0)

# Adjust margins of PDF file
plt.savefig('PIcS2.pdf', bbox_inches='tight')

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Read CSV file into DataFrame
df = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Select columns to include in correlation matrix
cols = df.columns.tolist()
cols.remove('price')

# Calculate correlation matrix
corr_matrix = df[cols].corrwith(df['price']).sort_values(ascending=False)

# Create heatmap using seaborn
plt.figure(figsize=(2,30))
sns.heatmap(corr_matrix.to_frame(), annot=True, cmap='coolwarm', linewidths=0.5, annot_kws={"size": 12}, fmt='.3f', cbar=False)

# Rotate x-axis tick labels to be horizontal
plt.xticks(rotation=0)

# Adjust font size of annotations
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)

# Adjust margins of PDF file
plt.tight_layout()
plt.savefig('priceCM2.pdf', bbox_inches='tight')

"""### Other Codes"""

df = pd.read_csv('/content/CarPrice_Assignment2.csv')

# Generate the correlation matrix
CM = df.corr()

# Create a styled correlation matrix with a coolwarm color map
styled_CM = CM.style.background_gradient(cmap='coolwarm')
CM.style.background_gradient(cmap='coolwarm')

CM_Sort= CM.sort_values(by='price', ascending=False)
CM_Sort.style.background_gradient(cmap='coolwarm').format(precision=3)

CM_Sort.iloc[:,23:24].style.background_gradient(cmap='coolwarm').format(precision=3)

df = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Generate the correlation matrix
CM = df.corr()

# Create a styled correlation matrix with a coolwarm color map
styled_CM = CM.style.background_gradient(cmap='coolwarm')
CM.style.background_gradient(cmap='coolwarm')

CM_Sort= CM.sort_values(by='price', ascending=False)
CM_Sort.style.background_gradient(cmap='coolwarm').format(precision=3)

CM_Sort.iloc[:,13:14].style.background_gradient(cmap='coolwarm').format(precision=3)

corr_matrix = df.corr()
print(corr_matrix.iloc[1])
j=-1
for  i in range(len(corr_matrix.iloc[13])):
  if (corr_matrix.iloc[13][i] == 1):
    continue
  else:
    if (j < corr_matrix.iloc[13][i] ):
      j = corr_matrix.iloc[13][i]
      ii=i

print ('Maximum Correlation Matrix','is', df.columns[ii],'and the value','=',j)

"""## Section VI"""

import pandas as pd
import matplotlib.pyplot as plt

# Load the CSV file into a pandas dataframe
df = pd.read_csv('/content/CarPrice_Assignment2.csv')

# Plot the distribution of the 'price' column as a histogram
plt.figure(figsize=(8, 6))  # Set the size of the figure
plt.subplot(2, 1, 1)  # Create the first subplot
plt.hist(df['price'], bins=50, color='blue')
plt.xlabel('Price')
plt.ylabel('Count')
plt.title('Distribution of Prices')

# Plot a boxplot of the 'price' column
plt.subplot(2, 1, 2)  # Create the second subplot
plt.boxplot(df['price'], vert=False, widths=0.7)
plt.xlabel('Price')
plt.title('Boxplot of Prices')

plt.tight_layout()  # Automatically adjust subplot parameters

# Save the plot as a PDF file with a fit margin
plt.savefig('dist1.pdf', bbox_inches='tight')
plt.show()

# Define the bins for the price ranges
price_ranges = pd.cut(df['price'], bins=range(0, 160000, 8000))
# Calculate the count of prices in each range
price_counts = price_ranges.value_counts().sort_index()
# Display the price range counts
print(price_counts)

import pandas as pd
import matplotlib.pyplot as plt

# Load the CSV file into a pandas dataframe
df = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Plot the distribution of the 'price' column as a histogram
plt.figure(figsize=(8, 6))  # Set the size of the figure
plt.subplot(2, 1, 1)  # Create the first subplot
plt.hist(df['price'], bins=50, color='blue')
plt.xlabel('Price')
plt.ylabel('Count')
plt.title('Distribution of Prices')

# Plot a boxplot of the 'price' column
plt.subplot(2, 1, 2)  # Create the second subplot
plt.boxplot(df['price'], vert=False, widths=0.7)
plt.xlabel('Price')
plt.title('Boxplot of Prices')

plt.tight_layout()  # Automatically adjust subplot parameters

# Save the plot as a PDF file with a fit margin
plt.savefig('dist2.pdf', bbox_inches='tight')
plt.show()

# Define the bins for the price ranges
price_ranges = pd.cut(df['price'], bins=range(0, 160000, 8000))
# Calculate the count of prices in each range
price_counts = price_ranges.value_counts().sort_index()
# Display the price range counts
print(price_counts)

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Load the CSV file into a pandas dataframe
df = pd.read_csv('/content/CarPrice_Assignment2.csv')

# Create a scatter plot of 'price' versus 'enginesize'
sns.regplot(x=df['enginesize'], y=df['price'])
plt.xlabel('Engine Size')
plt.ylabel('Price')
plt.title('Price vs. Engine Size Scatter Plot')

# Save the plot as a PDF file with a fit margin
plt.savefig('pricevsenginesize1.pdf', bbox_inches='tight')

plt.show()

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Load the CSV file into a pandas dataframe
df = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Create a scatter plot of 'price' versus 'enginesize'
sns.regplot(x=df['enginesize'], y=df['price'])
plt.xlabel('Engine Size')
plt.ylabel('Price')
plt.title('Price vs. Engine Size Scatter Plot')

# Save the plot as a PDF file with a fit margin
plt.savefig('pricevsenginesize2.pdf', bbox_inches='tight')

plt.show()

import numpy as np
import re
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
#visualizing house prices
fig = plt.figure(figsize=(10,7))
fig.add_subplot(2,1,1)
sns.histplot(df['price'])
fig.add_subplot(2,1,2)
sns.boxplot(df['price'])
plt.tight_layout()

df_price_sqft= (df.loc[:,['price','enginesize']] - df.loc[:,['price','enginesize']].mean()) / df.loc[:,['price','enginesize']].std()
df_price_sqft

sns.regplot(x= df_price_sqft['enginesize'],y= df_price_sqft['price'])

"""### Other Codes"""

import matplotlib.pyplot as plt
import seaborn as sns
fig = plt.figure(figsize=(10,7))
fig.add_subplot(2,2,1)
sns.distplot(df['price'])
fig.add_subplot(2,2,2)
sns.boxplot(df['price'])
plt.tight_layout()
fig.add_subplot(2,2,3)
sns.scatterplot(df[df.columns[ii]],df['price'])
plt.tight_layout()
plt.show()

"""### E"""

df['year']= df.date.apply(lambda x: x[:4])

df['month']= df.date.apply(lambda x: x[4:6])

df.drop('date', axis= 1, inplace= True)

df.year= df.year.astype(int)
df.month= df.month.astype(int)

df.head()

fig = plt.figure(figsize=(16,5))
fig.add_subplot(1,2,1)
df.groupby('year').mean()['price'].plot()

fig.add_subplot(1,2,2)
df.groupby('month').mean()['price'].plot(color= 'green')

"""## Section VII"""

import pandas as pd
from sklearn.model_selection import train_test_split

# Read data from CSV file into a Pandas dataframe
data = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Split the data into train and test sets
train_data, test_data, train_label, test_label = train_test_split(data.drop(['price'], axis=1), data['price'], test_size=0.15, random_state=42)

# Save train and test data to new CSV files
train_data.to_csv('train_data.csv', index=False)
test_data.to_csv('test_data.csv', index=False)

# Save train and test labels to new CSV files
train_label.to_csv('train_label.csv', index=False)
test_label.to_csv('test_label.csv', index=False)

# Print shape of each set
print(f"train_data shape: {train_data.shape}")
print(f"test_data shape: {test_data.shape}")
print(f"train_label shape: {train_label.shape}")
print(f"test_label shape: {test_label.shape}")

"""### Other Codes"""

#split train & test
data= np.array(df.iloc[:,1:])
np.random.shuffle(data)

X= data[:,1:]
Y= data[:, 0]

split= int(len(data)*0.8)
x_train, x_test= X[:split] , X[split:]
y_train, y_test= Y[:split] , Y[split:]

y_test.shape

y_train= y_train.reshape((164,1))
y_test= y_test.reshape((41,1))

print("shape of X_train : ", x_train.shape)
print("shape of y_train : ", y_train.shape)

print("shape of X_test : ", x_test.shape)
print("shape of y_test : ", y_test.shape)

"""## Section VIII"""

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

# read csv file into a pandas dataframe
data = pd.read_csv('/content/CarPrice_Assignment4.csv')

# extract label column as y
y = data['price']

# extract all other columns as X
X = data.drop(columns=['price'])

# split data into train and test sets with a 85/15 split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=42)

# scale the training data
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train)

# scale the test data using the same scaler used for training data
X_test_scaled = scaler.transform(X_test)

# save train and test data/label in new files
pd.DataFrame(X_train_scaled).to_csv('Xtrain.csv', index=False)
y_train.to_csv('ytrain.csv', index=False)
pd.DataFrame(X_test_scaled).to_csv('Xtest.csv', index=False)
y_test.to_csv('ytest.csv', index=False)

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

# Read the CSV file
data = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Separate the label and data
label = df['price']
data = df.drop('price', axis=1)

# Remove the header from the variables
data = data.values
label = label.values

# Split the data into train/validation/test sets
train_data, test_data, train_label, test_label = train_test_split(data, label, test_size=0.15, random_state=42)
train_data, val_data, train_label, val_label = train_test_split(train_data, train_label, test_size=0.15, random_state=42)

# Scale the data using MinMaxScaler
scaler = MinMaxScaler()
train_data = scaler.fit_transform(train_data)
val_data = scaler.transform(val_data)
test_data = scaler.transform(test_data)

# Print shape of each set
print(f"train_data shape: {train_data.shape}")
print(f"val_data shape: {val_data.shape}")
print(f"test_data shape: {test_data.shape}")
print(f"train_label shape: {train_label.shape}")
print(f"val_label shape: {val_label.shape}")
print(f"test_label shape: {test_label.shape}")

# Save the train/validation/test data/label to new files
pd.DataFrame(train_data).to_csv('/content/finaltrain_data.csv', index=False, header=False)
pd.DataFrame(val_data).to_csv('/content/finalval_data.csv', index=False, header=False)
pd.DataFrame(test_data).to_csv('/content/finaltest_data.csv', index=False, header=False)
pd.DataFrame(train_label).to_csv('/content/finaltrain_label.csv', index=False, header=False)
pd.DataFrame(val_label).to_csv('/content/finalval_label.csv', index=False, header=False)
pd.DataFrame(test_label).to_csv('/content/finaltest_label.csv', index=False, header=False)

"""# Part 2

## Section I
"""

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

"""## Section II & III"""

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__)
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__)
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.001
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adam(model1.parameters(), lr=learning_rate)
criterion1 = nn.MSELoss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adam(model2.parameters(), lr=learning_rate)
criterion2 = nn.MSELoss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adam(model3.parameters(), lr=learning_rate)
criterion3 = nn.MSELoss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

"""## Section IV

### Diff Loss
"""

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__ + ' (MSE loss)')
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__ + ' (MSE loss)')
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.001
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adam(model1.parameters(), lr=learning_rate)
criterion1 = nn.MSELoss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adam(model2.parameters(), lr=learning_rate)
criterion2 = nn.MSELoss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adam(model3.parameters(), lr=learning_rate)
criterion3 = nn.MSELoss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__ + ' (L1 loss)')
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__ + ' (L1 loss)')
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.001
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adam(model1.parameters(), lr=learning_rate)
criterion1 = nn.L1Loss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adam(model2.parameters(), lr=learning_rate)
criterion2 = nn.L1Loss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adam(model3.parameters(), lr=learning_rate)
criterion3 = nn.L1Loss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

"""### Diff Opt."""

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__ + ' (Adam opt.)')
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__ + ' (Adam opt.)')
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.001
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adam(model1.parameters(), lr=learning_rate)
criterion1 = nn.MSELoss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adam(model2.parameters(), lr=learning_rate)
criterion2 = nn.MSELoss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adam(model3.parameters(), lr=learning_rate)
criterion3 = nn.MSELoss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__ + ' (Adagrad opt.)')
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__ + ' (Adagrad opt.)')
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.001
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adagrad(model1.parameters(), lr=learning_rate)
criterion1 = nn.MSELoss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adagrad(model2.parameters(), lr=learning_rate)
criterion2 = nn.MSELoss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adagrad(model3.parameters(), lr=learning_rate)
criterion3 = nn.MSELoss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__ + ' (Adagrad opt.)')
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__ + ' (Adagrad opt.)')
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.01
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adagrad(model1.parameters(), lr=learning_rate)
criterion1 = nn.MSELoss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adagrad(model2.parameters(), lr=learning_rate)
criterion2 = nn.MSELoss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adagrad(model3.parameters(), lr=learning_rate)
criterion3 = nn.MSELoss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

"""## Section V"""

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

# Read the CSV file
data1 = pd.read_csv('/content/CarPrice_Assignment4.csv')

# Separate the label and data
label1 = data1['price']
data1 = data1.drop('price', axis=1)

# Remove the header from the variables
data1 = data1.values
label1 = label1.values

# Split the data into train/validation/test sets
train_data1, test_data1, train_label1, test_label1 = train_test_split(data1, label1, test_size=0.15, random_state=42)
train_data1, val_data1, train_label1, val_label1 = train_test_split(train_data1, train_label1, test_size=0.15, random_state=42)

scaler_y = MinMaxScaler().fit(test_label1.reshape(-1, 1))
scaler_y

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__)
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__)
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.001
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adam(model1.parameters(), lr=learning_rate)
criterion1 = nn.MSELoss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adam(model2.parameters(), lr=learning_rate)
criterion2 = nn.MSELoss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adam(model3.parameters(), lr=learning_rate)
criterion3 = nn.MSELoss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

import random

# Select 5 random indices from the test set
test_indices = random.sample(range(len(test_data)), 5)

# Evaluate the models on the selected test data
for i in test_indices:
    input_data = test_data[i]
    true_label = test_label[i]
    
    model1.load_state_dict(best_model1)
    model1.eval()
    with torch.no_grad():
        output1 = model1(input_data)
        unscaled_output1 = scaler_y.inverse_transform(output1.reshape(-1, 1))
        pred_label1 = unscaled_output1.squeeze().item()
        unscaled_true_label = scaler_y.inverse_transform(true_label.reshape(-1, 1))
        diff1 = abs(pred_label1 - unscaled_true_label.item())
    
    model2.load_state_dict(best_model2)
    model2.eval()
    with torch.no_grad():
        output2 = model2(input_data)
        unscaled_output2 = scaler_y.inverse_transform(output2.reshape(-1, 1))
        pred_label2 = unscaled_output2.squeeze().item()
        unscaled_true_label = scaler_y.inverse_transform(true_label.reshape(-1, 1))
        diff2 = abs(pred_label2 - unscaled_true_label.item())
    
    model3.load_state_dict(best_model3)
    model3.eval()
    with torch.no_grad():
        output3 = model3(input_data)
        unscaled_output3 = scaler_y.inverse_transform(output3.reshape(-1, 1))
        pred_label3 = unscaled_output3.squeeze().item()
        diff3 = abs(pred_label3 - unscaled_true_label.item())
    
    # Print the predicted and true values, and their differences
    print(f"Data {i}:")
    print("MLP1 Predicted Value: {:.4f}, True Value: {:.4f}, Difference: {:.4f}".format(pred_label1, unscaled_true_label.item(), diff1))
    print("MLP2 Predicted Value: {:.4f}, True Value: {:.4f}, Difference: {:.4f}".format(pred_label2, unscaled_true_label.item(), diff2))
    print("MLP3 Predicted Value: {:.4f}, True Value: {:.4f}, Difference: {:.4f}".format(pred_label3, unscaled_true_label.item(), diff3))

import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

# Define the MLP models
class MLP1(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(MLP1, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

class MLP2(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, output_dim):
        super(MLP2, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MLP3(nn.Module):
    def __init__(self, input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim):
        super(MLP3, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim1)
        self.fc2 = nn.Linear(hidden_dim1, hidden_dim2)
        self.fc3 = nn.Linear(hidden_dim2, hidden_dim3)
        self.fc4 = nn.Linear(hidden_dim3, output_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = torch.relu(self.fc3(x))
        x = self.fc4(x)
        return x

# Define the training function
def train(model, optimizer, criterion, train_data, train_label, val_data, val_label, num_epochs=1000):
    train_loss_list = []
    val_loss_list = []
    r2score_list = []
    train_r2score_list = []
    best_model = None
    best_r2score = -1
    
    for epoch in range(num_epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        train_output = model(train_data)
        train_loss = criterion(train_output.squeeze(), train_label)
        train_loss.backward()
        optimizer.step()
        train_loss_list.append(train_loss.item())
        train_r2score = r2_score(train_label, train_output.squeeze().detach().numpy())
        train_r2score_list.append(train_r2score)
        
        # Validation
        model.eval()
        with torch.no_grad():
            val_output = model(val_data)
            val_loss = criterion(val_output.squeeze(), val_label)
            val_loss_list.append(val_loss.item())
            r2score = r2_score(val_label, val_output.squeeze().detach().numpy())
            r2score_list.append(r2score)
            if r2score > best_r2score:
                best_r2score = r2score
                best_model = model.state_dict()
        
        # Print loss and R2 score every 100 epochs
        if (epoch+1) % 5 == 0:
            print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train R2 score: {train_r2score:.4f}, Val R2 score: {r2score:.4f}")
    
    # Plot loss curve and save as pdf
    plt.plot(train_loss_list, label='Train Loss')
    plt.plot(val_loss_list, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    plt.title(type(model).__name__)
    plt.savefig(f"{type(model).__name__}_loss.pdf")
    plt.show()
    
    # Plot R2 score curve and save as pdf
    plt.plot(train_r2score_list, label='Train R2 score')
    plt.plot(r2score_list, label='Val R2 score')
    plt.xlabel('Epoch')
    plt.ylabel('R2 score')
    plt.legend()
    plt.title(type(model).__name__)
    plt.savefig(f"{type(model).__name__}_r2score.pdf")
    plt.show()

    # Return the best model based on the validation R2 score
    return best_model
    
# Define the data and labels
train_data = torch.Tensor(train_data)
train_label = torch.Tensor(train_label)
val_data = torch.Tensor(val_data)
val_label = torch.Tensor(val_label)
test_data = torch.Tensor(test_data)
test_label = torch.Tensor(test_label)

# Define the hyperparameters
input_dim = 23
output_dim = 1
hidden_dim1 = 16
hidden_dim2 = 64
hidden_dim3 = 128
learning_rate = 0.001
num_epochs = 120

# Train the models
model1 = MLP1(input_dim, hidden_dim1, output_dim)
optimizer1 = optim.Adam(model1.parameters(), lr=learning_rate)
criterion1 = nn.MSELoss()
best_model1 = train(model1, optimizer1, criterion1, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model2 = MLP2(input_dim, hidden_dim1, hidden_dim2, output_dim)
optimizer2 = optim.Adam(model2.parameters(), lr=learning_rate)
criterion2 = nn.MSELoss()
best_model2 = train(model2, optimizer2, criterion2, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

model3 = MLP3(input_dim, hidden_dim1, hidden_dim2, hidden_dim3, output_dim)
optimizer3 = optim.Adam(model3.parameters(), lr=learning_rate)
criterion3 = nn.MSELoss()
best_model3 = train(model3, optimizer3, criterion3, train_data, train_label, val_data, val_label, num_epochs=num_epochs)

# Evaluate the models on the test set
model1.load_state_dict(best_model1)
model1.eval()
with torch.no_grad():
    test_output1 = model1(test_data)
    test_loss1 = criterion1(test_output1.squeeze(), test_label)
    test_r2score1 = r2_score(test_label, test_output1.squeeze().detach().numpy())

model2.load_state_dict(best_model2)
model2.eval()
with torch.no_grad():
    test_output2 = model2(test_data)
    test_loss2 = criterion2(test_output2.squeeze(), test_label)
    test_r2score2 = r2_score(test_label, test_output2.squeeze().detach().numpy())

model3.load_state_dict(best_model3)
model3.eval()
with torch.no_grad():
    test_output3 = model3(test_data)
    test_loss3 = criterion3(test_output3.squeeze(), test_label)
    test_r2score3 = r2_score(test_label, test_output3.squeeze().detach().numpy())

# Print the test set results
print("MLP1 Test Set Results:")
print(f"Loss: {test_loss1:.4f}, R2 Score: {test_r2score1:.4f}")

print("MLP2 Test Set Results:")
print(f"Loss: {test_loss2:.4f}, R2 Score: {test_r2score2:.4f}")

print("MLP3 Test Set Results:")
print(f"Loss: {test_loss3:.4f}, R2 Score: {test_r2score3:.4f}")

import random

# Select 5 random indices from the test set
test_indices = random.sample(range(len(test_data)), 5)

# Evaluate the models on the selected test data
for i in test_indices:
    input_data = test_data[i]
    true_label = test_label[i]
    
    model1.load_state_dict(best_model1)
    model1.eval()
    with torch.no_grad():
        output1 = model1(input_data)
        pred_label1 = output1.squeeze().item()
        diff1 = abs(pred_label1 - true_label.item())
    
    model2.load_state_dict(best_model2)
    model2.eval()
    with torch.no_grad():
        output2 = model2(input_data)
        pred_label2 = output2.squeeze().item()
        diff2 = abs(pred_label2 - true_label.item())
    
    model3.load_state_dict(best_model3)
    model3.eval()
    with torch.no_grad():
        output3 = model3(input_data)
        pred_label3 = output3.squeeze().item()
        diff3 = abs(pred_label3 - true_label.item())
    
    # Print the predicted and true values, and their differences
    print(f"Data {i}:")
    print("MLP1 Predicted Value: {:.4f}, True Value: {:.4f}, Difference: {:.4f}".format(pred_label1, true_label.item(), diff1))
    print("MLP2 Predicted Value: {:.4f}, True Value: {:.4f}, Difference: {:.4f}".format(pred_label2, true_label.item(), diff2))
    print("MLP3 Predicted Value: {:.4f}, True Value: {:.4f}, Difference: {:.4f}".format(pred_label3, true_label.item(), diff3))