PyTorch: change in validation and test accuracies during and after training respectively

Hi I am learning deep learning and I am trying to use the RNN with train, test and validation sets on time series finance data. Below is my code :

def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']


# In[63]:


def train_model(epoch, model, optimizer, train_loader):
    model.train()
    t0 = time.time()
    correct = 0
    total = 0
    final_loss = 0
    for batch_idx, (X,labels) in enumerate(train_loader):
        data,labels = map(lambda x: Variable(x), [X,labels])
        optimizer.zero_grad()
        output = model(data)
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        ##print('device : ', device)
        final_output=output.to(device)
        loss = F.cross_entropy(final_output, labels)
        final_loss += loss.item()
        loss.backward()
        optimizer.step()
        print('predicted labels',final_output.squeeze())
        #print('Actual labels',labels.squeeze())
        print('Train Epoch: {} Batch: {} [{}/{} ({:.2f}%, time:{:.2f}s)]\tBatch Loss: {:.6f}'.format(
                epoch, batch_idx, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), time.time() - t0,
                final_loss))
                ##avg_loss))
        _, predicted = torch.max(output.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
        t0 = time.time()
    final_loss /= (batch_idx+1)
    accuracy =  100*correct/total
    lr = get_lr(optimizer)
    learning_rates.append(lr)
    print('Training Accuracy : ',accuracy)
    print('Training Loss : ',final_loss)
    print('Learning Rate : ',lr)
    if epoch%epoch_interval == 0 or epoch ==1 or epoch == epochs: 
        path = base_path + 'models/RNN/rnn_'
        torch.save(model,path+str(epoch)+'.pth')
        ##torch.save(model,path)
        print('model saved')
    if epoch%plot_epoch_interval == 0 or epoch ==1 or epoch == epochs: 
        epochs_list.append(epoch)
        train_loss.append(final_loss)
        train_accuracies.append(accuracy)
    return lr,final_loss,accuracy


# In[166]:


def validate(epoch,model, val_loader,optimizer):
    model.eval()
    val_loss = 0
    correct = 0
    total = 0
    loss = 0
    ypred,ytrue,scores = [],[],[]
    for batch_idx,(X,labels) in enumerate(val_loader):
        data,labels = map(lambda x: Variable(x), [X,labels])
        output = model(data)
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        final_val_output=output.to(device)
        val_loss += F.cross_entropy(final_val_output, labels)  # sum up batch loss
        _, predicted = torch.max(output.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
        ypred.extend(predicted.tolist())
        ytrue.extend(labels.tolist())
        scores.extend(output.tolist())
    val_loss /= (batch_idx+1)
    accuracy =  100*correct/total
    if epoch%plot_epoch_interval == 0 or epoch ==1 or epoch == epochs: 
        validation_loss.append(val_loss.item())
        val_accuracies.append(accuracy)

    print('Accuracy : ',accuracy)
    print('\nVal set: Average loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\n'.format(
        val_loss, correct,total,accuracy))
    print("==============================================")
    return "{:.4f}%".format(100.* correct / total), accuracy,loss,ypred,ytrue,scores


# In[276]:

def test(data_loader,model):
    torch.manual_seed(1)
    np.random.seed(1)
    #data_loader = DataLoader(FinancialData(xtest,ytest), batch_size = batch_size, shuffle = False)
    model = torch.load(path)
    model.eval()
    for params in model.parameters():
         print(params)
    val_loss = 0
    correct = 0
    total = 0
    loss = 0
    ypred,ytrue,scores = [],[],[]
    with torch.no_grad():
        for batch_idx,(X,labels) in enumerate(data_loader):
            data,labels = map(lambda x: Variable(x), [X,labels])
            output = model(data)
    #         device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    #         final_val_output=output.to(device)
    #         val_loss += F.cross_entropy(final_val_output, labels)  # sum up batch loss
            _, predicted = torch.max(output.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
            ypred.extend(predicted.tolist())
            ytrue.extend(labels.tolist())
            scores.extend(output.tolist())
            accuracy =  100*correct/total
    #     if epoch%plot_epoch_interval == 0 or epoch ==1 or epoch == epochs: 
    #         validation_loss.append(val_loss.item())
    #         val_accuracies.append(accuracy)

    print('Accuracy : ',accuracy)
#     print('\nVal set: Average loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\n'.format(
#         val_loss, correct,total,accuracy))
    print("==============================================")
    #return "{:.4f}%".format(100.* correct / total), accuracy,loss,ypred,ytrue,scores


# In[288]:


def train_on_batch(lr,epochs,momentum,X_train,Y_train,X_val,Y_val,batch_size):
    cuda=False
    seed=1
    torch.manual_seed(seed)

    train_loader = DataLoader(FinancialData(X_train,Y_train),batch_size=batch_size,shuffle=True) 
    val_loader = DataLoader(FinancialData(X_val,Y_val),batch_size=batch_size,shuffle=False) 
    test_loader = DataLoader(FinancialData(X_test_new,Y_test), batch_size = batch_size, shuffle = False)
    input_size = 1
    hid_size = 10
    num_layers = 2
    num_classes = len(np.unique(Y_train))
    dropRate = 0.0
    bidirection = True
    model = Network(input_size=input_size,hid_size =hid_size,window_size = window_size,num_layers=num_layers,
                num_classes=num_classes,dropRate = dropRate,bidirection=bidirection)
    ypred,ytrue, scores = [],[],[]
    for params in model.parameters():
        print(params)
    optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-4, amsgrad=False)

    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,'max', factor=0.25, patience=6, verbose=True,
                                                            threshold_mode='abs', threshold=0.01, min_lr=1e-6)

    #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', factor=0.5, patience=5,
    #                                                       verbose=True,threshold_mode='abs', threshold=0.01, 
    #                                                       min_lr=1e-6)
    path = base_path + 'models/RNN/rnn_best_model.pth'
    best_val_loss = 0
    best_val_acc = 0
    best_epoch = 0
    best_lr = lr
    best_tr_acc = 0
    for epoch in range(1, epochs + 1):
        tuned_lr,tr_loss,tr_acc = train_model(epoch, model, optimizer, train_loader)
        acc_str, val_acc, val_loss, ypred, ytrue, scores = validate(epoch,model,val_loader,optimizer)
        if val_acc >= best_val_acc:
                torch.save(model,path)
                #best_val_loss = val_loss
                best_val_acc = val_acc
                best_epoch = epoch
                best_lr = tuned_lr
                best_tr_acc = tr_acc
        scheduler.step(val_acc)
        #scheduler.step(tr_acc)
        print('='*100)
#         for params in model.parameters():
#             print(params)
#         print('='*100)
    test(val_loader,model)
    test(test_loader,model)
    #validate(epoch,model,val_loader,optimizer)
    #validate(epoch,model,test_loader,optimizer)
    print('best epoch : {}, best_lr : {}, best_tr_acc : {}, best val_acc : {:.4f}\n'.format(best_epoch,best_lr,best_tr_acc,best_val_acc))    
    scores = np.asarray(scores)
    return tr_acc,val_acc, ypred, ytrue, scores


# In[289]:


cuda=torch.cuda.is_available()


X_train,Y_train,X_val,Y_val,X_test,Y_test = splitDataWithVal(feat_wise_data,labels_new,test_size=0.2,val_size=0.25)

X_train_new, X_val_new, X_test_new = standardizeDataVal(X_train, X_test, X_val, mode = 'Normalizer') 


# # Check for Class Imbalance

# In[292]:


Ytrain_df= pd.DataFrame(Y_train,columns=[0])
print(Ytrain_df.shape)
print(Ytrain_df.columns)
print(Ytrain_df.groupby(0).size())




train_loss = []
validation_loss = []
epochs_list = []
train_accuracies = []
val_accuracies = []
learning_rates = []
epoch_interval = 1#10
plot_epoch_interval = 5



lr = 0.01
momentum = 0.9
epochs = 3
batch_size = 4
print('batch_size : ',batch_size)
tr_acc,val_acc, ypred, ytrue, scores = train_on_batch(lr,epochs,momentum,X_train_new,Y_train,X_val_new,Y_val,batch_size)

I tested it for 3 epochs and saved models after every epoch. However, after 3rd epoch i.e. complete 3 epochs of training, when I test my model by calling test() function of my code, it gives 49.7% validation accuracy and 59.3% test accuracy. Whereas if I use validate() function of my code, it gives 51.146% validation accuracy when called after 3rd epoch of training within training loop. Using validate() function after complete training of 3 epochs ie. outside for loop, I get 49.12% validation accuracy and 54.0697% test accuracy.

Why does validation accuracy change calling the same validate function twice i.e. once inside training epochs loop and other time after training epochs loop ? Also, Which function is correct way of testing and validating, validate() or test()?

I even loaded all the models which I am saving after every epoch and checked their weights which are same as what they were seen during training. Please help as I am new to this domain.