Caffe always returns one label

I have trained a model with caffe tools under bin and now I am trying to do testing using python script, I read in an image and preprocess it myself (as I did for my training dataset) and I load the pretrained weights to the net, but I am almost always (99.99% of the time) receiving the same result -0- for every test image. I did consider that my model might be overfitting but after training a few models, I have come to realize the labels I get from predictions are most likely the cause. I have also increased dropout and took random crops to overcome overfitting and I have about 60K for training. The dataset is also roughly balanced. I get between 77 to 87 accuracy during evaluation step of training (depending on how I process data, what architecture I use etc)

Excuse my super hacky code, I have been distant to caffe testing for some time so I suspect the problem is how I pass the input data to the network, but I can't put my finger on it:

import h5py, os
import sys
sys.path.append("/home/X/Desktop/caffe-caffe-0.16/python")
from caffe.io import oversample
from caffe.io import resize_image
import caffe
from random import randint
import numpy as np
import cv2
import matplotlib.pyplot as plt
from collections import Counter as Cnt

meanImg = cv2.imread('/home/caffe/data/Ch/Final_meanImg.png')

model_def = '/home/X/Desktop/caffe-caffe-0.16/models/bvlc_googlenet/deploy.prototxt'
model_weights = '/media/X/DATA/SDet/Google__iter_140000.caffemodel'

# load the model
#caffe.set_mode_gpu()
#caffe.set_device(0)

net = caffe.Net(model_def,      # defines the structure of the model
                model_weights,  # contains the trained weights
                caffe.TEST)     # use test mode (e.g., don't perform dropout)


with open( '/home/caffe/examples/sdet/SDet/test_random.txt', 'r' ) as T, open('/media/X/DATA/SDet/results/testResults.txt','w') as testResultsFile:


    readImgCounter = 0
    runningCorrect = 0
    runningAcc = 0.0

    #testResultsFile.write('filename'+' '+'prediction'+' '+'GT')
    lines = T.readlines()

    for i,l in enumerate(lines):


        sp = l.split(' ')

        video = sp[0].split('_')[0]

        impath =  '/home/caffe/data/Ch/images/'+video+'/'+sp[0] +'.jpg' 

        img = cv2.imread(impath)

        resized_img = resize_image(img, (255,255))

        oversampledImages = oversample([resized_img], (224,224)) #5 crops x 2 mirror flips = return 10 images

        transposed_img = np.zeros( (10, 3, 224, 224), dtype='f4' ) 
        tp =  np.zeros( (1, 3, 224, 224), dtype='f4' ) 

        predictedLabels = []

        for j in range(0,oversampledImages.shape[0]-1):
            transposed_img[j] = oversampledImages[j].transpose((2,0,1))


            tp[0] = transposed_img[j]
            net.blobs['data'].data[0]  = tp

            pred =  net.forward(data=tp)

            predictedLabels.append(pred['prob'].argmax())


        print(predictedLabels)

        prediction,num_most_common = Cnt(predictedLabels).most_common(1)[0]


        print(prediction)

        readImgCounter = readImgCounter + 1

        if (prediction == int(sp[1])):
            runningCorrect = runningCorrect + 1

        runningAcc = runningCorrect / readImgCounter
        print('runningAcc:')
        print(runningAcc)
        print('-----------')
        print('runningCorrect:')
        print(runningCorrect)
        print('-----------')
        print('totalImgRead:')
        print(readImgCounter)
        print('-----------')

        testResultsFile.write(sp[0]+' '+str(prediction)+' '+sp[1])
        testResultsFile.write('\n')