I am trying to write custom dataset and dataloader for pascal-voc-2007. It is a multi-label classification problem. There is csv file to hold the name of the images and their corresponding labels. I want to resize images and create one-hot encoded vectors for labels. Also I don't want to load images to RAM, I want to load them when they appear in mini-batch.

So far I did all of them but with gtx 1070 (I dont't think it is relevant when loading data), and i5 8600k, Samsung M.2 solid state, just loading 1 epoch of data(without feeding to the network, I just iterate over dataloader and look) took 1 min 7s.

Is it normal or can be considered as slow? Or is there some optimization problem in my code?

My code:

d = {}
for e in df["labels"]:
l = e.split(" ")
for k in l:
if k not in d:
d[k] = len(d)

def create_code(df, idx):
l = [0] * len(d)

labels = df.iloc[idx]["labels"].split(" ")
labels_code = []

for e in labels:
l[d[e]] = 1

def transform_image(pth):
img = Image.open(pth)
return (ToTensor()(img.resize((256,256), resample = PIL.Image.ANTIALIAS))).unsqueeze(0).type(torch.float64)

class Dataset():
def __init__(self, df, path):
self.df = df
self.path = path

def __len__(self):
return len(self.df)
def __getitem__(self, idxs):
if isinstance(idxs, int):
imgs = transform_image(self.path/self.df.iloc[idxs]["fname"])
labels = create_code(self.df, idxs)
else:
sub_df = self.df.iloc[idxs]
imgs = transform_image(self.path/sub_df.iloc[0]["fname"])
labels = create_code(sub_df, 0)
for i in range(1, len(idxs)):
e = sub_df.iloc[i]["fname"]
imgs = torch.cat((imgs,transform_image(self.path/e)))
labels = torch.cat((labels, create_code(sub_df, i)))

return imgs, labels

def __init__(self, ds, bs):
self.ds, self.bs = ds, bs
def __iter__(self):
n = len(self.ds)
l = torch.randperm(n)

for i in range(0, n, self.bs):
idxs_l = l[i:i+self.bs]
yield self.ds[idxs_l]



Then:

train_ds = Dataset(df_train, path/"train")