diff --git a/README.md b/README.md index a7af1d3..999ab76 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ If you have installed anaconda, you can run the following command : In order to train a model on the PlantNet-300K dataset, run the following command : -```python main.py --lr=0.05 --n_epochs=80 --model=resnet50 --root=path_to_data --save_name_xp=xp1``` +```python main.py --lr=0.05 --n_epochs=80 --k 1 3 5 10 --model=resnet50 --root=path_to_data --save_name_xp=xp1``` You must provide in the "root" option the path to the train val and test folders. The "save_name_xp" option is the name of the directory where the weights of the model and the results (metrics) will be stored. diff --git a/cli.py b/cli.py index 9067e97..0fab429 100644 --- a/cli.py +++ b/cli.py @@ -13,8 +13,6 @@ def add_all_parsers(parser): def _add_loss_parser(parser): group_loss = parser.add_argument_group('Loss parameters') - group_loss.add_argument('--k', type=int, help='value of k for computing the topk loss and calculating topk accuracy', - default=3) group_loss.add_argument('--mu', type=float, default=0., help='weight decay parameter') @@ -23,6 +21,8 @@ def _add_training_parser(parser): group_training.add_argument('--lr', type=float, help='learning rate to use') group_training.add_argument('--batch_size', type=int, default=256, help='default is 256') group_training.add_argument('--n_epochs', type=int) + group_training.add_argument('--k', nargs='+', help='value of k for computing the topk loss and calculating topk accuracy', + required=True, type=int) def _add_model_parser(parser): @@ -38,7 +38,7 @@ def _add_hardware_parser(parser): def _add_dataset_parser(parser): group_dataset = parser.add_argument_group('Dataset parameters') - group_dataset.add_argument('--size_image', type=int, default=128, + group_dataset.add_argument('--size_image', type=int, default=256, help='size you want to resize the images to') diff --git a/epoch.py b/epoch.py index 3b0b22a..241e886 100644 --- a/epoch.py +++ b/epoch.py @@ -2,13 +2,15 @@ import torch from tqdm import tqdm from utils import count_correct_top_k, count_correct_average_k import torch.nn.functional as F +from collections import defaultdict -def train_epoch(model, optimizer, train_loader, criteria, loss_train, train_accuracy, topk_train_accuracy, k, n_train, use_gpu): +def train_epoch(model, optimizer, train_loader, criteria, loss_train, train_accuracy, topk_train_accuracy, list_k, n_train, use_gpu): model.train() loss_epoch_train = 0 n_correct_train = 0 - n_correct_top_k_train = 0 + n_correct_top_k_train = defaultdict(int) + epoch_top_k_accuracy_train = {} for batch_idx, (batch_x_train, batch_y_train) in enumerate(tqdm(train_loader, desc='train', position=0)): if use_gpu: batch_x_train, batch_y_train = batch_x_train.cuda(), batch_y_train.cuda() @@ -21,13 +23,15 @@ def train_epoch(model, optimizer, train_loader, criteria, loss_train, train_accu optimizer.step() with torch.no_grad(): n_correct_train += torch.sum(torch.eq(batch_y_train, torch.argmax(batch_output_train, dim=-1))).item() - n_correct_top_k_train += count_correct_top_k(scores=batch_output_train, labels=batch_y_train, - k=k).item() + for k in list_k: + n_correct_top_k_train[k] += count_correct_top_k(scores=batch_output_train, labels=batch_y_train, + k=k).item() # At the end of epoch compute average of statistics over batches and store them with torch.no_grad(): loss_epoch_train /= batch_idx epoch_accuracy_train = n_correct_train / n_train - epoch_top_k_accuracy_train = n_correct_top_k_train / n_train + for k in list_k: + epoch_top_k_accuracy_train[k] = n_correct_top_k_train[k] / n_train loss_train.append(loss_epoch_train), train_accuracy.append(epoch_accuracy_train), topk_train_accuracy.append( epoch_top_k_accuracy_train) @@ -35,14 +39,17 @@ def train_epoch(model, optimizer, train_loader, criteria, loss_train, train_accu return loss_epoch_train, epoch_accuracy_train, epoch_top_k_accuracy_train -def val_epoch(model, val_loader, criteria, loss_val, val_accuracy, topk_val_accuracy, averagek_val_accuracy, k, +def val_epoch(model, val_loader, criteria, loss_val, val_accuracy, topk_val_accuracy, averagek_val_accuracy, list_k, dataset_attributes, use_gpu): model.eval() with torch.no_grad(): loss_epoch_val = 0 n_correct_val = 0 - n_correct_top_k_val = 0 + n_correct_top_k_val = defaultdict(int) + epoch_top_k_accuracy_val, epoch_average_k_accuracy_val, lmbda_val = {}, {}, {} + n_correct_average_k_val = defaultdict(int) + list_val_proba = [] list_val_labels = [] for batch_idx, (batch_x_val, batch_y_val) in enumerate(tqdm(val_loader, desc='val', position=0)): @@ -57,21 +64,24 @@ def val_epoch(model, val_loader, criteria, loss_val, val_accuracy, topk_val_accu loss_epoch_val += loss_batch_val.item() n_correct_val += torch.sum(torch.eq(batch_y_val, torch.argmax(batch_output_val, dim=-1))).item() - n_correct_top_k_val += count_correct_top_k(scores=batch_output_val, labels=batch_y_val, k=k).item() + for k in list_k: + n_correct_top_k_val[k] += count_correct_top_k(scores=batch_output_val, labels=batch_y_val, k=k).item() val_probas = torch.cat(list_val_proba) val_labels = torch.cat(list_val_labels) flat_val_probas = torch.flatten(val_probas) sorted_probas, _ = torch.sort(flat_val_probas, descending=True) - lmbda_val = 0.5 * (sorted_probas[dataset_attributes['n_val'] * k - 1] + sorted_probas[dataset_attributes['n_val'] * k]) - n_correct_average_k_val = count_correct_average_k(probas=val_probas, labels=val_labels, lmbda=lmbda_val).item() + for k in list_k: + lmbda_val[k] = 0.5 * (sorted_probas[dataset_attributes['n_val'] * k - 1] + sorted_probas[dataset_attributes['n_val'] * k]) + n_correct_average_k_val[k] += count_correct_average_k(probas=val_probas, labels=val_labels, lmbda=lmbda_val[k]).item() # After seeing val update the statistics over batches and store them loss_epoch_val /= batch_idx epoch_accuracy_val = n_correct_val / dataset_attributes['n_val'] - epoch_top_k_accuracy_val = n_correct_top_k_val / dataset_attributes['n_val'] - epoch_average_k_accuracy_val = n_correct_average_k_val / dataset_attributes['n_val'] + for k in list_k: + epoch_top_k_accuracy_val[k] = n_correct_top_k_val[k] / dataset_attributes['n_val'] + epoch_average_k_accuracy_val[k] = n_correct_average_k_val[k] / dataset_attributes['n_val'] loss_val.append(loss_epoch_val), val_accuracy.append(epoch_accuracy_val), topk_val_accuracy.append( epoch_top_k_accuracy_val), averagek_val_accuracy.append(epoch_average_k_accuracy_val) @@ -79,15 +89,15 @@ def val_epoch(model, val_loader, criteria, loss_val, val_accuracy, topk_val_accu return loss_epoch_val, epoch_accuracy_val, epoch_top_k_accuracy_val, epoch_average_k_accuracy_val, lmbda_val -def test_epoch(model, test_loader, criteria, k, lmbda, use_gpu, n_test): +def test_epoch(model, test_loader, criteria, list_k, lmbda, use_gpu, n_test): print() model.eval() with torch.no_grad(): loss_epoch_test = 0 n_correct_test = 0 - n_correct_top_k_test = 0 - n_correct_average_k_test = 0 + epoch_top_k_accuracy_test, epoch_average_k_accuracy_test = {}, {} + n_correct_top_k_test, n_correct_average_k_test = defaultdict(int), defaultdict(int) for batch_idx, (batch_x_test, batch_y_test) in enumerate(tqdm(test_loader, desc='test', position=0)): if use_gpu: batch_x_test, batch_y_test = batch_x_test.cuda(), batch_y_test.cuda() @@ -97,14 +107,16 @@ def test_epoch(model, test_loader, criteria, k, lmbda, use_gpu, n_test): loss_epoch_test += loss_batch_test.item() n_correct_test += torch.sum(torch.eq(batch_y_test, torch.argmax(batch_output_test, dim=-1))).item() - n_correct_top_k_test += count_correct_top_k(scores=batch_output_test, labels=batch_y_test, k=k).item() - n_correct_average_k_test += count_correct_average_k(probas=batch_ouput_probra_test, labels=batch_y_test, - lmbda=lmbda).item() + for k in list_k: + n_correct_top_k_test[k] += count_correct_top_k(scores=batch_output_test, labels=batch_y_test, k=k).item() + n_correct_average_k_test[k] += count_correct_average_k(probas=batch_ouput_probra_test, labels=batch_y_test, + lmbda=lmbda[k]).item() # After seeing test test update the statistics over batches and store them loss_epoch_test /= batch_idx epoch_accuracy_test = n_correct_test / n_test - epoch_top_k_accuracy_test = n_correct_top_k_test / n_test - epoch_average_k_accuracy_test = n_correct_average_k_test / n_test + for k in list_k: + epoch_top_k_accuracy_test[k] = n_correct_top_k_test[k] / n_test + epoch_average_k_accuracy_test[k] = n_correct_average_k_test[k] / n_test return loss_epoch_test, epoch_accuracy_test, epoch_top_k_accuracy_test, epoch_average_k_accuracy_test diff --git a/main.py b/main.py index caef615..7810566 100644 --- a/main.py +++ b/main.py @@ -40,6 +40,8 @@ def train(args): if not os.path.exists(save_dir): os.makedirs(save_dir) + print('args.k : ', args.k) + lmbda_best_acc = None for epoch in tqdm(range(args.n_epochs), desc='epoch', position=0): @@ -72,8 +74,9 @@ def train(args): print(f'epoch {epoch} took {time.time()-t:.2f}') print(f'loss_epoch_train : {loss_epoch_train}') print(f'loss_epoch_val : {loss_epoch_val}') - print(f'train accuracy : {epoch_accuracy_train} / train top_{args.k} accuracy : {epoch_top_k_accuracy_train}') - print(f'val accuracy : {epoch_accuracy_val} / val top_{args.k} accuracy : {epoch_top_k_accuracy_val} / val average_{args.k} accuracy : {epoch_average_k_accuracy_val}') + print(f'train accuracy : {epoch_accuracy_train} / train top_k accuracy : {epoch_top_k_accuracy_train}') + print(f'val accuracy : {epoch_accuracy_val} / val top_k accuracy : {epoch_top_k_accuracy_val} / ' + f'val average_k accuracy : {epoch_average_k_accuracy_val}') # load weights corresponding to best val accuracy and evaluate on test load_model(model, os.path.join(save_dir, save_name + '_weights_best_acc.tar'), args.use_gpu) @@ -87,11 +90,11 @@ def train(args): results = {'loss_train': loss_train, 'train_accuracy': train_accuracy, 'topk_train_accuracy': topk_train_accuracy, 'loss_val': loss_val, 'val_accuracy': val_accuracy, 'topk_val_accuracy': topk_val_accuracy, 'average_k_val_accuracy': average_k_val_accuracy, - 'test_results': {'best_val_accuracy': {'loss': loss_test_ba, - 'accuracy': accuracy_test_ba, - 'topk-accuracy': top_k_accuracy_test_ba, - 'averagek-accuracy': average_k_accuracy_test_ba} - }, 'params': args.__dict__} + 'test_results': {'loss': loss_test_ba, + 'accuracy': accuracy_test_ba, + 'topk-accuracy': top_k_accuracy_test_ba, + 'averagek-accuracy': average_k_accuracy_test_ba}, + 'params': args.__dict__} with open(os.path.join(save_dir, save_name + '.pkl'), 'wb') as f: pickle.dump(results, f)