From 65c0be6e0f42d184c0b2c55e375901cde20b0ae8 Mon Sep 17 00:00:00 2001 From: Het-Shah Date: Fri, 11 Jun 2021 16:24:48 +0530 Subject: [PATCH 1/5] Correct tests --- .travis.yml | 1 - KD_Lib/KD/common/base_class.py | 34 ++++++++++++++++----------------- KD_Lib/KD/vision/DML/dml.py | 4 +++- tests/test_KD_Lib.py | 35 +++++++++++++++++++++++++++------- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 580eec0d..b7894402 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ install: - pip install -U tox-travis codecov black - python setup.py install - jobs: include: # Deploy Documentation diff --git a/KD_Lib/KD/common/base_class.py b/KD_Lib/KD/common/base_class.py index 4c0328e2..9d74220d 100755 --- a/KD_Lib/KD/common/base_class.py +++ b/KD_Lib/KD/common/base_class.py @@ -53,28 +53,26 @@ def __init__( if self.log: self.writer = SummaryWriter(logdir) - try: - torch.Tensor(0).to(device) - self.device = device - except: - print( - "Either an invalid device or CUDA is not available. Defaulting to CPU." - ) + if device == "cpu": self.device = torch.device("cpu") + elif device == "cuda": + if torch.cuda.is_available(): + self.device = torch.device("cuda") + else: + print( + "Either an invalid device or CUDA is not available. Defaulting to CPU." + ) + self.device = torch.device("cpu") - try: + if teacher_model: self.teacher_model = teacher_model.to(self.device) - except: + else: print("Warning!!! Teacher is NONE.") + self.student_model = student_model.to(self.device) - try: - self.loss_fn = loss_fn.to(self.device) - self.ce_fn = nn.CrossEntropyLoss().to(self.device) - except: - self.loss_fn = loss_fn - self.ce_fn = nn.CrossEntropyLoss() - print("Warning: Loss Function can't be moved to device.") - + self.loss_fn = loss_fn.to(self.device) + self.ce_fn = nn.CrossEntropyLoss().to(self.device) + def train_teacher( self, epochs=20, @@ -208,7 +206,7 @@ def _train_student( epoch_acc = correct / length_of_dataset - epoch_val_acc = self.evaluate(teacher=False) + _, epoch_val_acc = self._evaluate_model(self.student_model, verbose=True) if epoch_val_acc > best_acc: best_acc = epoch_val_acc diff --git a/KD_Lib/KD/vision/DML/dml.py b/KD_Lib/KD/vision/DML/dml.py index cd47ad9c..20f7dd84 100755 --- a/KD_Lib/KD/vision/DML/dml.py +++ b/KD_Lib/KD/vision/DML/dml.py @@ -171,7 +171,9 @@ def _evaluate_model(self, model, verbose=True): if verbose: print(f"Accuracy: {correct/length_of_dataset}") - return outputs + + epoch_val_acc = correct/length_of_dataset + return outputs, epoch_val_acc def evaluate(self): """ diff --git a/tests/test_KD_Lib.py b/tests/test_KD_Lib.py index 4989fb2a..f3f4a4cd 100644 --- a/tests/test_KD_Lib.py +++ b/tests/test_KD_Lib.py @@ -23,6 +23,7 @@ ProbShift, LabelSmoothReg, DML, + BaseClass ) from KD_Lib.models import ( @@ -94,6 +95,10 @@ def test_resnet(): ResNet50(params) ResNet101(params) ResNet152(params) + ResNet34(params, att=True) + ResNet34(params, mean=True) + ResNet101(params, att=True) + ResNet101(params, mean=True) def test_attention_model(): @@ -159,6 +164,22 @@ def test_LSTMNet(): # Strategy TESTS # +def test_BaseClass() + teac = Shallow(hidden_size=400) + stud = Shallow(hidden_size=100) + + t_optimizer = optim.SGD(teac.parameters(), 0.01) + s_optimizer = optim.SGD(stud.parameters(), 0.01) + + distiller = BaseClass( + teac, stud, train_loader, test_loader, t_optimizer, s_optimizer, log=True + ) + + distiller.train_teacher(epochs=1, plot_losses=True, save_model=True) + distiller.train_student(epochs=1, plot_losses=True, save_model=True) + distiller.evaluate(teacher=False) + distiller.get_parameters() + def test_VanillaKD(): teac = Shallow(hidden_size=400) @@ -168,11 +189,11 @@ def test_VanillaKD(): s_optimizer = optim.SGD(stud.parameters(), 0.01) distiller = VanillaKD( - teac, stud, train_loader, test_loader, t_optimizer, s_optimizer + teac, stud, train_loader, test_loader, t_optimizer, s_optimizer, log=True ) - distiller.train_teacher(epochs=1, plot_losses=False, save_model=False) - distiller.train_student(epochs=1, plot_losses=False, save_model=False) + distiller.train_teacher(epochs=1, plot_losses=True, save_model=True) + distiller.train_student(epochs=1, plot_losses=True, save_model=True) distiller.evaluate(teacher=False) distiller.get_parameters() @@ -289,8 +310,8 @@ def test_SelfTraining(): def test_mean_teacher(): - teacher_params = [4, 4, 8, 4, 4] - student_params = [4, 4, 4, 4, 4] + teacher_params = [16, 16, 32, 16, 16] + student_params = [16, 16, 16, 16, 16] teacher_model = ResNet50(teacher_params, 1, 10, mean=True) student_model = ResNet18(student_params, 1, 10, mean=True) @@ -488,7 +509,7 @@ def test_lottery_tickets(): teacher_params = [4, 4, 8, 4, 4] teacher_model = ResNet50(teacher_params, 1, 10, True) pruner = Lottery_Tickets_Pruner(teacher_model, train_loader, test_loader) - pruner.prune(num_iterations=1, train_iterations=1, valid_freq=1, print_freq=1) + pruner.prune(num_iterations=2, train_iterations=2, valid_freq=1, print_freq=1) # @@ -539,6 +560,6 @@ def test_qat_quantization(): model.fc.out_features = 10 optimizer = torch.optim.Adam(model.parameters()) quantizer = QAT_Quantizer(model, cifar_trainloader, cifar_testloader, optimizer) - quantized_model = quantizer.quantize(1, 1, -1, -1) + quantized_model = quantizer.quantize(1, 1, 1, 1) quantizer.get_model_sizes() quantizer.get_performance_statistics() From 87c1e2a67d34735daeb9ebf37ca2bfebec20135b Mon Sep 17 00:00:00 2001 From: Het-Shah Date: Fri, 11 Jun 2021 16:47:59 +0530 Subject: [PATCH 2/5] Update tests and apply black --- .github/workflows/python-package-test.yml | 5 ----- KD_Lib/KD/common/base_class.py | 14 +++++++------- KD_Lib/KD/vision/DML/dml.py | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-package-test.yml b/.github/workflows/python-package-test.yml index 9fae0f4b..0ea735dd 100644 --- a/.github/workflows/python-package-test.yml +++ b/.github/workflows/python-package-test.yml @@ -33,11 +33,6 @@ jobs: pip install build - name: Build package run: python -m build - - name: Black - run: | - # stop the build if there are Python syntax errors or undefined names - black --check KD_Lib - black --check tests - name: Test with pytest run: | pytest diff --git a/KD_Lib/KD/common/base_class.py b/KD_Lib/KD/common/base_class.py index 9d74220d..74ed175b 100755 --- a/KD_Lib/KD/common/base_class.py +++ b/KD_Lib/KD/common/base_class.py @@ -68,11 +68,11 @@ def __init__( self.teacher_model = teacher_model.to(self.device) else: print("Warning!!! Teacher is NONE.") - + self.student_model = student_model.to(self.device) self.loss_fn = loss_fn.to(self.device) self.ce_fn = nn.CrossEntropyLoss().to(self.device) - + def train_teacher( self, epochs=20, @@ -140,7 +140,7 @@ def train_teacher( ) loss_arr.append(epoch_loss) - print(f"Epoch: {ep+1}, Loss: {epoch_loss}, Accuracy: {epoch_acc}") + print("Epoch: {}, Loss: {}, Accuracy: {}".format(ep+1, epoch_loss, epoch_acc)) self.post_epoch_call(ep) @@ -222,7 +222,7 @@ def _train_student( ) loss_arr.append(epoch_loss) - print(f"Epoch: {ep+1}, Loss: {epoch_loss}, Accuracy: {epoch_acc}") + print("Epoch: {}, Loss: {}, Accuracy: {}".format(ep+1, epoch_loss, epoch_acc)) self.student_model.load_state_dict(self.best_student_model_weights) if save_model: @@ -288,7 +288,7 @@ def _evaluate_model(self, model, verbose=True): if verbose: print("-" * 80) - print(f"Validation Accuracy: {accuracy}") + print("Validation Accuracy: {}".format(accuracy)) return outputs, accuracy def evaluate(self, teacher=False): @@ -313,8 +313,8 @@ def get_parameters(self): student_params = sum(p.numel() for p in self.student_model.parameters()) print("-" * 80) - print(f"Total parameters for the teacher network are: {teacher_params}") - print(f"Total parameters for the student network are: {student_params}") + print("Total parameters for the teacher network are: {}".format(teacher_params)) + print("Total parameters for the student network are: {}".format(student_params)) def post_epoch_call(self, epoch): """ diff --git a/KD_Lib/KD/vision/DML/dml.py b/KD_Lib/KD/vision/DML/dml.py index 20f7dd84..8b37347d 100755 --- a/KD_Lib/KD/vision/DML/dml.py +++ b/KD_Lib/KD/vision/DML/dml.py @@ -172,7 +172,7 @@ def _evaluate_model(self, model, verbose=True): if verbose: print(f"Accuracy: {correct/length_of_dataset}") - epoch_val_acc = correct/length_of_dataset + epoch_val_acc = correct / length_of_dataset return outputs, epoch_val_acc def evaluate(self): From 5cfade60104e28d43c326701c34e81c24e1aa5ed Mon Sep 17 00:00:00 2001 From: Khizir Siddiqui Date: Fri, 11 Jun 2021 16:49:02 +0530 Subject: [PATCH 3/5] fix test failures --- KD_Lib/KD/common/base_class.py | 2 +- KD_Lib/KD/vision/DML/dml.py | 4 +--- KD_Lib/Pruning/lottery_tickets/lottery_tickets.py | 5 +++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/KD_Lib/KD/common/base_class.py b/KD_Lib/KD/common/base_class.py index 74ed175b..7a8a93d8 100755 --- a/KD_Lib/KD/common/base_class.py +++ b/KD_Lib/KD/common/base_class.py @@ -206,7 +206,7 @@ def _train_student( epoch_acc = correct / length_of_dataset - _, epoch_val_acc = self._evaluate_model(self.student_model, verbose=True) + epoch_val_acc = self.evaluate() if epoch_val_acc > best_acc: best_acc = epoch_val_acc diff --git a/KD_Lib/KD/vision/DML/dml.py b/KD_Lib/KD/vision/DML/dml.py index 8b37347d..32e0b91a 100755 --- a/KD_Lib/KD/vision/DML/dml.py +++ b/KD_Lib/KD/vision/DML/dml.py @@ -178,14 +178,12 @@ def _evaluate_model(self, model, verbose=True): def evaluate(self): """ Evaluate method for printing accuracies of the trained student networks - """ - for i, student in enumerate(self.student_cohort): print("-" * 80) model = deepcopy(student).to(self.device) print(f"Evaluating student {i}") - _ = self._evaluate_model(model) + _, _ = self._evaluate_model(model) def get_parameters(self): """ diff --git a/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py b/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py index 99b9932f..8346d616 100755 --- a/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py +++ b/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py @@ -181,7 +181,7 @@ def _train_pruned_model(self): eps = 1e-6 self.model.train() correct = 0 - + training_loss = 0.0 step = 0 for data, targets in self.train_loader: self.optimizer.zero_grad() @@ -192,6 +192,7 @@ def _train_pruned_model(self): outputs = outputs[0] train_loss = self.loss_fn(outputs, targets) + training_loss += train_loss.item() train_loss.backward() pred = outputs.argmax(dim=1, keepdim=True) @@ -212,7 +213,7 @@ def _train_pruned_model(self): step += 1 train_acc = 100.0 * correct / len(self.train_loader.dataset) - return train_loss.item(), train_acc + return training_loss / len(self.train_loader), train_acc def _save_model(self, prune_it, best_weights): file_name = f"{os.getcwd()}/pruned_model_{prune_it}.pth.tar" From 7e76967e7044076eb01b0cfba6a46acc13e9f8fb Mon Sep 17 00:00:00 2001 From: Khizir Siddiqui Date: Fri, 11 Jun 2021 16:54:20 +0530 Subject: [PATCH 4/5] apply black --- .../lottery_tickets/lottery_tickets.py | 2 +- setup.py | 115 ++++++++++-------- 2 files changed, 62 insertions(+), 55 deletions(-) diff --git a/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py b/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py index 8346d616..6c46cb97 100755 --- a/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py +++ b/KD_Lib/Pruning/lottery_tickets/lottery_tickets.py @@ -181,7 +181,7 @@ def _train_pruned_model(self): eps = 1e-6 self.model.train() correct = 0 - training_loss = 0.0 + training_loss = 0.0 step = 0 for data, targets in self.train_loader: self.optimizer.zero_grad() diff --git a/setup.py b/setup.py index dd5add73..d840bb84 100755 --- a/setup.py +++ b/setup.py @@ -13,67 +13,74 @@ LONG_DESCRIPTION = f.read() # Define the keywords -KEYWORDS = ["Knowledge Distillation", "Pruning", "Quantization", "pytorch", "machine learning", "deep learning"] +KEYWORDS = [ + "Knowledge Distillation", + "Pruning", + "Quantization", + "pytorch", + "machine learning", + "deep learning", +] REQUIRE_PATH = "requirements.txt" PROJECT = os.path.abspath(os.path.dirname(__file__)) -setup_requirements = ['pytest-runner'] +setup_requirements = ["pytest-runner"] -test_requirements = ['pytest', 'pytest-cov'] +test_requirements = ["pytest", "pytest-cov"] requirements = [ -'pip==19.3.1', -'transformers==4.6.1', -'sacremoses', -'tokenizers==0.10.1', -'huggingface-hub==0.0.8', -'torchtext==0.9.1', -'bumpversion==0.5.3', -'wheel==0.32.1', -'watchdog==0.9.0', -'flake8==3.5.0', -'tox==3.5.2', -'coverage==4.5.1', -'Sphinx==1.8.1', -'twine==1.12.1', -'pytest==3.8.2', -'pytest-runner==4.2', -'pytest-cov==2.6.1', -'matplotlib==3.2.1', -'torch==1.8.1', -'torchvision==0.9.1', -'tensorboard==2.2.1', -'contextlib2==0.6.0.post1', -'pandas==1.0.1', -'tqdm==4.42.1', -'numpy==1.18.1', -'sphinx-rtd-theme==0.5.0', + "pip==19.3.1", + "transformers==4.6.1", + "sacremoses", + "tokenizers==0.10.1", + "huggingface-hub==0.0.8", + "torchtext==0.9.1", + "bumpversion==0.5.3", + "wheel==0.32.1", + "watchdog==0.9.0", + "flake8==3.5.0", + "tox==3.5.2", + "coverage==4.5.1", + "Sphinx==1.8.1", + "twine==1.12.1", + "pytest==3.8.2", + "pytest-runner==4.2", + "pytest-cov==2.6.1", + "matplotlib==3.2.1", + "torch==1.8.1", + "torchvision==0.9.1", + "tensorboard==2.2.1", + "contextlib2==0.6.0.post1", + "pandas==1.0.1", + "tqdm==4.42.1", + "numpy==1.18.1", + "sphinx-rtd-theme==0.5.0", ] if __name__ == "__main__": setup( - author="Het Shah", - author_email='divhet163@gmail.com', - classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - ], - description="A Pytorch Library to help extend all Knowledge Distillation works", - install_requires=requirements, - license="MIT license", - long_description=LONG_DESCRIPTION, - include_package_data=True, - keywords=KEYWORDS, - name='KD_Lib', - packages=find_packages(where=PROJECT), - setup_requires=setup_requirements, - test_suite="tests", - tests_require=test_requirements, - url="https://github.com/SforAiDL/KD_Lib", - version='0.0.29', - zip_safe=False, -) + author="Het Shah", + author_email="divhet163@gmail.com", + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], + description="A Pytorch Library to help extend all Knowledge Distillation works", + install_requires=requirements, + license="MIT license", + long_description=LONG_DESCRIPTION, + include_package_data=True, + keywords=KEYWORDS, + name="KD_Lib", + packages=find_packages(where=PROJECT), + setup_requires=setup_requirements, + test_suite="tests", + tests_require=test_requirements, + url="https://github.com/SforAiDL/KD_Lib", + version="0.0.29", + zip_safe=False, + ) From 4d12fdb5a1b3455413d3e403aaede164923f1311 Mon Sep 17 00:00:00 2001 From: Khizir Siddiqui Date: Sun, 4 Jul 2021 12:12:17 +0530 Subject: [PATCH 5/5] fix #1 --- KD_Lib/KD/common/base_class.py | 2 +- tests/test_KD_Lib.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/KD_Lib/KD/common/base_class.py b/KD_Lib/KD/common/base_class.py index 7a8a93d8..064f1dbc 100755 --- a/KD_Lib/KD/common/base_class.py +++ b/KD_Lib/KD/common/base_class.py @@ -56,7 +56,7 @@ def __init__( if device == "cpu": self.device = torch.device("cpu") elif device == "cuda": - if torch.cuda.is_available(): + if torch.cuda.is_available(): self.device = torch.device("cuda") else: print( diff --git a/tests/test_KD_Lib.py b/tests/test_KD_Lib.py index f3f4a4cd..0aab0882 100644 --- a/tests/test_KD_Lib.py +++ b/tests/test_KD_Lib.py @@ -23,7 +23,7 @@ ProbShift, LabelSmoothReg, DML, - BaseClass + BaseClass, ) from KD_Lib.models import ( @@ -164,7 +164,8 @@ def test_LSTMNet(): # Strategy TESTS # -def test_BaseClass() + +def test_BaseClass(): teac = Shallow(hidden_size=400) stud = Shallow(hidden_size=100)