diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..b9c50a517 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/recognition/ISIC Improved UNet 47479049/data +recognition/ISIC Improved UNet 47479049/model +my_checkpoint.pth.tar diff --git a/recognition/ISIC Improved UNet 47479049/.gitignore b/recognition/ISIC Improved UNet 47479049/.gitignore new file mode 100644 index 000000000..2a89354c5 --- /dev/null +++ b/recognition/ISIC Improved UNet 47479049/.gitignore @@ -0,0 +1,3 @@ +__pycache__ +model +model2 diff --git a/recognition/ISIC Improved UNet 47479049/README.md b/recognition/ISIC Improved UNet 47479049/README.md new file mode 100644 index 000000000..1f88c6585 --- /dev/null +++ b/recognition/ISIC Improved UNet 47479049/README.md @@ -0,0 +1,85 @@ +# Improved Unet on ISIC +This reports the usage of the improved Unet on the idetification of skin lesions in the ISIC dataset, The expected result is that the dice corefficent to be greater than 0.8 on the test set. + +## Preprocessed ISIC Data set +The data set contains 2,594 images of skin lesions along wwith a mask that contains pixel values of 0 (white) and (255) black. The white represetns the area that the skin lession is in and the black is where it does not contain the skin lession. + +![segmented](./images/ISIC_0000000.jpg) + +Figure 1: Original image for ISIC_0000000 + +![segmented](./images/ISIC_0000000_segmentation.png) + +Figure 2: Original image for ISIC_0000000_segmentation + +### Data pre processing +Images are scaled down to 96 x 128 comapred to the origional 511 x 384 this way it saves on computer processing time while also fitting the model so that no extra resizes are needed in the model. The mask is then mapped to 0 or 1 where if the pixel values is greater than 128 it is maped to 1 and 0 if it is below 128. + +The model is then split into validation and training with a 0.1/0.9 split the test set was given in the database so it will be used in the model. + +### Improved Unet Model +![segmented](./images/Model.png) + +Figure 3: Improved Unet + + +The model is an improvement of the origional Unet model with residual adding, context module and localization modules + +The Context module is a combination of a (3,3) same convolution, followed by a dropout layer of 0.3 the followed by another 3,3 same Convolution, with instance normalization and a leakyRelu activations. + +The localization module is a (3,3) same convolution followed by a (1,1) same convolution. With instance normalization and LeakyRelu activations. + +The Upsampling module uses the upsample module from pytorch followed by a (3,3) same convolution. + +The Origional model was developed for 3D models however the ISIC data set is 2D so the model was changed to fit the new data set. + +# Method +Download the dataset place the files within the data folder you chould have the paths +- data\ISIC2018_Task1_Training_GroundTruth_x2 +- data\ISIC2018_Task1-2_Test +- data\ISIC2018_Task1-2_Training_Input_x2 + +remove the txt files inside run train then run predict + +## Results +![segmented](./images/Lossgraph.jpg) + +Figure 4: Loss Curve + +![segmented](./images/Dice_Score.png) + +figure 5: Final Dice Scores + +The model Achived a dice score of 0.88 after a batch of 16 and 16 epochs however the data set seemed to have stagnated around the 7th epoch so 16 was not needed + +![segmented](./images/ISIC_0000001.jpg) + +Figure 5: Original ISIC image + +![segmented](./images/ISIC_0000001_segmentation.png) + +figure 6: Original ISIC Mask + +![segmented](./images/ISIC_0000001_segmentation_from_model.png) + + +figure 7: ISIC Mask From model + + +# Dependecies +- Python +- Pyroch +- PIL +- numpy +- matplotlib +- Dataset(https://filesender.aarnet.edu.au/?s=download&token=d93a02ff-5b61-465f-9cba-fc1566613384) + + +# References +F. Isensee, P. Kickingereder, W. Wick, M. Bendszus, and K. H. Maier-Hein, “Brain Tumor Segmentation +and Radiomics Survival Prediction: Contribution to the BRATS 2017 Challenge,” Feb. 2018. [Online]. +Available: https://arxiv.org/abs/1802.10508v1 + +https://discuss.pytorch.org/t/implementation-of-dice-loss/53552 + +https://github.com/mcost45/ISICs-improved-unet/blob/main/layers_model.py diff --git a/recognition/ISIC Improved UNet 47479049/dataset.py b/recognition/ISIC Improved UNet 47479049/dataset.py new file mode 100644 index 000000000..40d80bde7 --- /dev/null +++ b/recognition/ISIC Improved UNet 47479049/dataset.py @@ -0,0 +1,75 @@ +import os +import torch +import torch.utils.data +import torchvision.transforms as T +from PIL import Image +from torch.utils.data import Dataset, DataLoader, random_split +import numpy as np + +class SkinDataset(Dataset): + def __init__(self, image_dir, mask_dir, transform=None): + self.image_dir = image_dir + self.mask_dir = mask_dir + self.transform = transform + self.images = os.listdir(image_dir) + + def __len__(self): + return len(self.images) + + def __getitem__(self, index): + img_path = os.path.join(self.image_dir, self.images[index]) + mask_path = os.path.join(self.mask_dir, self.images[index].replace(".jpg", "_segmentation.png")) + + # Changes image to numpy array then changes to values to rbg(3 channels) L(greysacle 1 channel) + image = np.array(Image.open(img_path).convert("RGB")) + mask = np.array(Image.open(mask_path).convert("L"), dtype=np.float32) + + + # if pixel is black change to 1 + mask[mask <= 128.0] = 0 + mask[mask > 128.0] = 1 + + image = self.transform(image) + mask = self.transform(mask) + + return image, mask + +def get_loaders(train_dir, mask_dir, batch_size, train_trasform): + """ + get_loaders takes in the directory of the data set then converts them into dataloaders + train_dir: directory of training dataset + mask_dir: directory of mask dataset + batch_size: batch size for loaders + trian_transform: transormation for dataset + + returns: Training Dataloder and validation Dataloader + """ + + dataset = SkinDataset( + image_dir=train_dir, + mask_dir=mask_dir, + transform=train_trasform + ) + + train_dataset, val_dataset = torch.utils.data.random_split(dataset, [0.9, 0.1], generator=torch.Generator().manual_seed(1)) + + train_loader = DataLoader( + train_dataset, + batch_size=batch_size, + shuffle=True, + pin_memory=True, + num_workers=8 + ) + + val_loader = DataLoader( + val_dataset, + batch_size=batch_size, + shuffle=False, + pin_memory=True, + num_workers=8 + ) + + + return train_loader, val_loader + + \ No newline at end of file diff --git a/recognition/ISIC Improved UNet 47479049/images/Dice_Score.png b/recognition/ISIC Improved UNet 47479049/images/Dice_Score.png new file mode 100644 index 000000000..4abe766c8 Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/Dice_Score.png differ diff --git a/recognition/ISIC Improved UNet 47479049/images/ISIC_0000000.jpg b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000000.jpg new file mode 100644 index 000000000..0f8e21eb2 Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000000.jpg differ diff --git a/recognition/ISIC Improved UNet 47479049/images/ISIC_0000000_segmentation.png b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000000_segmentation.png new file mode 100644 index 000000000..caa4c0a4d Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000000_segmentation.png differ diff --git a/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001.jpg b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001.jpg new file mode 100644 index 000000000..9733036bc Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001.jpg differ diff --git a/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001_segmentation.png b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001_segmentation.png new file mode 100644 index 000000000..649c1748f Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001_segmentation.png differ diff --git a/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001_segmentation_from_model.png b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001_segmentation_from_model.png new file mode 100644 index 000000000..de1184078 Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/ISIC_0000001_segmentation_from_model.png differ diff --git a/recognition/ISIC Improved UNet 47479049/images/Lossgraph.jpg b/recognition/ISIC Improved UNet 47479049/images/Lossgraph.jpg new file mode 100644 index 000000000..aa0d778a6 Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/Lossgraph.jpg differ diff --git a/recognition/ISIC Improved UNet 47479049/images/Model.png b/recognition/ISIC Improved UNet 47479049/images/Model.png new file mode 100644 index 000000000..de6e9041b Binary files /dev/null and b/recognition/ISIC Improved UNet 47479049/images/Model.png differ diff --git a/recognition/ISIC Improved UNet 47479049/modules.py b/recognition/ISIC Improved UNet 47479049/modules.py new file mode 100644 index 000000000..26350c3c3 --- /dev/null +++ b/recognition/ISIC Improved UNet 47479049/modules.py @@ -0,0 +1,175 @@ +import torch +import torch.nn as nn +import torchvision.transforms.functional as TF + + +class UNet(nn.Module): + """ + Improved unet from the paper https://arxiv.org/pdf/1802.10508v1.pdf + Modified from https://github.com/mcost45/ISICs-improved-unet/blob/main/layers_model.py + """ + def __init__(self, in_channels=3, out_channels=1, base_features=16): + super(UNet, self).__init__() + + self.sigmoid = nn.Sigmoid() + + self.x1 = nn.Conv2d(in_channels, base_features, 3, 1, 1) + self.x2 = nn.InstanceNorm2d(base_features) + self.x3 = nn.LeakyReLU() + self.x4 = self.ContexModule(base_features, base_features) + + self.x6 = nn.Conv2d(base_features, base_features*2, 3, stride=2, padding=1) + self.x7 = nn.InstanceNorm2d(base_features*2) + self.x8 = nn.LeakyReLU() + self.x9 = self.ContexModule(base_features*2, base_features*2) + + self.x11 = nn.Conv2d(base_features*2, base_features*4, 3, stride=2, padding=1) + self.x12 = nn.InstanceNorm2d(base_features*4) + self.x13 = nn.LeakyReLU() + self.x14 = self.ContexModule(base_features*4, base_features*4) + + self.x16 = nn.Conv2d(base_features*4, base_features*8, 3, stride=2, padding=1) + self.x17 = nn.InstanceNorm2d(base_features*8) + self.x18 = nn.LeakyReLU() + self.x19 = self.ContexModule(base_features*8, base_features*8) + + self.x21 = nn.Conv2d(base_features*8, base_features*16, 3, stride=2, padding=1) + self.x22 = nn.InstanceNorm2d(base_features*16) + self.x23 = nn.LeakyReLU() + self.x24 = self.ContexModule(base_features*16, base_features*16) + self.x26 = self.UpsampleModule(base_features*16, base_features*8) + + self.x28 = self.LocalizationModule(base_features*16, base_features*8) + self.x29 = self.UpsampleModule(base_features*8, base_features*4) + + self.x31 = self.LocalizationModule(base_features*8, base_features*4) + self.x32 = self.UpsampleModule(base_features*4, base_features*2) + + self.x34 = self.LocalizationModule(base_features*4, base_features*2) + self.x35 = self.UpsampleModule(base_features*2, base_features) + + self.x37 = nn.Conv2d(base_features*2, base_features*2, kernel_size=1) + self.x38 = nn.InstanceNorm2d(base_features*2) + self.x39 = nn.LeakyReLU() + + self.u1 = self.UpsampleModule(base_features*4, base_features*2) + + self.u2 = self.UpsampleModule(base_features*2, base_features*2) + + self.output = nn.Conv2d(base_features*2, out_channels, 1) + + def ContexModule(self, in_channels, out_channels): + return nn.Sequential( + nn.Conv2d(in_channels, out_channels, 3, 1, 1), + nn.InstanceNorm2d(out_channels), + nn.LeakyReLU(), + nn.Dropout2d(0.3), + nn.Conv2d(out_channels, out_channels, 3, 1, 1), + nn.InstanceNorm2d(out_channels), + nn.LeakyReLU(), + ) + + def UpsampleModule(self, in_channels, out_channels): + return nn.Sequential( + nn.Upsample(scale_factor=2, mode='nearest'), + nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False), + nn.InstanceNorm2d(out_channels), + nn.LeakyReLU(), + ) + + def LocalizationModule(self, in_channels, out_channels): + return nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False), + nn.InstanceNorm2d(out_channels), + nn.LeakyReLU(), + nn.Conv2d(out_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False), + nn.InstanceNorm2d(out_channels), + nn.LeakyReLU(), + ) + + + def forward(self, x): + + out = self.x1(x) + residual = out + out = self.x2(out) + out = self.x3(out) + out = self.x4(out) + out += residual + hold_over_0 = out + + out = self.x6(out) + out = self.x7(out) + out = self.x8(out) + residual = out + out = self.x9(out) + out += residual + hold_over_1 = out + + out = self.x11(out) + out = self.x12(out) + out = self.x13(out) + residual = out + out = self.x14(out) + out += residual + hold_over_2 = out + + out = self.x16(out) + out = self.x17(out) + out = self.x18(out) + residual = out + out = self.x19(out) + out += residual + hold_over_3 = out + + out = self.x21(out) + out = self.x22(out) + out = self.x23(out) + residual = out + out = self.x24(out) + out += residual + out = self.x26(out) + + out = torch.cat([hold_over_3, out], dim=1) + out = self.x28(out) + out = self.x29(out) + + out = torch.cat([hold_over_2, out], dim=1) + out = self.x31(out) + segment1 = out + out = self.x32(out) + + out = torch.cat([hold_over_1, out], dim=1) + out = self.x34(out) + segment2 = out + out = self.x35(out) + + out = torch.cat([hold_over_0, out], dim=1) + out = self.x37(out) + out = self.x38(out) + out = self.x39(out) + + seg1 = self.u1(segment1) + seg2 = segment2 + seg2 += seg1 + seg2 = self.u2(seg2) + seg3 = out + seg3 += seg2 + + output = self.output(seg3) + return self.sigmoid(output) + + + + +def test(): + x = torch.randn((1, 3, 96, 128)) + model = UNet(in_channels=3, out_channels=1) + preds = model(x) + + print(x.shape) + print(preds.shape) + + +if __name__ == "__main__": + test() \ No newline at end of file diff --git a/recognition/ISIC Improved UNet 47479049/predict.py b/recognition/ISIC Improved UNet 47479049/predict.py new file mode 100644 index 000000000..e92eecd08 --- /dev/null +++ b/recognition/ISIC Improved UNet 47479049/predict.py @@ -0,0 +1,32 @@ +import torch +from modules import UNet +import torchvision +import torchvision.transforms as transforms +import numpy as np +from PIL import Image + +image_height = 96 +image_width = 128 +img_path = "data\ISIC2018_Task1-2_Training_Input_x2\ISIC_0000001.jpg" + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +model = UNet(3, 1).to(device) + +model.load_state_dict(torch.load("model2")) + +model.eval() + +train_transforms = transforms.Compose([ + transforms.ToTensor(), + transforms.Resize((image_height, image_width), antialias=None) + ]) + + + +img = np.array(Image.open(img_path).convert("RGB")) +img = train_transforms(img) +img = img[None, :, :, :] +img = img.to(device) +img = model(img) +torchvision.utils.save_image(img, "ISIC_0000001_segmentation_from_model.png") \ No newline at end of file diff --git a/recognition/ISIC Improved UNet 47479049/train.py b/recognition/ISIC Improved UNet 47479049/train.py new file mode 100644 index 000000000..a4247cdd0 --- /dev/null +++ b/recognition/ISIC Improved UNet 47479049/train.py @@ -0,0 +1,149 @@ +import torch +import torch.nn as nn +import torch.optim as optim +import torchvision +import torchvision.transforms as transforms +import matplotlib.pyplot as plt +from modules import UNet +from dataset import get_loaders + +# Hyperparameters +learning_rate = 1e-4 +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +batch_size = 2 +num_epochs = 16 +num_workers = 4 +image_height = 96 +image_width = 128 +test_dir = "data\ISIC2018_Task1-2_Test_Input" +test_out_dir = "data\ISIC2018_Task1-2_Test_Output" +train_dir = "data\ISIC2018_Task1-2_Training_Input_x2" +mask_dir = "data\ISIC2018_Task1_Training_GroundTruth_x2" + +def train(loader, val_loader, model, optimizer, criterion, num_epochs): + print("Training Start") + validation_loss_history = [] + loss_history = [] + + for epoch in range(num_epochs): + model.train() + total_loss = 0 + + for data, mask in loader: + data = data.to(device) + mask = mask.to(device) + + # Forward + predictions = model(data) + loss = criterion(predictions, mask) + + # Backward + optimizer.zero_grad() + loss.backward() + optimizer.step() + + total_loss += loss.item() + + avg_loss = total_loss / len(loader) + + total_validation_loss = 0 + for data, mask in val_loader: + data = data.to(device) + mask = mask.to(device) + + predictions = model(data) + validation_loss = criterion(predictions, mask) + + total_validation_loss += validation_loss.item() + + total_validation_loss = total_validation_loss/len(val_loader) + + print(f"Epoch [{epoch + 1}/{num_epochs}] - Loss: {avg_loss:.4f}, Total Validation Loss: {total_validation_loss:.4f}, Dice Score of Validation: {(1-total_validation_loss):.4f}") + validation_loss_history.append(total_validation_loss) + loss_history.append(avg_loss) + + return validation_loss_history, loss_history + + +class diceloss(torch.nn.Module): + """ + Calculates diceloss + taken from: https://discuss.pytorch.org/t/implementation-of-dice-loss/53552 + """ + + def init(self): + super(diceloss, self).__init__() + def forward(self, pred, target): + smooth = 1 + iflat = pred.contiguous().view(-1) + tflat = target.contiguous().view(-1) + intersection = (iflat * tflat).sum() + A_sum = torch.sum(iflat*iflat).sum() + B_sum = torch.sum(tflat*tflat).sum() + return 1 - ((2. * intersection + smooth)/ (A_sum + B_sum + smooth)) + + +def loss_graph(results, results_val, num_epochs): + plt.plot(range(1, num_epochs+1), results, 'k', label="Training") + plt.plot(range(1, num_epochs+1), results_val, 'r', label="Validation") + plt.title('Loss Graph') + plt.xlabel("Epochs") + plt.ylabel("Loss") + plt.savefig("Lossgraph.jpg") + plt.legend() + plt.show() + + +def create_mask(loader, model, max_pics, device=device): + """ + create_mask will create a mask given the laoder using the model given + """ + + model.eval() + + for idx, (x,y) in enumerate(loader): + if idx > max_pics: + break + x = x.to(device=device) + with torch.no_grad(): + pred = model(x) + pred = (pred>0.5).float() + torchvision.utils.save_image( + pred, f"{y}" + ) + + + model.train() + + + +def main(): + train_transforms = transforms.Compose([ + transforms.ToTensor(), + transforms.Resize((image_height, image_width), antialias=None) + ]) + + model = UNet(3, 1).to(device) + criterion = diceloss() + optimizer = optim.Adam(model.parameters(), lr=learning_rate) + + + train_loader, val_loader = get_loaders( + train_dir, + mask_dir, + batch_size, + train_transforms, + ) + + + validation_loss_history, loss_history = train(train_loader, val_loader, model, optimizer, criterion, num_epochs) + torch.save(model.state_dict(), "model") + + loss_graph(validation_loss_history, loss_history, num_epochs) + + +if __name__ == "__main__": + main() + + + diff --git a/recognition/README.md b/recognition/README.md new file mode 100644 index 000000000..5c646231c --- /dev/null +++ b/recognition/README.md @@ -0,0 +1,10 @@ +# Recognition Tasks +Various recognition tasks solved in deep learning frameworks. + +Tasks may include: +* Image Segmentation +* Object detection +* Graph node classification +* Image super resolution +* Disease classification +* Generative modelling with StyleGAN and Stable Diffusion