{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a9e632b6",
   "metadata": {
    "papermill": {
     "duration": 0.030251,
     "end_time": "2021-08-04T16:52:22.255921",
     "exception": false,
     "start_time": "2021-08-04T16:52:22.225670",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Lane Boundary Segmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b268969",
   "metadata": {},
   "source": [
    "## Setting up Colab"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a655101f",
   "metadata": {},
   "source": [
    "You can delete this \"Setting up Colab\" section if you work locally and do not want to use Google Colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3809c558",
   "metadata": {},
   "outputs": [],
   "source": [
    "colab_nb = 'google.colab' in str(get_ipython())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed0eac7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "if colab_nb:\n",
    "  from google.colab import drive\n",
    "  drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01456626",
   "metadata": {},
   "outputs": [],
   "source": [
    "if colab_nb:\n",
    "  %cd drive/My\\ Drive/aad/code/solutions/lane_detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a89923db",
   "metadata": {},
   "outputs": [],
   "source": [
    "if colab_nb:\n",
    "    !pip install fastseg",
    "    !pip install fastai --upgrade\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b6498ef",
   "metadata": {
    "papermill": {
     "duration": 0.034292,
     "end_time": "2021-08-04T16:52:22.325580",
     "exception": false,
     "start_time": "2021-08-04T16:52:22.291288",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 1. Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e50b3a79",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:52:22.429571Z",
     "iopub.status.busy": "2021-08-04T16:52:22.428939Z",
     "iopub.status.idle": "2021-08-04T16:54:00.732713Z",
     "shell.execute_reply": "2021-08-04T16:54:00.732046Z",
     "shell.execute_reply.started": "2021-08-04T16:38:45.460165Z"
    },
    "papermill": {
     "duration": 98.357125,
     "end_time": "2021-08-04T16:54:00.732864",
     "exception": false,
     "start_time": "2021-08-04T16:52:22.375739",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "\n",
    "import numpy as np\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import sys\n",
    "sys.path.append(\"../../util\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0d818f6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:01.356739Z",
     "iopub.status.busy": "2021-08-04T16:54:01.355928Z",
     "iopub.status.idle": "2021-08-04T16:54:03.085318Z",
     "shell.execute_reply": "2021-08-04T16:54:03.084820Z",
     "shell.execute_reply.started": "2021-08-04T16:48:54.301507Z"
    },
    "papermill": {
     "duration": 2.043901,
     "end_time": "2021-08-04T16:54:03.085445",
     "exception": false,
     "start_time": "2021-08-04T16:54:01.041544",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "If you have collected data yourself in a folder \"data\" using `collect_data.py` and you want to use it for training, set the boolean in the next cell to `True`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94dfb70a",
   "metadata": {},
   "outputs": [],
   "source": [
    "own_data = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a06aa44a",
   "metadata": {},
   "outputs": [],
   "source": [
    "if own_data:\n",
    "    from seg_data_util import sort_collected_data\n",
    "    # copy and sort content of 'data' into 'data_lane_segmentation' folder:\n",
    "    sort_collected_data()\n",
    "    # Since data was copied, you can remove files in 'data' directory afterwards\n",
    "else:\n",
    "    # if you stopped the download before completion, please delete the 'data_lane_segmentation' folder and run this cell again\n",
    "    from seg_data_util import download_segmentation_data\n",
    "    download_segmentation_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d56bf76",
   "metadata": {},
   "source": [
    "Independent of what you chose, you will have a directory 'data_lane_segmentation' now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d76bead",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:03.927467Z",
     "iopub.status.busy": "2021-08-04T16:54:03.926589Z",
     "iopub.status.idle": "2021-08-04T16:54:04.890126Z",
     "shell.execute_reply": "2021-08-04T16:54:04.889665Z",
     "shell.execute_reply.started": "2021-08-04T16:49:40.181183Z"
    },
    "papermill": {
     "duration": 1.467902,
     "end_time": "2021-08-04T16:54:04.890296",
     "exception": false,
     "start_time": "2021-08-04T16:54:03.422394",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from fastai.vision.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6c09b99",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:06.738326Z",
     "iopub.status.busy": "2021-08-04T16:54:06.737742Z",
     "iopub.status.idle": "2021-08-04T16:54:06.741597Z",
     "shell.execute_reply": "2021-08-04T16:54:06.741152Z",
     "shell.execute_reply.started": "2021-08-04T16:39:11.814210Z"
    },
    "papermill": {
     "duration": 0.311084,
     "end_time": "2021-08-04T16:54:06.741718",
     "exception": false,
     "start_time": "2021-08-04T16:54:06.430634",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"data_lane_segmentation\"\n",
    "\n",
    "\n",
    "x_train_dir = os.path.join(DATA_DIR, 'train')\n",
    "y_train_dir = os.path.join(DATA_DIR, 'train_label')\n",
    "\n",
    "x_valid_dir = os.path.join(DATA_DIR, 'val')\n",
    "y_valid_dir = os.path.join(DATA_DIR, 'val_label')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "360dc68e",
   "metadata": {
    "papermill": {
     "duration": 0.301466,
     "end_time": "2021-08-04T16:54:06.112594",
     "exception": false,
     "start_time": "2021-08-04T16:54:05.811128",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 2. Import fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed4d5c0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e98f05e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# some other usefuls libs\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "def get_image_array_from_fn(fn):\n",
    "    image = cv2.imread(fn)\n",
    "    return cv2.cvtColor(image, cv2.COLOR_BGR2RGB)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52d01bb5",
   "metadata": {
    "papermill": {
     "duration": 0.306593,
     "end_time": "2021-08-04T16:54:07.344927",
     "exception": false,
     "start_time": "2021-08-04T16:54:07.038334",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 3. Prepare data for usage with fastai library"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efa92203",
   "metadata": {
    "papermill": {
     "duration": 0.300404,
     "end_time": "2021-08-04T16:54:07.945113",
     "exception": false,
     "start_time": "2021-08-04T16:54:07.644709",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We will use a modified version of the fastai code for image segmentation that is given in the fastai documentation: https://docs.fast.ai/tutorial.vision.html#Segmentation---With-the-data-block-API"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3e7da8f",
   "metadata": {
    "papermill": {
     "duration": 0.298371,
     "end_time": "2021-08-04T16:54:08.543471",
     "exception": false,
     "start_time": "2021-08-04T16:54:08.245100",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### 3.1 label_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7b098a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sys import platform\n",
    "folder_token = \"\\\\\" if platform == \"win32\" else \"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44fbf2a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:09.149212Z",
     "iopub.status.busy": "2021-08-04T16:54:09.148351Z",
     "iopub.status.idle": "2021-08-04T16:54:09.151025Z",
     "shell.execute_reply": "2021-08-04T16:54:09.150622Z",
     "shell.execute_reply.started": "2021-08-04T16:39:14.894740Z"
    },
    "papermill": {
     "duration": 0.309389,
     "end_time": "2021-08-04T16:54:09.151153",
     "exception": false,
     "start_time": "2021-08-04T16:54:08.841764",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# function that takes filename of a training image 'fn' and returns the filename of the corresponding label image\n",
    "def label_func(fn): \n",
    "    return str(fn).replace(\".png\", \"_label.png\").replace(\"train\", \"train_label\").replace(\"val\"+folder_token, \"val_label\"+folder_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69e3d0fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:09.771441Z",
     "iopub.status.busy": "2021-08-04T16:54:09.770581Z",
     "iopub.status.idle": "2021-08-04T16:54:10.234412Z",
     "shell.execute_reply": "2021-08-04T16:54:10.233924Z",
     "shell.execute_reply.started": "2021-08-04T16:39:14.905402Z"
    },
    "papermill": {
     "duration": 0.786009,
     "end_time": "2021-08-04T16:54:10.234529",
     "exception": false,
     "start_time": "2021-08-04T16:54:09.448520",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pick the first image from the training directory and show it\n",
    "sample_fn = os.path.join(x_valid_dir, os.listdir(x_valid_dir)[0])\n",
    "print(sample_fn)\n",
    "plt.imshow(get_image_array_from_fn(sample_fn));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8af22aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:10.850400Z",
     "iopub.status.busy": "2021-08-04T16:54:10.849861Z",
     "iopub.status.idle": "2021-08-04T16:54:11.062371Z",
     "shell.execute_reply": "2021-08-04T16:54:11.062905Z",
     "shell.execute_reply.started": "2021-08-04T16:39:15.320444Z"
    },
    "papermill": {
     "duration": 0.52274,
     "end_time": "2021-08-04T16:54:11.063070",
     "exception": false,
     "start_time": "2021-08-04T16:54:10.540330",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# get corresponding label image using our 'label_func' function\n",
    "label_fn = label_func(sample_fn)\n",
    "print(label_fn)\n",
    "# we multiply the image intensity by 100 to make lane lines visible for the human eye:\n",
    "plt.imshow(100*get_image_array_from_fn(label_fn)); "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f1159d8",
   "metadata": {
    "papermill": {
     "duration": 0.396564,
     "end_time": "2021-08-04T16:54:11.775949",
     "exception": false,
     "start_time": "2021-08-04T16:54:11.379385",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### 3.2 get_image_files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90ef11bc",
   "metadata": {
    "papermill": {
     "duration": 0.328026,
     "end_time": "2021-08-04T16:54:12.409939",
     "exception": false,
     "start_time": "2021-08-04T16:54:12.081913",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "For the datablock API of the fastai library we need a function that takes a file path and returns a list of all the training images. We cannot **directly** use the built-in function 'get_image_files', since it would fetch all images, even the label images. Hence we define a function 'my_get_image_files' that does the same thing as 'get_image_files', just that it only looks into the folders \"train\" and \"val\". It will not look into \"train_label\" and \"val_label\". We can do this by inspecting the documentation of get_image_files on [docs.fast.ai](https://docs.fast.ai/data.transforms.html#get_image_files) and using ['partial'](https://www.geeksforgeeks.org/partial-functions-python/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bbc3247",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:13.019500Z",
     "iopub.status.busy": "2021-08-04T16:54:13.017698Z",
     "iopub.status.idle": "2021-08-04T16:54:13.024282Z",
     "shell.execute_reply": "2021-08-04T16:54:13.024863Z",
     "shell.execute_reply.started": "2021-08-04T16:39:15.697206Z"
    },
    "papermill": {
     "duration": 0.312265,
     "end_time": "2021-08-04T16:54:13.025029",
     "exception": false,
     "start_time": "2021-08-04T16:54:12.712764",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "my_get_image_files = partial(get_image_files, folders=[\"train\", \"val\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9540d468",
   "metadata": {
    "papermill": {
     "duration": 0.300873,
     "end_time": "2021-08-04T16:54:13.629336",
     "exception": false,
     "start_time": "2021-08-04T16:54:13.328463",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### 3.3 DataBlock, DataLoaders and data augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60c89b94",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:15.710820Z",
     "iopub.status.busy": "2021-08-04T16:54:15.710043Z",
     "iopub.status.idle": "2021-08-04T16:54:15.711876Z",
     "shell.execute_reply": "2021-08-04T16:54:15.711381Z",
     "shell.execute_reply.started": "2021-08-04T16:39:16.952135Z"
    },
    "papermill": {
     "duration": 0.339486,
     "end_time": "2021-08-04T16:54:15.712005",
     "exception": false,
     "start_time": "2021-08-04T16:54:15.372519",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "codes = np.array(['back', 'left','right'],dtype=str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fecc6d7f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:16.377418Z",
     "iopub.status.busy": "2021-08-04T16:54:16.376486Z",
     "iopub.status.idle": "2021-08-04T16:54:16.380279Z",
     "shell.execute_reply": "2021-08-04T16:54:16.379823Z",
     "shell.execute_reply.started": "2021-08-04T16:43:24.771203Z"
    },
    "papermill": {
     "duration": 0.343826,
     "end_time": "2021-08-04T16:54:16.380412",
     "exception": false,
     "start_time": "2021-08-04T16:54:16.036586",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "carla = DataBlock(blocks=(ImageBlock, MaskBlock(codes)),\n",
    "                   get_items = my_get_image_files,\n",
    "                   get_y = label_func,\n",
    "                   splitter = FuncSplitter(lambda x: str(x).find('validation_set')!=-1),\n",
    "                   batch_tfms=aug_transforms(do_flip=False, p_affine=0, p_lighting=0.75))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e72dca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:17.166138Z",
     "iopub.status.busy": "2021-08-04T16:54:17.164861Z",
     "iopub.status.idle": "2021-08-04T16:54:22.933661Z",
     "shell.execute_reply": "2021-08-04T16:54:22.932616Z",
     "shell.execute_reply.started": "2021-08-04T16:43:25.716958Z"
    },
    "papermill": {
     "duration": 6.102841,
     "end_time": "2021-08-04T16:54:22.933789",
     "exception": false,
     "start_time": "2021-08-04T16:54:16.830948",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dls = carla.dataloaders(Path(DATA_DIR), path=Path(\".\"), bs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ead65eff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:23.605328Z",
     "iopub.status.busy": "2021-08-04T16:54:23.604672Z",
     "iopub.status.idle": "2021-08-04T16:54:24.197721Z",
     "shell.execute_reply": "2021-08-04T16:54:24.196952Z",
     "shell.execute_reply.started": "2021-08-04T16:43:36.550259Z"
    },
    "papermill": {
     "duration": 0.934572,
     "end_time": "2021-08-04T16:54:24.197848",
     "exception": false,
     "start_time": "2021-08-04T16:54:23.263276",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dls.show_batch(max_n=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3da4f992",
   "metadata": {
    "papermill": {
     "duration": 0.328361,
     "end_time": "2021-08-04T16:54:24.859907",
     "exception": false,
     "start_time": "2021-08-04T16:54:24.531546",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 4. Model and training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43139a2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:26.483710Z",
     "iopub.status.busy": "2021-08-04T16:54:26.482575Z",
     "iopub.status.idle": "2021-08-04T16:54:27.294881Z",
     "shell.execute_reply": "2021-08-04T16:54:27.294414Z",
     "shell.execute_reply.started": "2021-08-02T09:59:05.105632Z"
    },
    "papermill": {
     "duration": 1.150237,
     "end_time": "2021-08-04T16:54:27.295012",
     "exception": false,
     "start_time": "2021-08-04T16:54:26.144775",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from fastseg import MobileV3Small\n",
    "\n",
    "model = MobileV3Small(num_classes=3, use_aspp=True, num_filters=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a14e92e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:27.964437Z",
     "iopub.status.busy": "2021-08-04T16:54:27.963744Z",
     "iopub.status.idle": "2021-08-04T16:54:27.966772Z",
     "shell.execute_reply": "2021-08-04T16:54:27.966277Z",
     "shell.execute_reply.started": "2021-08-02T09:59:06.280305Z"
    },
    "papermill": {
     "duration": 0.346916,
     "end_time": "2021-08-04T16:54:27.966891",
     "exception": false,
     "start_time": "2021-08-04T16:54:27.619975",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "learn = Learner(dls, model, metrics=[DiceMulti(), foreground_acc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "256ee17f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T16:54:28.667530Z",
     "iopub.status.busy": "2021-08-04T16:54:28.665192Z",
     "iopub.status.idle": "2021-08-04T17:12:16.758485Z",
     "shell.execute_reply": "2021-08-04T17:12:16.757992Z",
     "shell.execute_reply.started": "2021-08-02T09:59:10.270023Z"
    },
    "papermill": {
     "duration": 1068.462679,
     "end_time": "2021-08-04T17:12:16.758638",
     "exception": false,
     "start_time": "2021-08-04T16:54:28.295959",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "learn.fine_tune(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "410e6559",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:17.367528Z",
     "iopub.status.busy": "2021-08-04T17:12:17.366804Z",
     "iopub.status.idle": "2021-08-04T17:12:18.313811Z",
     "shell.execute_reply": "2021-08-04T17:12:18.313397Z",
     "shell.execute_reply.started": "2021-08-02T10:03:19.749417Z"
    },
    "papermill": {
     "duration": 1.253736,
     "end_time": "2021-08-04T17:12:18.313931",
     "exception": false,
     "start_time": "2021-08-04T17:12:17.060195",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "learn.show_results(max_n=6, figsize=(7,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10c0c3a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:19.693252Z",
     "iopub.status.busy": "2021-08-04T17:12:19.687063Z",
     "iopub.status.idle": "2021-08-04T17:12:19.718331Z",
     "shell.execute_reply": "2021-08-04T17:12:19.717884Z",
     "shell.execute_reply.started": "2021-08-02T08:32:28.405936Z"
    },
    "papermill": {
     "duration": 0.353179,
     "end_time": "2021-08-04T17:12:19.718457",
     "exception": false,
     "start_time": "2021-08-04T17:12:19.365278",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.save(learn.model, './fastai_model.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c0bd524",
   "metadata": {
    "papermill": {
     "duration": 0.299431,
     "end_time": "2021-08-04T17:12:20.317154",
     "exception": false,
     "start_time": "2021-08-04T17:12:20.017723",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Experiments with inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "deffc034",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:20.925460Z",
     "iopub.status.busy": "2021-08-04T17:12:20.924398Z",
     "iopub.status.idle": "2021-08-04T17:12:20.944833Z",
     "shell.execute_reply": "2021-08-04T17:12:20.944315Z",
     "shell.execute_reply.started": "2021-08-02T10:03:26.654095Z"
    },
    "papermill": {
     "duration": 0.329076,
     "end_time": "2021-08-04T17:12:20.944957",
     "exception": false,
     "start_time": "2021-08-04T17:12:20.615881",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "img = cv2.imread(str(get_image_files(x_valid_dir)[3]))\n",
    "img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c360d949",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:21.553596Z",
     "iopub.status.busy": "2021-08-04T17:12:21.552801Z",
     "iopub.status.idle": "2021-08-04T17:12:21.939651Z",
     "shell.execute_reply": "2021-08-04T17:12:21.940060Z",
     "shell.execute_reply.started": "2021-08-02T10:03:27.517688Z"
    },
    "papermill": {
     "duration": 0.698676,
     "end_time": "2021-08-04T17:12:21.940219",
     "exception": false,
     "start_time": "2021-08-04T17:12:21.241543",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plt.imshow(np.array(learn.predict(img)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab78a05b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:22.678052Z",
     "iopub.status.busy": "2021-08-04T17:12:22.677224Z",
     "iopub.status.idle": "2021-08-04T17:12:22.680069Z",
     "shell.execute_reply": "2021-08-04T17:12:22.679638Z",
     "shell.execute_reply.started": "2021-08-02T10:03:28.744211Z"
    },
    "papermill": {
     "duration": 0.439219,
     "end_time": "2021-08-04T17:12:22.680177",
     "exception": false,
     "start_time": "2021-08-04T17:12:22.240958",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# %timeit learn.predict(img); # => more than 100ms!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e34a364b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:23.302941Z",
     "iopub.status.busy": "2021-08-04T17:12:23.302060Z",
     "iopub.status.idle": "2021-08-04T17:12:23.305036Z",
     "shell.execute_reply": "2021-08-04T17:12:23.304597Z",
     "shell.execute_reply.started": "2021-08-02T10:03:29.235785Z"
    },
    "papermill": {
     "duration": 0.31245,
     "end_time": "2021-08-04T17:12:23.305148",
     "exception": false,
     "start_time": "2021-08-04T17:12:22.992698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_pred_for_mobilenet(model, img_array):\n",
    "    with torch.no_grad():\n",
    "        image_tensor = img_array.transpose(2,0,1).astype('float32')/255\n",
    "        x_tensor = torch.from_numpy(image_tensor).to(\"cuda\").unsqueeze(0)\n",
    "        model_output = F.softmax( model.forward(x_tensor), dim=1 ).cpu().numpy()\n",
    "    return model_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa6e3cd2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:24.189056Z",
     "iopub.status.busy": "2021-08-04T17:12:24.188167Z",
     "iopub.status.idle": "2021-08-04T17:12:24.190976Z",
     "shell.execute_reply": "2021-08-04T17:12:24.190540Z",
     "shell.execute_reply.started": "2021-08-02T10:03:29.696614Z"
    },
    "papermill": {
     "duration": 0.373577,
     "end_time": "2021-08-04T17:12:24.191129",
     "exception": false,
     "start_time": "2021-08-04T17:12:23.817552",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "learn.model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "934bb58f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:24.829604Z",
     "iopub.status.busy": "2021-08-04T17:12:24.828331Z",
     "iopub.status.idle": "2021-08-04T17:12:25.049099Z",
     "shell.execute_reply": "2021-08-04T17:12:25.049608Z",
     "shell.execute_reply.started": "2021-08-02T10:03:30.382262Z"
    },
    "papermill": {
     "duration": 0.546922,
     "end_time": "2021-08-04T17:12:25.049782",
     "exception": false,
     "start_time": "2021-08-04T17:12:24.502860",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "plt.imshow(get_pred_for_mobilenet(learn.model,img)[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66b85787",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:25.727144Z",
     "iopub.status.busy": "2021-08-04T17:12:25.725461Z",
     "iopub.status.idle": "2021-08-04T17:12:39.118286Z",
     "shell.execute_reply": "2021-08-04T17:12:39.117441Z",
     "shell.execute_reply.started": "2021-08-02T10:03:31.324371Z"
    },
    "papermill": {
     "duration": 13.733593,
     "end_time": "2021-08-04T17:12:39.118408",
     "exception": false,
     "start_time": "2021-08-04T17:12:25.384815",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%timeit get_pred_for_mobilenet(learn.model,img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ff8665c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-08-04T17:12:39.780474Z",
     "iopub.status.busy": "2021-08-04T17:12:39.779589Z",
     "iopub.status.idle": "2021-08-04T17:12:39.781453Z",
     "shell.execute_reply": "2021-08-04T17:12:39.780985Z",
     "shell.execute_reply.started": "2021-08-02T10:03:51.25621Z"
    },
    "papermill": {
     "duration": 0.333819,
     "end_time": "2021-08-04T17:12:39.781570",
     "exception": false,
     "start_time": "2021-08-04T17:12:39.447751",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# this is much faster!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3428b09",
   "metadata": {
    "papermill": {
     "duration": 0.334518,
     "end_time": "2021-08-04T17:12:44.318053",
     "exception": false,
     "start_time": "2021-08-04T17:12:43.983535",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1233.59645,
   "end_time": "2021-08-04T17:12:48.899150",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-08-04T16:52:15.302700",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}