add experiment runner

dev4.ipynb

@@ -0,0 +1,261 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import glob\n",
+    "\n",
+    "from experiments import ExperimentRunner\n",
+    "import params"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ex = ExperimentRunner(params.experiment1, n_jobs=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'vgg', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.001, 'use_vggish': False, 'momentum': 0.9}\n",
+      "Downloading: \"https://download.pytorch.org/models/vgg11_bn-6002323d.pth\" to /Users/aziai/.cache/torch/checkpoints/vgg11_bn-6002323d.pth\n",
+      "100%|██████████| 531503671/531503671 [00:40<00:00, 13079409.73it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Updating ALL params\n",
+      "Epoch 0/9\n",
+      "----------\n",
+      "train Loss: 1.0483 F1: 0.2951 Acc: 0.4557\n",
+      "val Loss: 0.5709 F1: 0.8189 Acc: 0.7294\n",
+      "\n",
+      "Epoch 1/9\n",
+      "----------\n",
+      "train Loss: 0.5769 F1: 0.7872 Acc: 0.7468\n",
+      "val Loss: 0.3201 F1: 0.9353 Acc: 0.8941\n",
+      "\n",
+      "Epoch 2/9\n",
+      "----------\n",
+      "train Loss: 0.3647 F1: 0.8247 Acc: 0.7848\n",
+      "val Loss: 0.4309 F1: 0.8333 Acc: 0.7647\n",
+      "\n",
+      "Epoch 3/9\n",
+      "----------\n",
+      "train Loss: 0.2243 F1: 0.8571 Acc: 0.8608\n",
+      "val Loss: 0.7989 F1: 0.6796 Acc: 0.6118\n",
+      "\n",
+      "Epoch 4/9\n",
+      "----------\n",
+      "train Loss: 0.1799 F1: 0.9231 Acc: 0.9241\n",
+      "val Loss: 0.8629 F1: 0.7407 Acc: 0.6706\n",
+      "\n",
+      "Epoch 5/9\n",
+      "----------\n"
+     ]
+    }
+   ],
+   "source": [
+    "ex.run()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "paths = glob.glob('results/*.csv') # * means all if need specific format then *.csv\n",
+    "latest = max(paths, key=os.path.getctime)\n",
+    "df = pd.read_csv(latest)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>batch_size</th>\n",
+       "      <th>conv_model_name</th>\n",
+       "      <th>data_path_base</th>\n",
+       "      <th>experiment_uuid</th>\n",
+       "      <th>feature_extract</th>\n",
+       "      <th>lr</th>\n",
+       "      <th>momentum</th>\n",
+       "      <th>num_epochs</th>\n",
+       "      <th>runner_uuid</th>\n",
+       "      <th>use_vggish</th>\n",
+       "      <th>val_acc</th>\n",
+       "      <th>val_f1</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>64</td>\n",
+       "      <td>resnet</td>\n",
+       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
+       "      <td>20191402191416</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.9</td>\n",
+       "      <td>10</td>\n",
+       "      <td>b3290017112d2116e4bdbb9c8dbf15a8e75adacb942afb...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.894118</td>\n",
+       "      <td>0.935252</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>64</td>\n",
+       "      <td>resnet</td>\n",
+       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
+       "      <td>20191402191416</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.9</td>\n",
+       "      <td>10</td>\n",
+       "      <td>0e57debc92afdd0dc7a209584b4d97860c9dba98f3aed4...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0.823529</td>\n",
+       "      <td>0.878049</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>64</td>\n",
+       "      <td>resnet</td>\n",
+       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
+       "      <td>20191402191416</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.9</td>\n",
+       "      <td>10</td>\n",
+       "      <td>8e644bc291a463725bf0bcb11825a196383a4860eeecd7...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.729412</td>\n",
+       "      <td>0.824427</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>64</td>\n",
+       "      <td>resnet</td>\n",
+       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
+       "      <td>20191402191416</td>\n",
+       "      <td>True</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.9</td>\n",
+       "      <td>10</td>\n",
+       "      <td>e07e5119b07164f06098d1adba9e4c43ad0344716a0746...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0.564706</td>\n",
+       "      <td>0.626263</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   batch_size conv_model_name                     data_path_base  ...  use_vggish   val_acc    val_f1\n",
+       "1          64          resnet  /Users/aziai/Downloads/vtest_new2  ...        True  0.894118  0.935252\n",
+       "0          64          resnet  /Users/aziai/Downloads/vtest_new2  ...       False  0.823529  0.878049\n",
+       "3          64          resnet  /Users/aziai/Downloads/vtest_new2  ...        True  0.729412  0.824427\n",
+       "2          64          resnet  /Users/aziai/Downloads/vtest_new2  ...       False  0.564706  0.626263\n",
+       "\n",
+       "[4 rows x 12 columns]"
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.sort_values(by='val_f1', ascending=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}