{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "\n",
    "from experiments import ExperimentRunner\n",
    "import params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex = ExperimentRunner(params.experiments, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': None, 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.001, 'use_vggish': False, 'momentum': 0.9}\n",
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': None, 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.001, 'use_vggish': True, 'momentum': 0.9}\n",
      "Loading experiment results from cache\n",
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': None, 'num_epochs': 10, 'feature_extract': True, 'batch_size': 64, 'lr': 0.001, 'use_vggish': False, 'momentum': 0.9}\n",
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': None, 'num_epochs': 10, 'feature_extract': True, 'batch_size': 64, 'lr': 0.001, 'use_vggish': True, 'momentum': 0.9}\n",
      "Loading experiment results from cache\n",
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'resnet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.001, 'use_vggish': False, 'momentum': 0.9}\n",
      "Loading experiment results from cache\n",
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'resnet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.001, 'use_vggish': True, 'momentum': 0.9}\n",
      "Loading experiment results from cache\n",
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'resnet', 'num_epochs': 10, 'feature_extract': True, 'batch_size': 64, 'lr': 0.001, 'use_vggish': False, 'momentum': 0.9}\n",
      "Loading experiment results from cache\n",
      "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'resnet', 'num_epochs': 10, 'feature_extract': True, 'batch_size': 64, 'lr': 0.001, 'use_vggish': True, 'momentum': 0.9}\n",
      "Loading experiment results from cache\n"
     ]
    }
   ],
   "source": [
    "ex.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths = glob.glob('results/*.csv') # * means all if need specific format then *.csv\n",
    "latest = max(paths, key=os.path.getctime)\n",
    "df = pd.read_csv(latest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batch_size</th>\n",
       "      <th>conv_model_name</th>\n",
       "      <th>data_path_base</th>\n",
       "      <th>experiment_uuid</th>\n",
       "      <th>feature_extract</th>\n",
       "      <th>lr</th>\n",
       "      <th>momentum</th>\n",
       "      <th>num_epochs</th>\n",
       "      <th>runner_uuid</th>\n",
       "      <th>use_vggish</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>64</td>\n",
       "      <td>resnet</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>False</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>b3290017112d2116e4bdbb9c8dbf15a8e75adacb942afb...</td>\n",
       "      <td>True</td>\n",
       "      <td>0.894118</td>\n",
       "      <td>0.935252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>False</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>4687cd536acf6c9b058a8d20719fa6910f50a6abee3ae2...</td>\n",
       "      <td>True</td>\n",
       "      <td>0.847059</td>\n",
       "      <td>0.897638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>64</td>\n",
       "      <td>resnet</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>False</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>0e57debc92afdd0dc7a209584b4d97860c9dba98f3aed4...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.823529</td>\n",
       "      <td>0.878049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>64</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>True</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>de7011637531360e5c76520f054d90327cc478dbabeab9...</td>\n",
       "      <td>True</td>\n",
       "      <td>0.776471</td>\n",
       "      <td>0.874172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>64</td>\n",
       "      <td>resnet</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>True</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>8e644bc291a463725bf0bcb11825a196383a4860eeecd7...</td>\n",
       "      <td>True</td>\n",
       "      <td>0.729412</td>\n",
       "      <td>0.824427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>64</td>\n",
       "      <td>resnet</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>True</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>e07e5119b07164f06098d1adba9e4c43ad0344716a0746...</td>\n",
       "      <td>False</td>\n",
       "      <td>0.564706</td>\n",
       "      <td>0.626263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>False</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>7324021853e0ca109ff0effaf0c9d06e68d72744305f34...</td>\n",
       "      <td>False</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>64</td>\n",
       "      <td>NaN</td>\n",
       "      <td>/Users/aziai/Downloads/vtest_new2</td>\n",
       "      <td>20190602194822</td>\n",
       "      <td>True</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10</td>\n",
       "      <td>9ca13084e79d88cec23574c0c37fa9109fe87a7026f9bd...</td>\n",
       "      <td>False</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   batch_size conv_model_name                     data_path_base  experiment_uuid  feature_extract     lr  momentum  num_epochs                                        runner_uuid  use_vggish   val_acc    val_f1\n",
       "5          64          resnet  /Users/aziai/Downloads/vtest_new2   20190602194822            False  0.001       0.9          10  b3290017112d2116e4bdbb9c8dbf15a8e75adacb942afb...        True  0.894118  0.935252\n",
       "1          64             NaN  /Users/aziai/Downloads/vtest_new2   20190602194822            False  0.001       0.9          10  4687cd536acf6c9b058a8d20719fa6910f50a6abee3ae2...        True  0.847059  0.897638\n",
       "4          64          resnet  /Users/aziai/Downloads/vtest_new2   20190602194822            False  0.001       0.9          10  0e57debc92afdd0dc7a209584b4d97860c9dba98f3aed4...       False  0.823529  0.878049\n",
       "3          64             NaN  /Users/aziai/Downloads/vtest_new2   20190602194822             True  0.001       0.9          10  de7011637531360e5c76520f054d90327cc478dbabeab9...        True  0.776471  0.874172\n",
       "7          64          resnet  /Users/aziai/Downloads/vtest_new2   20190602194822             True  0.001       0.9          10  8e644bc291a463725bf0bcb11825a196383a4860eeecd7...        True  0.729412  0.824427\n",
       "6          64          resnet  /Users/aziai/Downloads/vtest_new2   20190602194822             True  0.001       0.9          10  e07e5119b07164f06098d1adba9e4c43ad0344716a0746...       False  0.564706  0.626263\n",
       "0          64             NaN  /Users/aziai/Downloads/vtest_new2   20190602194822            False  0.001       0.9          10  7324021853e0ca109ff0effaf0c9d06e68d72744305f34...       False -1.000000 -1.000000\n",
       "2          64             NaN  /Users/aziai/Downloads/vtest_new2   20190602194822             True  0.001       0.9          10  9ca13084e79d88cec23574c0c37fa9109fe87a7026f9bd...       False -1.000000 -1.000000"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values(by='val_f1', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}