{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "\n", "from experiments import ExperimentRunner\n", "import params" ] }, { "cell_type": "code", "execution_count": 64, "metadata": {}, "outputs": [], "source": [ "ex = ExperimentRunner(params.experiments, n_jobs=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'densenet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.001, 'use_vggish': False, 'momentum': 0.9}\n", "Downloading: \"https://download.pytorch.org/models/densenet121-a639ec97.pth\" to /Users/aziai/.cache/torch/checkpoints/densenet121-a639ec97.pth\n", "100%|██████████| 32342954/32342954 [00:03<00:00, 9210604.74it/s] \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Updating ALL params\n", "Epoch 0/9\n", "----------\n", "train Loss: 0.7583 F1: 0.6549 Acc: 0.5063\n", "val Loss: 0.6403 F1: 0.7969 Acc: 0.6941\n", "\n", "Epoch 1/9\n", "----------\n", "train Loss: 0.6932 F1: 0.6531 Acc: 0.5696\n", "val Loss: 0.7109 F1: 0.4898 Acc: 0.4118\n", "\n", "Epoch 2/9\n", "----------\n", "train Loss: 0.6125 F1: 0.6667 Acc: 0.6709\n", "val Loss: 0.7488 F1: 0.3820 Acc: 0.3529\n", "\n", "Epoch 3/9\n", "----------\n", "train Loss: 0.5145 F1: 0.8462 Acc: 0.8481\n", "val Loss: 0.6949 F1: 0.5455 Acc: 0.4706\n", "\n", "Epoch 4/9\n", "----------\n", "train Loss: 0.4379 F1: 0.9070 Acc: 0.8987\n", "val Loss: 0.6436 F1: 0.6607 Acc: 0.5529\n", "\n", "Epoch 5/9\n", "----------\n", "train Loss: 0.3507 F1: 0.9524 Acc: 0.9494\n", "val Loss: 0.6525 F1: 0.6019 Acc: 0.5176\n", "\n", "Epoch 6/9\n", "----------\n", "train Loss: 0.2782 F1: 0.9639 Acc: 0.9620\n", "val Loss: 0.7106 F1: 0.4946 Acc: 0.4471\n", "\n", "Epoch 7/9\n", "----------\n", "train Loss: 0.2353 F1: 0.9877 Acc: 0.9873\n", "val Loss: 0.7582 F1: 0.4889 Acc: 0.4588\n", "\n", "Epoch 8/9\n", "----------\n", "train Loss: 0.1920 F1: 0.9877 Acc: 0.9873\n", "val Loss: 0.7496 F1: 0.4946 Acc: 0.4471\n", "\n", "Epoch 9/9\n", "----------\n", "train Loss: 0.1625 F1: 0.9880 Acc: 0.9873\n", "val Loss: 0.7183 F1: 0.5657 Acc: 0.4941\n", "\n", "Training complete in 6m 48s\n", "Best val F1 : 0.796875\n", "Best val Acc : 0.694118\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'densenet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.001, 'use_vggish': True, 'momentum': 0.9}\n", "Downloading: \"https://users.cs.cf.ac.uk/taylorh23/pytorch/models/vggish-918c2d05.pth\" to /Users/aziai/.cache/torch/checkpoints/vggish-918c2d05.pth\n", "100%|██████████| 288567959/288567959 [04:26<00:00, 1082574.06it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Updating ALL params\n", "Epoch 0/9\n", "----------\n", "train Loss: 0.6834 F1: 0.4516 Acc: 0.5696\n", "val Loss: 0.6310 F1: 0.8154 Acc: 0.7176\n", "\n", "Epoch 1/9\n", "----------\n", "train Loss: 0.6429 F1: 0.6593 Acc: 0.6076\n", "val Loss: 0.5617 F1: 0.8966 Acc: 0.8235\n", "\n", "Epoch 2/9\n", "----------\n", "train Loss: 0.5885 F1: 0.7407 Acc: 0.6456\n", "val Loss: 0.5736 F1: 0.8951 Acc: 0.8235\n", "\n", "Epoch 3/9\n", "----------\n", "train Loss: 0.4830 F1: 0.8421 Acc: 0.8101\n", "val Loss: 0.6669 F1: 0.6275 Acc: 0.5529\n", "\n", "Epoch 4/9\n", "----------\n", "train Loss: 0.4137 F1: 0.9750 Acc: 0.9747\n", "val Loss: 0.7561 F1: 0.2597 Acc: 0.3294\n", "\n", "Epoch 5/9\n", "----------\n", "train Loss: 0.3440 F1: 0.9877 Acc: 0.9873\n", "val Loss: 0.7366 F1: 0.4419 Acc: 0.4353\n", "\n", "Epoch 6/9\n", "----------\n", "train Loss: 0.2656 F1: 0.9877 Acc: 0.9873\n", "val Loss: 0.6959 F1: 0.5895 Acc: 0.5412\n", "\n", "Epoch 7/9\n", "----------\n", "train Loss: 0.2115 F1: 1.0000 Acc: 1.0000\n", "val Loss: 0.6215 F1: 0.6981 Acc: 0.6235\n", "\n", "Epoch 8/9\n", "----------\n", "train Loss: 0.1942 F1: 0.9647 Acc: 0.9620\n", "val Loss: 0.6100 F1: 0.7273 Acc: 0.6471\n", "\n", "Epoch 9/9\n", "----------\n", "train Loss: 0.1419 F1: 1.0000 Acc: 1.0000\n", "val Loss: 0.7175 F1: 0.6535 Acc: 0.5882\n", "\n", "Training complete in 8m 22s\n", "Best val F1 : 0.896552\n", "Best val Acc : 0.823529\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': '/Users/aziai/Downloads/vtest_new2', 'conv_model_name': 'densenet', 'num_epochs': 10, 'feature_extract': True, 'batch_size': 64, 'lr': 0.001, 'use_vggish': False, 'momentum': 0.9}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Params to update\n", "* combined.weight\n", "* combined.bias\n", "Epoch 0/9\n", "----------\n", "train Loss: 0.6883 F1: 0.6538 Acc: 0.5443\n", "val Loss: 0.7030 F1: 0.6261 Acc: 0.4941\n", "\n", "Epoch 1/9\n", "----------\n", "train Loss: 0.6528 F1: 0.6729 Acc: 0.5570\n", "val Loss: 0.6945 F1: 0.6306 Acc: 0.5176\n", "\n", "Epoch 2/9\n", "----------\n", "train Loss: 0.6262 F1: 0.7184 Acc: 0.6329\n", "val Loss: 0.7183 F1: 0.6154 Acc: 0.5294\n", "\n", "Epoch 3/9\n", "----------\n", "train Loss: 0.5804 F1: 0.7191 Acc: 0.6835\n", "val Loss: 0.7631 F1: 0.4667 Acc: 0.4353\n", "\n", "Epoch 4/9\n", "----------\n", "train Loss: 0.5529 F1: 0.7179 Acc: 0.7215\n", "val Loss: 0.7377 F1: 0.5161 Acc: 0.4706\n", "\n", "Epoch 5/9\n", "----------\n", "train Loss: 0.4921 F1: 0.8434 Acc: 0.8354\n", "val Loss: 0.6402 F1: 0.7500 Acc: 0.6706\n", "\n", "Epoch 6/9\n", "----------\n", "train Loss: 0.4559 F1: 0.8667 Acc: 0.8481\n", "val Loss: 0.5956 F1: 0.8455 Acc: 0.7765\n", "\n", "Epoch 7/9\n", "----------\n", "train Loss: 0.4243 F1: 0.8723 Acc: 0.8481\n", "val Loss: 0.5929 F1: 0.8167 Acc: 0.7412\n", "\n", "Epoch 8/9\n", "----------\n" ] } ], "source": [ "ex.run()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "paths = glob.glob('results/*.csv') # * means all if need specific format then *.csv\n", "latest = max(paths, key=os.path.getctime)\n", "df = pd.read_csv(latest)" ] }, { "cell_type": "code", "execution_count": 62, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
batch_sizeconv_model_namedata_path_baseexperiment_uuidfeature_extractlrmomentumnum_epochsrunner_uuiduse_vggishval_accval_f1
564resnet/Users/aziai/Downloads/vtest_new220190602194822False0.0010.910b3290017112d2116e4bdbb9c8dbf15a8e75adacb942afb...True0.8941180.935252
164NaN/Users/aziai/Downloads/vtest_new220190602194822False0.0010.9104687cd536acf6c9b058a8d20719fa6910f50a6abee3ae2...True0.8470590.897638
464resnet/Users/aziai/Downloads/vtest_new220190602194822False0.0010.9100e57debc92afdd0dc7a209584b4d97860c9dba98f3aed4...False0.8235290.878049
364NaN/Users/aziai/Downloads/vtest_new220190602194822True0.0010.910de7011637531360e5c76520f054d90327cc478dbabeab9...True0.7764710.874172
764resnet/Users/aziai/Downloads/vtest_new220190602194822True0.0010.9108e644bc291a463725bf0bcb11825a196383a4860eeecd7...True0.7294120.824427
664resnet/Users/aziai/Downloads/vtest_new220190602194822True0.0010.910e07e5119b07164f06098d1adba9e4c43ad0344716a0746...False0.5647060.626263
064NaN/Users/aziai/Downloads/vtest_new220190602194822False0.0010.9107324021853e0ca109ff0effaf0c9d06e68d72744305f34...False-1.000000-1.000000
264NaN/Users/aziai/Downloads/vtest_new220190602194822True0.0010.9109ca13084e79d88cec23574c0c37fa9109fe87a7026f9bd...False-1.000000-1.000000
\n", "
" ], "text/plain": [ " batch_size conv_model_name data_path_base experiment_uuid feature_extract lr momentum num_epochs runner_uuid use_vggish val_acc val_f1\n", "5 64 resnet /Users/aziai/Downloads/vtest_new2 20190602194822 False 0.001 0.9 10 b3290017112d2116e4bdbb9c8dbf15a8e75adacb942afb... True 0.894118 0.935252\n", "1 64 NaN /Users/aziai/Downloads/vtest_new2 20190602194822 False 0.001 0.9 10 4687cd536acf6c9b058a8d20719fa6910f50a6abee3ae2... True 0.847059 0.897638\n", "4 64 resnet /Users/aziai/Downloads/vtest_new2 20190602194822 False 0.001 0.9 10 0e57debc92afdd0dc7a209584b4d97860c9dba98f3aed4... False 0.823529 0.878049\n", "3 64 NaN /Users/aziai/Downloads/vtest_new2 20190602194822 True 0.001 0.9 10 de7011637531360e5c76520f054d90327cc478dbabeab9... True 0.776471 0.874172\n", "7 64 resnet /Users/aziai/Downloads/vtest_new2 20190602194822 True 0.001 0.9 10 8e644bc291a463725bf0bcb11825a196383a4860eeecd7... True 0.729412 0.824427\n", "6 64 resnet /Users/aziai/Downloads/vtest_new2 20190602194822 True 0.001 0.9 10 e07e5119b07164f06098d1adba9e4c43ad0344716a0746... False 0.564706 0.626263\n", "0 64 NaN /Users/aziai/Downloads/vtest_new2 20190602194822 False 0.001 0.9 10 7324021853e0ca109ff0effaf0c9d06e68d72744305f34... False -1.000000 -1.000000\n", "2 64 NaN /Users/aziai/Downloads/vtest_new2 20190602194822 True 0.001 0.9 10 9ca13084e79d88cec23574c0c37fa9109fe87a7026f9bd... False -1.000000 -1.000000" ] }, "execution_count": 62, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.sort_values(by='val_f1', ascending=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.2" } }, "nbformat": 4, "nbformat_minor": 2 }