{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "\n", "from experiments import ExperimentRunner\n", "import params" ] }, { "cell_type": "code", "execution_count": 71, "metadata": {}, "outputs": [], "source": [ "ex = ExperimentRunner(params.experiments, n_jobs=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': 'vtest_new2', 'conv_model_name': 'densenet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.01, 'use_vggish': False, 'momentum': 0.9}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Updating ALL params\n", "Epoch 0/9\n", "----------\n", "train Loss: 0.8410 F1: 0.3704 Acc: 0.5696\n", "val Loss: 0.6466 F1: 0.8667 Acc: 0.7647\n", "\n", "Epoch 1/9\n", "----------\n", "train Loss: 1.0247 F1: 0.6949 Acc: 0.5443\n", "val Loss: 1.7111 F1: nan Acc: 0.2235\n", "\n", "Epoch 2/9\n", "----------\n", "train Loss: 0.4048 F1: 0.6557 Acc: 0.7342\n", "val Loss: 1.5030 F1: 0.1370 Acc: 0.2588\n", "\n", "Epoch 3/9\n", "----------\n", "train Loss: 0.0870 F1: 0.9877 Acc: 0.9873\n", "val Loss: 0.6543 F1: 0.7627 Acc: 0.6706\n", "\n", "Epoch 4/9\n", "----------\n", "train Loss: 0.0952 F1: 0.9535 Acc: 0.9494\n", "val Loss: 0.6773 F1: 0.8358 Acc: 0.7412\n", "\n", "Epoch 5/9\n", "----------\n", "train Loss: 0.0807 F1: 0.9647 Acc: 0.9620\n", "val Loss: 0.8060 F1: 0.8182 Acc: 0.7176\n", "\n", "Epoch 6/9\n", "----------\n", "train Loss: 0.0083 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.2097 F1: 0.6667 Acc: 0.5529\n", "\n", "Epoch 7/9\n", "----------\n", "train Loss: 0.0021 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.7171 F1: 0.5102 Acc: 0.4353\n", "\n", "Epoch 8/9\n", "----------\n", "train Loss: 0.0003 F1: 1.0000 Acc: 1.0000\n", "val Loss: 2.0735 F1: 0.4421 Acc: 0.3765\n", "\n", "Epoch 9/9\n", "----------\n", "train Loss: 0.0002 F1: 1.0000 Acc: 1.0000\n", "val Loss: 2.5907 F1: 0.3218 Acc: 0.3059\n", "\n", "Training complete in 6m 49s\n", "Best val F1 : 0.866667\n", "Best val Acc : 0.764706\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': 'vtest_new2', 'conv_model_name': 'densenet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.01, 'use_vggish': False, 'momentum': 0.95}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Updating ALL params\n", "Epoch 0/9\n", "----------\n", "train Loss: 0.6807 F1: 0.6667 Acc: 0.5696\n", "val Loss: 0.6620 F1: 0.6729 Acc: 0.5882\n", "\n", "Epoch 1/9\n", "----------\n", "train Loss: 0.3776 F1: 0.9425 Acc: 0.9367\n", "val Loss: 0.8433 F1: 0.3000 Acc: 0.3412\n", "\n", "Epoch 2/9\n", "----------\n", "train Loss: 0.1782 F1: 0.9750 Acc: 0.9747\n", "val Loss: 0.7165 F1: 0.5376 Acc: 0.4941\n", "\n", "Epoch 3/9\n", "----------\n", "train Loss: 0.1016 F1: 0.9762 Acc: 0.9747\n", "val Loss: 0.6558 F1: 0.6981 Acc: 0.6235\n", "\n", "Epoch 4/9\n", "----------\n", "train Loss: 0.0343 F1: 1.0000 Acc: 1.0000\n", "val Loss: 0.8724 F1: 0.5111 Acc: 0.4824\n", "\n", "Epoch 5/9\n", "----------\n", "train Loss: 0.0107 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.0340 F1: 0.4889 Acc: 0.4588\n", "\n", "Epoch 6/9\n", "----------\n", "train Loss: 0.0066 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.3202 F1: 0.4186 Acc: 0.4118\n", "\n", "Epoch 7/9\n", "----------\n", "train Loss: 0.0046 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.5957 F1: 0.3953 Acc: 0.3882\n", "\n", "Epoch 8/9\n", "----------\n", "train Loss: 0.0015 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.9662 F1: 0.4186 Acc: 0.4118\n", "\n", "Epoch 9/9\n", "----------\n", "train Loss: 0.0007 F1: 1.0000 Acc: 1.0000\n", "val Loss: 2.1900 F1: 0.4045 Acc: 0.3765\n", "\n", "Training complete in 6m 60s\n", "Best val F1 : 0.698113\n", "Best val Acc : 0.623529\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': 'vtest_new2', 'conv_model_name': 'densenet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.01, 'use_vggish': True, 'momentum': 0.9}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Updating ALL params\n", "Epoch 0/9\n", "----------\n", "train Loss: 0.6751 F1: 0.5352 Acc: 0.5823\n", "val Loss: 0.5813 F1: 0.8742 Acc: 0.7765\n", "\n", "Epoch 1/9\n", "----------\n", "train Loss: 0.6840 F1: 0.7130 Acc: 0.5823\n", "val Loss: 2.9018 F1: nan Acc: 0.2235\n", "\n", "Epoch 2/9\n", "----------\n", "train Loss: 1.1845 F1: nan Acc: 0.4810\n", "val Loss: 0.6255 F1: 0.7581 Acc: 0.6471\n", "\n", "Epoch 3/9\n", "----------\n", "train Loss: 0.1377 F1: 0.9535 Acc: 0.9494\n", "val Loss: 1.1300 F1: 0.8800 Acc: 0.7882\n", "\n", "Epoch 4/9\n", "----------\n", "train Loss: 1.2311 F1: 0.7321 Acc: 0.6203\n", "val Loss: 0.9068 F1: 0.8321 Acc: 0.7294\n", "\n", "Epoch 5/9\n", "----------\n", "train Loss: 0.0355 F1: 0.9756 Acc: 0.9747\n", "val Loss: 2.1568 F1: 0.3059 Acc: 0.3059\n", "\n", "Epoch 6/9\n", "----------\n", "train Loss: 0.0030 F1: 1.0000 Acc: 1.0000\n", "val Loss: 6.2178 F1: nan Acc: 0.2235\n", "\n", "Epoch 7/9\n", "----------\n", "train Loss: 0.0112 F1: 1.0000 Acc: 1.0000\n", "val Loss: 8.0201 F1: nan Acc: 0.2235\n", "\n", "Epoch 8/9\n", "----------\n", "train Loss: 0.0022 F1: 1.0000 Acc: 1.0000\n", "val Loss: 9.1074 F1: nan Acc: 0.2235\n", "\n", "Epoch 9/9\n", "----------\n", "train Loss: 0.0037 F1: 1.0000 Acc: 1.0000\n", "val Loss: 9.3362 F1: nan Acc: 0.2235\n", "\n", "Training complete in 7m 50s\n", "Best val F1 : 0.880000\n", "Best val Acc : 0.788235\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': 'vtest_new2', 'conv_model_name': 'densenet', 'num_epochs': 10, 'feature_extract': False, 'batch_size': 64, 'lr': 0.01, 'use_vggish': True, 'momentum': 0.95}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Updating ALL params\n", "Epoch 0/9\n", "----------\n", "train Loss: 0.6752 F1: 0.5526 Acc: 0.5696\n", "val Loss: 0.5195 F1: 0.8742 Acc: 0.7765\n", "\n", "Epoch 1/9\n", "----------\n", "train Loss: 0.5782 F1: 0.7387 Acc: 0.6329\n", "val Loss: 1.4869 F1: 0.0299 Acc: 0.2353\n", "\n", "Epoch 2/9\n", "----------\n", "train Loss: 0.3623 F1: 0.7941 Acc: 0.8228\n", "val Loss: 0.5903 F1: 0.7692 Acc: 0.6824\n", "\n", "Epoch 3/9\n", "----------\n", "train Loss: 0.0622 F1: 0.9880 Acc: 0.9873\n", "val Loss: 0.4745 F1: 0.8714 Acc: 0.7882\n", "\n", "Epoch 4/9\n", "----------\n", "train Loss: 0.0707 F1: 0.9762 Acc: 0.9747\n", "val Loss: 0.5149 F1: 0.8759 Acc: 0.8000\n", "\n", "Epoch 5/9\n", "----------\n", "train Loss: 0.0126 F1: 1.0000 Acc: 1.0000\n", "val Loss: 0.6557 F1: 0.8244 Acc: 0.7294\n", "\n", "Epoch 6/9\n", "----------\n", "train Loss: 0.0048 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.0149 F1: 0.7069 Acc: 0.6000\n", "\n", "Epoch 7/9\n", "----------\n", "train Loss: 0.0042 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.3682 F1: 0.7009 Acc: 0.5882\n", "\n", "Epoch 8/9\n", "----------\n", "train Loss: 0.0025 F1: 1.0000 Acc: 1.0000\n", "val Loss: 1.7442 F1: 0.6607 Acc: 0.5529\n", "\n", "Epoch 9/9\n", "----------\n" ] } ], "source": [ "ex.run()" ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 69, "metadata": {}, "outputs": [], "source": [ "paths = glob.glob('results/*.csv') # * means all if need specific format then *.csv\n", "latest = max(paths, key=os.path.getctime)\n", "df = pd.read_csv(latest)" ] }, { "cell_type": "code", "execution_count": 70, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
batch_sizeconv_model_namedata_path_baseexperiment_uuidfeature_extractlrmomentumnum_epochsrunner_uuiduse_vggishval_accval_f1
964resnet/Users/aziai/Downloads/vtest_new220190602195657False0.0010.910b3290017112d2116e4bdbb9c8dbf15a8e75adacb942afb...True0.8941180.935252
564NaN/Users/aziai/Downloads/vtest_new220190602195657False0.0010.9104687cd536acf6c9b058a8d20719fa6910f50a6abee3ae2...True0.8470590.897638
164densenet/Users/aziai/Downloads/vtest_new220190602195657False0.0010.910697154af4871003b02cf60d53531fb21a80e853646b458...True0.8235290.896552
864resnet/Users/aziai/Downloads/vtest_new220190602195657False0.0010.9100e57debc92afdd0dc7a209584b4d97860c9dba98f3aed4...False0.8235290.878049
764NaN/Users/aziai/Downloads/vtest_new220190602195657True0.0010.910de7011637531360e5c76520f054d90327cc478dbabeab9...True0.7764710.874172
264densenet/Users/aziai/Downloads/vtest_new220190602195657True0.0010.91089b8a27230c7f2ee2dc0ece6fd1f9deccae873fce8288f...False0.7764710.845528
1164resnet/Users/aziai/Downloads/vtest_new220190602195657True0.0010.9108e644bc291a463725bf0bcb11825a196383a4860eeecd7...True0.7294120.824427
064densenet/Users/aziai/Downloads/vtest_new220190602195657False0.0010.9105c8b5c5ceb49ba7d53ccc921e116ae183fc5b44037c410...False0.6941180.796875
364densenet/Users/aziai/Downloads/vtest_new220190602195657True0.0010.91031c3e541f0e5d5b5c1823de1f23c1bc4c8c4be3c22eef1...True0.5294120.629630
1064resnet/Users/aziai/Downloads/vtest_new220190602195657True0.0010.910e07e5119b07164f06098d1adba9e4c43ad0344716a0746...False0.5647060.626263
464NaN/Users/aziai/Downloads/vtest_new220190602195657False0.0010.9107324021853e0ca109ff0effaf0c9d06e68d72744305f34...False-1.000000-1.000000
664NaN/Users/aziai/Downloads/vtest_new220190602195657True0.0010.9109ca13084e79d88cec23574c0c37fa9109fe87a7026f9bd...False-1.000000-1.000000
\n", "
" ], "text/plain": [ " batch_size conv_model_name data_path_base experiment_uuid feature_extract lr momentum num_epochs runner_uuid use_vggish val_acc val_f1\n", "9 64 resnet /Users/aziai/Downloads/vtest_new2 20190602195657 False 0.001 0.9 10 b3290017112d2116e4bdbb9c8dbf15a8e75adacb942afb... True 0.894118 0.935252\n", "5 64 NaN /Users/aziai/Downloads/vtest_new2 20190602195657 False 0.001 0.9 10 4687cd536acf6c9b058a8d20719fa6910f50a6abee3ae2... True 0.847059 0.897638\n", "1 64 densenet /Users/aziai/Downloads/vtest_new2 20190602195657 False 0.001 0.9 10 697154af4871003b02cf60d53531fb21a80e853646b458... True 0.823529 0.896552\n", "8 64 resnet /Users/aziai/Downloads/vtest_new2 20190602195657 False 0.001 0.9 10 0e57debc92afdd0dc7a209584b4d97860c9dba98f3aed4... False 0.823529 0.878049\n", "7 64 NaN /Users/aziai/Downloads/vtest_new2 20190602195657 True 0.001 0.9 10 de7011637531360e5c76520f054d90327cc478dbabeab9... True 0.776471 0.874172\n", "2 64 densenet /Users/aziai/Downloads/vtest_new2 20190602195657 True 0.001 0.9 10 89b8a27230c7f2ee2dc0ece6fd1f9deccae873fce8288f... False 0.776471 0.845528\n", "11 64 resnet /Users/aziai/Downloads/vtest_new2 20190602195657 True 0.001 0.9 10 8e644bc291a463725bf0bcb11825a196383a4860eeecd7... True 0.729412 0.824427\n", "0 64 densenet /Users/aziai/Downloads/vtest_new2 20190602195657 False 0.001 0.9 10 5c8b5c5ceb49ba7d53ccc921e116ae183fc5b44037c410... False 0.694118 0.796875\n", "3 64 densenet /Users/aziai/Downloads/vtest_new2 20190602195657 True 0.001 0.9 10 31c3e541f0e5d5b5c1823de1f23c1bc4c8c4be3c22eef1... True 0.529412 0.629630\n", "10 64 resnet /Users/aziai/Downloads/vtest_new2 20190602195657 True 0.001 0.9 10 e07e5119b07164f06098d1adba9e4c43ad0344716a0746... False 0.564706 0.626263\n", "4 64 NaN /Users/aziai/Downloads/vtest_new2 20190602195657 False 0.001 0.9 10 7324021853e0ca109ff0effaf0c9d06e68d72744305f34... False -1.000000 -1.000000\n", "6 64 NaN /Users/aziai/Downloads/vtest_new2 20190602195657 True 0.001 0.9 10 9ca13084e79d88cec23574c0c37fa9109fe87a7026f9bd... False -1.000000 -1.000000" ] }, "execution_count": 70, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.sort_values(by='val_f1', ascending=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.2" } }, "nbformat": 4, "nbformat_minor": 2 }