{ "cells": [ { "cell_type": "code", "execution_count": 95, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 102, "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "\n", "from experiments import ExperimentRunner\n", "import params" ] }, { "cell_type": "code", "execution_count": 106, "metadata": {}, "outputs": [], "source": [ "ex = ExperimentRunner(params.experiments, n_jobs=1)" ] }, { "cell_type": "code", "execution_count": 104, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Running param set: {'data_path_base': 'vtest_new2', 'conv_model_name': 'resnet', 'num_epochs': 20, 'feature_extract': True, 'batch_size': 64, 'lr': 0.001, 'use_vggish': True, 'momentum': 0.9}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Params to update\n", "* combined.weight\n", "* combined.bias\n", "Epoch 0/19\n", "----------\n", "train Loss: 0.6817 F1: 0.5897 Acc: 0.5949\n", "val Loss: 0.7901 F1: 0.4043 Acc: 0.3412\n", "\n", "Epoch 1/19\n", "----------\n", "train Loss: 0.6104 F1: 0.7294 Acc: 0.7089\n", "val Loss: 0.6452 F1: 0.8125 Acc: 0.7176\n", "\n", "Epoch 2/19\n", "----------\n", "train Loss: 0.6116 F1: 0.7222 Acc: 0.6203\n", "val Loss: 0.6177 F1: 0.8217 Acc: 0.7294\n", "\n", "Epoch 3/19\n", "----------\n", "train Loss: 0.5695 F1: 0.7692 Acc: 0.6962\n", "val Loss: 0.6339 F1: 0.8095 Acc: 0.7176\n", "\n", "Epoch 4/19\n", "----------\n", "train Loss: 0.5504 F1: 0.7816 Acc: 0.7595\n", "val Loss: 0.6923 F1: 0.5743 Acc: 0.4941\n", "\n", "Epoch 5/19\n", "----------\n", "train Loss: 0.4833 F1: 0.8148 Acc: 0.8101\n", "val Loss: 0.6607 F1: 0.6789 Acc: 0.5882\n", "\n", "Epoch 6/19\n", "----------\n", "train Loss: 0.4592 F1: 0.8293 Acc: 0.8228\n", "val Loss: 0.6051 F1: 0.8130 Acc: 0.7294\n", "\n", "Epoch 7/19\n", "----------\n", "train Loss: 0.4290 F1: 0.8706 Acc: 0.8608\n", "val Loss: 0.5585 F1: 0.8722 Acc: 0.8000\n", "\n", "Epoch 8/19\n", "----------\n", "train Loss: 0.4170 F1: 0.8696 Acc: 0.8481\n", "val Loss: 0.5424 F1: 0.8722 Acc: 0.8000\n", "\n", "Epoch 9/19\n", "----------\n", "train Loss: 0.4004 F1: 0.8696 Acc: 0.8481\n", "val Loss: 0.5398 F1: 0.8788 Acc: 0.8118\n", "\n", "Epoch 10/19\n", "----------\n", "train Loss: 0.3825 F1: 0.9011 Acc: 0.8861\n", "val Loss: 0.5567 F1: 0.8594 Acc: 0.7882\n", "\n", "Epoch 11/19\n", "----------\n", "train Loss: 0.3598 F1: 0.9213 Acc: 0.9114\n", "val Loss: 0.5710 F1: 0.8571 Acc: 0.7882\n", "\n", "Epoch 12/19\n", "----------\n", "train Loss: 0.3281 F1: 0.9412 Acc: 0.9367\n", "val Loss: 0.5708 F1: 0.8480 Acc: 0.7765\n", "\n", "Epoch 13/19\n", "----------\n", "train Loss: 0.3187 F1: 0.9302 Acc: 0.9241\n", "val Loss: 0.5715 F1: 0.8226 Acc: 0.7412\n", "\n", "Epoch 14/19\n", "----------\n", "train Loss: 0.3049 F1: 0.9647 Acc: 0.9620\n", "val Loss: 0.5688 F1: 0.8033 Acc: 0.7176\n", "\n", "Epoch 15/19\n", "----------\n", "train Loss: 0.2929 F1: 0.9535 Acc: 0.9494\n", "val Loss: 0.5558 F1: 0.8320 Acc: 0.7529\n", "\n", "Epoch 16/19\n", "----------\n", "train Loss: 0.2833 F1: 0.9425 Acc: 0.9367\n", "val Loss: 0.5366 F1: 0.8504 Acc: 0.7765\n", "\n", "Epoch 17/19\n", "----------\n", "train Loss: 0.2662 F1: 0.9647 Acc: 0.9620\n", "val Loss: 0.5275 F1: 0.8594 Acc: 0.7882\n", "\n", "Epoch 18/19\n", "----------\n", "train Loss: 0.2576 F1: 0.9535 Acc: 0.9494\n", "val Loss: 0.5512 F1: 0.8293 Acc: 0.7529\n", "\n", "Epoch 19/19\n", "----------\n", "train Loss: 0.2463 F1: 0.9535 Acc: 0.9494\n", "val Loss: 0.5820 F1: 0.7863 Acc: 0.7059\n", "\n", "Training complete in 2m 51s\n", "Best val F1 : 0.878788\n", "Best val Acc : 0.811765\n" ] } ], "source": [ "ex.run()" ] }, { "cell_type": "code", "execution_count": 88, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 89, "metadata": {}, "outputs": [], "source": [ "paths = glob.glob('results/*.csv') # * means all if need specific format then *.csv\n", "latest = max(paths, key=os.path.getctime)\n", "df = pd.read_csv(latest)" ] }, { "cell_type": "code", "execution_count": 90, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "-rw-r--r-- 1 aziai staff 4.6K Jun 2 23:21 results/results_20190602232142.csv\r\n", "-rw-r--r--@ 1 aziai staff 2.1K Jun 2 20:25 results/results_20190602195657.csv\r\n", "-rw-r--r--@ 1 aziai staff 1.4K Jun 2 19:48 results/results_20190602194822.csv\r\n", "-rw-r--r--@ 1 aziai staff 478B Jun 2 19:45 results/results_20190602194401.csv\r\n", "-rw-r--r--@ 1 aziai staff 846B Jun 2 19:28 results/results_20191402191416.csv\r\n" ] } ], "source": [ "!ls -lht results/*.csv" ] }, { "cell_type": "code", "execution_count": 91, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
batch_sizeconv_model_namedata_path_baseexperiment_uuidfeature_extractlrmomentumnum_epochsrunner_uuiduse_vggishval_accval_f1
1764resnetvtest_new220190602232142False0.0100.9510b5d41c44c98e2fa432656659c4fbcb77ef2a66d30dbe51...False0.9176470.946565
2264resnetvtest_new220190602232142False0.0010.9010c6cd69a63c6b7670802afa33af35b13d6c687340f71bf7...True0.8941180.936170
664NaNvtest_new220190602232142False0.0010.901057fac69720ed9689a5805e15ddbb89ee26d9b64fbd2cf2...True0.8823530.929577
364NaNvtest_new220190602232142False0.0100.9510bc7e48700485911fd1dcfa46fb408e93d744c2ec3afd26...True0.8588240.916667
2464resnetvtest_new220190602232142True0.0100.90109715fe9781571c61eb6bf38cbc5173df40e9e41ae3eaec...False0.8705880.916031
2664resnetvtest_new220190602232142True0.0100.9010454e165857fea306cac78a54c635a15545fab0e7a5f606...True0.8588240.907692
2364resnetvtest_new220190602232142False0.0010.9510af38e4c808ff3998977167f1e5138bdd0301f92a2dc206...True0.8352940.893939
2764resnetvtest_new220190602232142True0.0100.95104566c335e71f215a1110a36a7cfec1882c86f86dbb7e3e...True0.8235290.893617
264NaNvtest_new220190602232142False0.0100.90100fd6f6ebe947c54bdcb0340e244f0a27254f0a358be9a0...True0.8117650.891892
3164resnetvtest_new220190602232142True0.0010.9510b0ba9005b21c06336b8a58c2ed784dcf5fec6028dd7bca...True0.8117650.887324
1864resnetvtest_new220190602232142False0.0100.9010a60fa62389b6418273ca349479228d39e55b7b357ba2e7...True0.8235290.887218
1964resnetvtest_new220190602232142False0.0100.9510dcb42ec2f883718f58a1612e4afff99249aa628afa8442...True0.8000000.880000
2964resnetvtest_new220190602232142True0.0010.951081bf2d2fb5c8083afc54b43f8c561e6c5c36304a2d0024...False0.8000000.877698
1164NaNvtest_new220190602232142True0.0100.9510db6e5ff043c83f9643e5fa4aecb3c85a03e2be1c8de569...True0.7764710.874172
764NaNvtest_new220190602232142False0.0010.95108686806f634f6da4be6bf84cca80c305e8a7f751dd8ca8...True0.7764710.874172
1464NaNvtest_new220190602232142True0.0010.9010c2c04441a29f2bcf11fade126cd304849d3d5e9a1c98fa...True0.7764710.874172
1564NaNvtest_new220190602232142True0.0010.95104d227fadf14ff66ea2c7f253a96c2111a78234f95089d0...True0.7764710.874172
1064NaNvtest_new220190602232142True0.0100.901010ece59983d31536981268d1e8bbdf4460e65b24a7567b...True0.7764710.874172
2564resnetvtest_new220190602232142True0.0100.951021f4a1394a48e89171341403ff9eccc6e080a9acb66005...False0.8117650.873016
2164resnetvtest_new220190602232142False0.0010.9510091aaed6d121608dc449c2f33a7c74bbe835ccc5abadc9...False0.8000000.872180
3064resnetvtest_new220190602232142True0.0010.90105f938c364c0de0bc3ab556f1b48971520006d0ed581d53...True0.7764710.868966
1664resnetvtest_new220190602232142False0.0100.9010a9d9eb5afabbea4dff607846fe1a0782c760f44cfbd0f9...False0.7882350.867647
2064resnetvtest_new220190602232142False0.0010.90108c4da8f6f9157db4dbd6f70b67c1db469ea034730c12eb...False0.7882350.854839
2864resnetvtest_new220190602232142True0.0010.90106fcff331f0df5ded20113ae3e7f2d1568e3f3fba9f2a92...False0.7294120.818898
164NaNvtest_new220190602232142False0.0100.9510c7b84e8c420a3be3a333ba41c2618399c2baebb470fc6a...False-1.000000-1.000000
1364NaNvtest_new220190602232142True0.0010.951041fd07df53b8400f9386cf6d6fd5e400290fc19cc96bd1...False-1.000000-1.000000
1264NaNvtest_new220190602232142True0.0010.901033ac0b6c3357fb2cfc22194be83a872435a7b3495506b8...False-1.000000-1.000000
964NaNvtest_new220190602232142True0.0100.95101613a061bc0ee4cb3e89ff00d7b183241ee4fccf84420c...False-1.000000-1.000000
864NaNvtest_new220190602232142True0.0100.9010dddc0808ccee29d6d96d7922bdce1e66e319ad29cd7206...False-1.000000-1.000000
564NaNvtest_new220190602232142False0.0010.95109af39512bef330a9bbab90f91063597b4d3d798a0a7dec...False-1.000000-1.000000
464NaNvtest_new220190602232142False0.0010.901054aaa1ed52b83839a188927adb63f64a6e8cc8f9d5ce37...False-1.000000-1.000000
064NaNvtest_new220190602232142False0.0100.90103d4d0ae6efebf77f6b7f5a4163558c892daae15e174c5b...False-1.000000-1.000000
\n", "
" ], "text/plain": [ " batch_size conv_model_name data_path_base experiment_uuid feature_extract lr momentum num_epochs runner_uuid use_vggish val_acc val_f1\n", "17 64 resnet vtest_new2 20190602232142 False 0.010 0.95 10 b5d41c44c98e2fa432656659c4fbcb77ef2a66d30dbe51... False 0.917647 0.946565\n", "22 64 resnet vtest_new2 20190602232142 False 0.001 0.90 10 c6cd69a63c6b7670802afa33af35b13d6c687340f71bf7... True 0.894118 0.936170\n", "6 64 NaN vtest_new2 20190602232142 False 0.001 0.90 10 57fac69720ed9689a5805e15ddbb89ee26d9b64fbd2cf2... True 0.882353 0.929577\n", "3 64 NaN vtest_new2 20190602232142 False 0.010 0.95 10 bc7e48700485911fd1dcfa46fb408e93d744c2ec3afd26... True 0.858824 0.916667\n", "24 64 resnet vtest_new2 20190602232142 True 0.010 0.90 10 9715fe9781571c61eb6bf38cbc5173df40e9e41ae3eaec... False 0.870588 0.916031\n", "26 64 resnet vtest_new2 20190602232142 True 0.010 0.90 10 454e165857fea306cac78a54c635a15545fab0e7a5f606... True 0.858824 0.907692\n", "23 64 resnet vtest_new2 20190602232142 False 0.001 0.95 10 af38e4c808ff3998977167f1e5138bdd0301f92a2dc206... True 0.835294 0.893939\n", "27 64 resnet vtest_new2 20190602232142 True 0.010 0.95 10 4566c335e71f215a1110a36a7cfec1882c86f86dbb7e3e... True 0.823529 0.893617\n", "2 64 NaN vtest_new2 20190602232142 False 0.010 0.90 10 0fd6f6ebe947c54bdcb0340e244f0a27254f0a358be9a0... True 0.811765 0.891892\n", "31 64 resnet vtest_new2 20190602232142 True 0.001 0.95 10 b0ba9005b21c06336b8a58c2ed784dcf5fec6028dd7bca... True 0.811765 0.887324\n", "18 64 resnet vtest_new2 20190602232142 False 0.010 0.90 10 a60fa62389b6418273ca349479228d39e55b7b357ba2e7... True 0.823529 0.887218\n", "19 64 resnet vtest_new2 20190602232142 False 0.010 0.95 10 dcb42ec2f883718f58a1612e4afff99249aa628afa8442... True 0.800000 0.880000\n", "29 64 resnet vtest_new2 20190602232142 True 0.001 0.95 10 81bf2d2fb5c8083afc54b43f8c561e6c5c36304a2d0024... False 0.800000 0.877698\n", "11 64 NaN vtest_new2 20190602232142 True 0.010 0.95 10 db6e5ff043c83f9643e5fa4aecb3c85a03e2be1c8de569... True 0.776471 0.874172\n", "7 64 NaN vtest_new2 20190602232142 False 0.001 0.95 10 8686806f634f6da4be6bf84cca80c305e8a7f751dd8ca8... True 0.776471 0.874172\n", "14 64 NaN vtest_new2 20190602232142 True 0.001 0.90 10 c2c04441a29f2bcf11fade126cd304849d3d5e9a1c98fa... True 0.776471 0.874172\n", "15 64 NaN vtest_new2 20190602232142 True 0.001 0.95 10 4d227fadf14ff66ea2c7f253a96c2111a78234f95089d0... True 0.776471 0.874172\n", "10 64 NaN vtest_new2 20190602232142 True 0.010 0.90 10 10ece59983d31536981268d1e8bbdf4460e65b24a7567b... True 0.776471 0.874172\n", "25 64 resnet vtest_new2 20190602232142 True 0.010 0.95 10 21f4a1394a48e89171341403ff9eccc6e080a9acb66005... False 0.811765 0.873016\n", "21 64 resnet vtest_new2 20190602232142 False 0.001 0.95 10 091aaed6d121608dc449c2f33a7c74bbe835ccc5abadc9... False 0.800000 0.872180\n", "30 64 resnet vtest_new2 20190602232142 True 0.001 0.90 10 5f938c364c0de0bc3ab556f1b48971520006d0ed581d53... True 0.776471 0.868966\n", "16 64 resnet vtest_new2 20190602232142 False 0.010 0.90 10 a9d9eb5afabbea4dff607846fe1a0782c760f44cfbd0f9... False 0.788235 0.867647\n", "20 64 resnet vtest_new2 20190602232142 False 0.001 0.90 10 8c4da8f6f9157db4dbd6f70b67c1db469ea034730c12eb... False 0.788235 0.854839\n", "28 64 resnet vtest_new2 20190602232142 True 0.001 0.90 10 6fcff331f0df5ded20113ae3e7f2d1568e3f3fba9f2a92... False 0.729412 0.818898\n", "1 64 NaN vtest_new2 20190602232142 False 0.010 0.95 10 c7b84e8c420a3be3a333ba41c2618399c2baebb470fc6a... False -1.000000 -1.000000\n", "13 64 NaN vtest_new2 20190602232142 True 0.001 0.95 10 41fd07df53b8400f9386cf6d6fd5e400290fc19cc96bd1... False -1.000000 -1.000000\n", "12 64 NaN vtest_new2 20190602232142 True 0.001 0.90 10 33ac0b6c3357fb2cfc22194be83a872435a7b3495506b8... False -1.000000 -1.000000\n", "9 64 NaN vtest_new2 20190602232142 True 0.010 0.95 10 1613a061bc0ee4cb3e89ff00d7b183241ee4fccf84420c... False -1.000000 -1.000000\n", "8 64 NaN vtest_new2 20190602232142 True 0.010 0.90 10 dddc0808ccee29d6d96d7922bdce1e66e319ad29cd7206... False -1.000000 -1.000000\n", "5 64 NaN vtest_new2 20190602232142 False 0.001 0.95 10 9af39512bef330a9bbab90f91063597b4d3d798a0a7dec... False -1.000000 -1.000000\n", "4 64 NaN vtest_new2 20190602232142 False 0.001 0.90 10 54aaa1ed52b83839a188927adb63f64a6e8cc8f9d5ce37... False -1.000000 -1.000000\n", "0 64 NaN vtest_new2 20190602232142 False 0.010 0.90 10 3d4d0ae6efebf77f6b7f5a4163558c892daae15e174c5b... False -1.000000 -1.000000" ] }, "execution_count": 91, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.sort_values(by='val_f1', ascending=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.2" } }, "nbformat": 4, "nbformat_minor": 2 }