diff --git a/classification_for_image_tagging/image_type/inspect_train_results.ipynb b/classification_for_image_tagging/image_type/inspect_train_results.ipynb index 56031e25..94d112f8 100644 --- a/classification_for_image_tagging/image_type/inspect_train_results.ipynb +++ b/classification_for_image_tagging/image_type/inspect_train_results.ipynb @@ -1,17 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "authorship_tag": "ABX9TyPr6hq1vMiWyeULGajg2ARF", - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, "cells": [ { "cell_type": "markdown", @@ -31,7 +18,7 @@ "source": [ "# Determine confidence threshold for Image Type Classification Models\n", "---\n", - "*Last Updated 2 December 2024* \n", + "*Last Updated 28 January 2025* \n", "Choose which trained model and confidence threshold values to use for classifying EOL images as maps, phylogenies, illustrations, or herbarium sheets. Threshold values should be chosen that maximize coverage and minimize error.\n", "\n", "First, choose the 2 best models trained in [image_type_train.ipynb](https://colab.research.google.com/github/aubricot/computer_vision_with_eol_images/blob/master/classification_for_image_tagging/image_type/image_type_preprocessing.ipynb). Then, run this notebook.\n", @@ -58,6 +45,11 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sE0mcQMfN-5t" + }, + "outputs": [], "source": [ "#@title Choose where to save results (or keep defaults) & set up environment\n", "import os\n", @@ -93,15 +85,15 @@ "\n", "# Install requirements.txt\n", "!pip3 -q install -r requirements.txt" - ], - "metadata": { - "id": "sE0mcQMfN-5t" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kN4sl-xxOP6v" + }, + "outputs": [], "source": [ "#@title Choose saved model parameters (if using EOL model, defaults are already selected)\n", "from setup import setup_dirs, load_saved_model, get_model_info, unpack_EOL_model\n", @@ -145,18 +137,15 @@ " for file_id in file_ids:\n", " !gdown $file_id\n", "print(\"\\nImage metadata directory set to: \\n\", data_wd)" - ], - "metadata": { - "id": "kN4sl-xxOP6v" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "1AGFM4fSWhbT" }, + "outputs": [], "source": [ "#@title Import libraries\n", "\n", @@ -185,9 +174,7 @@ "# Set number of seconds to timeout if image url taking too long to open\n", "import socket\n", "socket.setdefaulttimeout(10)" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -204,9 +191,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "efIHQCtAAmYg" }, + "outputs": [], "source": [ "# Define functions\n", "from wrangle_data import read_datafile, set_start_stop, image_from_url, get_predict_info\n", @@ -232,14 +221,15 @@ " fpath = data_wd + '/' + demo_dict[imclass][0]\n", " elif 'null' in imclass:\n", " print(\"\\n\\033[91m Null image class doesn't have demo image bundle available. Moving onto the next class.\\033[0m\\n\")\n", + " stop, start = 0, 0\n", + " TEST_IMAGE_PATHS = None\n", " pass\n", " else:\n", " fpath = data_wd + '/' + demo_dict[imclass]\n", " df = pd.read_table(fpath)\n", " start = 1\n", - " stop = start + 500\n", + " stop = start + 2000\n", " TEST_IMAGE_PATHS = df.iloc[start:stop, 0].values.tolist()\n", - " print(\"\\nUsing up to 5 random images from EOL image type bundle: \\n\", fpath)\n", "\n", " except:\n", " pass\n", @@ -294,22 +284,23 @@ " results.to_csv(outfpath, index=False, header=(\"filename\", \"confidence\",\n", " \"true_id\", \"det_id\", \"colormode\"))\n", " print(\"\\nClassification predictions for image class {} being saved to : \\n{}\\n\".format(\n", - " true_imclass, outfpath))" - ], - "execution_count": null, - "outputs": [] + " true_imclass, outfpath))\n", + "\n" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "0ZXo6iVvBF0G" }, + "outputs": [], "source": [ "#@title Run inference for chosen Training Session Number (11, 13) and dataset size\n", "%cd $cwd\n", "\n", "# Choose training attempt number to inspect results for\n", - "TRAIN_SESS_NUM = \"11\" #@param [\"11\", \"13\"] {allow-input: true}\n", + "TRAIN_SESS_NUM = \"13\" #@param [\"11\", \"13\"] {allow-input: true}\n", "\n", "# Test pipeline with a smaller subset than 5k images?\n", "run = \"test with tiny subset\" #@param [\"test with tiny subset\", \"for 500 images\"]\n", @@ -363,24 +354,27 @@ " pass\n", "\n", " # Combine to df and export results\n", - " export_results(results, outfpath)\n", + " if 'null' not in true_imclass:\n", + " export_results(results, outfpath)\n", "\n", "print(\"\\n\\n~~~\\n\\033[92m Inference complete!\\033[0m\\n~~~\")" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "mnUCpn8sWVzi" }, + "outputs": [], "source": [ "#@title Combine model outputs for image type classes\n", "\n", "# Combine prediction files created in codeblock above\n", "base = classif_type + '_' + TRAIN_SESS_NUM + '_'\n", "imclasses = filters\n", + "if 'null' in imclasses:\n", + " imclasses.remove('null') # No evaluation image bundle available for null\n", "all_filenames = [base + imclass + '.csv' for imclass in imclasses]\n", "all_predictions = pd.concat([pd.read_csv(f, sep=',', header=0, na_filter = False) for f in all_filenames])\n", "outfpath = base + handle_base + '_all_predictions' + '.csv'\n", @@ -390,9 +384,7 @@ "print(\"No. Images: {}\\n\".format(len(all_predictions)))\n", "print(\"Model predictions for Training Attempt {}, {} with numeric classes:\\n{}\".format(\\\n", " TRAIN_SESS_NUM, handle_base, all_predictions[['filename', 'true_id', 'det_id']].head()))" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -409,13 +401,18 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5-h2sxHscHcS" + }, + "outputs": [], "source": [ "#@title Load model predictions (Optional: If you want to load in all_predictions.csv from a previous run, specify TRAIN_SESS_NUM)\n", "from setup import get_model_info\n", "\n", "%cd $cwd\n", "# Load combined prediction results from above\n", - "run_from_file = \"no\" # @param [\"no\",\"yes\"]\n", + "run_from_file = \"yes\" # @param [\"no\",\"yes\"]\n", "\n", "# Copy dataframe from above\n", "if run_from_file==\"no\":\n", @@ -433,19 +430,16 @@ " fname = base + handle_base + '_all_predictions' + '.csv'\n", " all_predictions = pd.read_csv(fname, sep=',', header=0, na_filter = False)\n", " print(\"Loading prediction data from file for train attempt {}, {} with labels {} for confusion matrix and histograms: \\n{}\".format(TRAIN_SESS_NUM, handle_base, dataset_labels, all_predictions[['filename', 'true_id', 'det_id']].head()))" - ], - "metadata": { - "id": "5-h2sxHscHcS" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "code", "id": "gEyoDC1rL1Ub" }, + "outputs": [], "source": [ "# Define functions\n", "\n", @@ -458,14 +452,14 @@ " # Set colormap\n", " if cmap is None:\n", " cmap = plt.get_cmap('Blues')\n", + " # Normalize data\n", + " if normalize:\n", + " cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", " # Build figure\n", " fig = plt.figure(figsize=(8, 8))\n", " plt.imshow(cm, interpolation='nearest', cmap=cmap)\n", " plt.title(title)\n", - " plt.colorbar()\n", - " # Normalize data\n", - " if normalize:\n", - " cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", + " plt.colorbar(shrink=0.6)\n", " # Add labels\n", " if target_names is not None:\n", " tick_marks = np.arange(len(target_names))\n", @@ -620,12 +614,13 @@ " f.writelines(tab)\n", " f.close()\n", " print(\"\\nTable saved to: \", outfpath, \"\\n\")" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "mpCLEatjdPX1" + }, "source": [ "### Make classification report and plot confusion matrix of precision/recall for each image class\n", "Use the confusion matrix and classification report table to evaluate model performance and determine class predictions to keep or filter out during post-processing. Precision and recall are related to accuracy. Precision measures what the model found (how many predicted positives are actually positive), while recall measures what the model missed (sensitivity / how many actual positives did the model identify).\n", @@ -637,13 +632,15 @@ "The classification report also includes an F1 score, which is the harmonic mean of precision and recall.\n", "\n", "$$F1 = 2* \\frac{precision*recall}{precision+recall}=\\frac{2*true\\,positives}{2*true\\,positives+false\\,positives+false\\,negatives}$$" - ], - "metadata": { - "id": "mpCLEatjdPX1" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sQJo3x_-dTsr" + }, + "outputs": [], "source": [ "#@title Make a classification report and confusion matrix\n", "from sklearn.metrics import confusion_matrix, classification_report\n", @@ -666,28 +663,25 @@ "\n", "# Export confusion matrix\n", "save_figure(fig, figtype = 'conf_mx', filetype = '.png')" - ], - "metadata": { - "id": "sQJo3x_-dTsr" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "H9S6z_ujddQc" + }, "source": [ "### Plot histograms of accuracy for each image class\n", "Use these plots to determine confidence thresholds or class predictions to keep or filter out during post-processing." - ], - "metadata": { - "id": "H9S6z_ujddQc" - } + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "10GhFabiCj3c" }, + "outputs": [], "source": [ "#@title Plot histograms (Optional: inspect for specific taxon and/or add a confidence threshold line)\n", "\n", @@ -696,7 +690,7 @@ "taxon = \"\" #@param {type:\"string\"}\n", "\n", "# Optional: Draw threshold value to help choose optimal balance b/w maximizing useful data and minimizing error\n", - "thresh = 1.5 #@param {type:\"number\"}\n", + "thresh = 3.5 #@param {type:\"number\"}\n", "\n", "# Valide predictions by image class (optionally, by taxon)\n", "tru, fal, taxon = validate_predict(all_predictions, inspect_by_taxon, taxon)\n", @@ -706,9 +700,7 @@ "\n", "# Export histograms\n", "save_figure(fig, figtype = 'hist', filetype = '.png', taxon = taxon)" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -721,10 +713,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "U748-euK2eKN", - "cellView": "code" + "id": "U748-euK2eKN" }, + "outputs": [], "source": [ "# Load combined prediction results\n", "df = all_predictions.copy()\n", @@ -735,12 +728,12 @@ "fal = df.loc[~df.det, :] # False ID\n", "\n", "# Confidence values to test\n", - "conf_vals = [1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2] #@param\n", + "conf_vals = [2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4] # @param [\"[1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.2]\",\"[2.2, 2.4, 2.6, 2.8, 3.0, 3.2, 3.4]\"] {\"type\":\"raw\",\"allow-input\":true}\n", "for conf_val in conf_vals:\n", " df_c = df.loc[df[\"confidence\"] > conf_val, :]\n", " true_c = tru.loc[tru[\"confidence\"] > conf_val, :]\n", " fal_c = fal.loc[fal[\"confidence\"] > conf_val, :]\n", - " all_vals = true_c.append(fal_c)\n", + " all_vals = pd.concat([true_c, fal_c], ignore_index = True)\n", " print(\"\\nConfidence Value: {}\\n\".format(conf_val))\n", " print(\"Accuracy for confidence > {}: {}\".format(conf_val, get_accuracy(len(true_c), len(all_vals))))\n", " print(\"Predictions Retained (%): {}\".format(len(df_c)/len(df)))\n", @@ -753,9 +746,7 @@ " all_det_c = len(all_vals.loc[all_vals[\"true_id\"] == imclass, :])\n", " accuracy = get_accuracy(true_det_c, all_det_c)\n", " print(\"{}: {}\".format(imclass, accuracy))" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -770,14 +761,16 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "id": "-GyhC_-DtYXX" }, + "outputs": [], "source": [ "# Break up predictions by image class and colorspace\n", "\n", "# Define variables\n", - "c0,c1,c2,c3,c4 = [imclasses[i] for i in range(0, len(imclasses))]\n", + "c0,c1,c2,c4 = [imclasses[i] for i in range(0, len(imclasses))] # no c3 bc null has no eval imgs\n", "# Check how many true/false predictions are at each confidence value\n", "# Class 0 - 'herb'\n", "c0t = tru.loc[tru['true_id'] == c0, :] # True dets\n", @@ -788,9 +781,6 @@ "# Class 2 - 'map'\n", "c2t = tru.loc[tru['true_id'] == c2, :]\n", "c2f = fal.loc[fal['true_id'] == c2, :]\n", - "# Class 3 - 'null'\n", - "c3t = tru.loc[tru['true_id'] == c3, :]\n", - "c3f = fal.loc[fal['true_id'] == c3, :]\n", "# Class 4 - 'phylo'\n", "c4t = tru.loc[tru['true_id'] == c4, :]\n", "c4f = fal.loc[fal['true_id'] == c4, :]\n", @@ -816,22 +806,26 @@ "t_by_col = c2t.loc[c2t[\"colormode\"]==\"('L',)\", :]\n", "print(\"False for greyscale: {}\\nTrue for greyscale: {}\".format(len(f_by_col), len(t_by_col)))\n", "\n", - "# Class 3 = Null\n", - "print(\"\\n{}\".format(c3))\n", - "print(\"False detections: {}\\nTrue detections: {}\".format(len(c3f), len(c3t)))\n", - "f_by_col = c3f.loc[c3f[\"colormode\"]==\"('L',)\", :]\n", - "t_by_col = c3t.loc[c3t[\"colormode\"]==\"('L',)\", :]\n", - "print(\"False for greyscale: {}\\nTrue for greyscale: {}\".format(len(f_by_col), len(t_by_col)))\n", - "\n", "# Class 4 = Phylogeny\n", "print(\"\\n{}\".format(c4))\n", "print(\"False detections: {}\\nTrue detections: {}\".format(len(c4f), len(c4t)))\n", "f_by_col = c4f.loc[c4f[\"colormode\"]==\"('L',)\", :]\n", "t_by_col = c4t.loc[c4t[\"colormode\"]==\"('L',)\", :]\n", "print(\"False for greyscale: {}\\nTrue for greyscale: {}\".format(len(f_by_col), len(t_by_col)))" - ], - "execution_count": null, - "outputs": [] + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "authorship_tag": "ABX9TyOObKhmdkURczycwThk6Oyx", + "include_colab_link": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] + }, + "nbformat": 4, + "nbformat_minor": 0 } \ No newline at end of file