# -*- coding: utf-8 -*-
"""
./dev.py -t custom:affine_invariance=False,adapteq=True,fg_on=False --db Elephants_drop1_ears --allgt --index=0:10 --guiview # NOQA
"""
import logging
from os.path import join
import numpy as np
import utool as ut
import vtool as vt
from wbia.expt import draw_helpers
from functools import reduce
print, rrr, profile = ut.inject2(__name__)
logger = logging.getLogger('wbia')
[docs]def scorediff(ibs, testres, f=None, verbose=None):
r"""
Args:
ibs (wbia.IBEISController): image analysis api
testres (wbia.TestResult): test result object
f (None): (default = None)
verbose (bool): verbosity flag(default = None)
CommandLine:
python -m wbia.expt.experiment_drawing scorediff --db PZ_Master1 -a timectrl -t best --show
python -m wbia.expt.experiment_drawing scorediff --db PZ_MTEST -a default -t best --show
python -m wbia.expt.experiment_drawing scorediff --db humpbacks_fb \
-a default:has_any=hasnotch,mingt=2 \
-t default:proot=BC_DTW,decision=max,crop_dim_size=500,crop_enabled=True,use_te_scorer=False,manual_extract=True,ignore_notch=True,te_net=annot_simple --show
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import * # NOQA
>>> from wbia.init import main_helpers
>>> defaultdb = 'PZ_MTEST'
>>> ibs, testres = main_helpers.testdata_expts(defaultdb, a=['timectrl'], t=['best'])
>>> f = ut.get_argval(('--filt', '-f'), type_=list, default=[''])
>>> scorediff(ibs, testres, f=f, verbose=ut.VERBOSE)
>>> ut.show_if_requested()
"""
import wbia.plottool as pt
for cfgx in range(testres.nConfig):
cm_list = testres.cfgx2_qreq_[cfgx].execute()
aid_list = [cm.qaid for cm in cm_list]
score_diffs = []
top_scores = []
for cm in cm_list:
annot_scores = sorted(cm.annot_score_list, key=lambda x: -x)
if False:
# We want to measure just the top 2 regardless of what they are
diff = annot_scores[0] - annot_scores[1]
else:
try:
# We want to measure just the top 2 wrt truth
top_true_aid = cm.get_top_gt_aids(ibs)[0]
top_false_aid = cm.get_top_gf_aids(ibs)[0]
gt_score = cm.get_annot_scores([top_true_aid])[0]
gf_score = cm.get_annot_scores([top_false_aid])[0]
diff = gt_score - gf_score
except IndexError:
continue
# diff = 0 # np.nan
top_scores.append(annot_scores[0])
score_diffs.append(diff)
score_diffs = np.array(score_diffs)
top_scores = np.array(top_scores)
succ = score_diffs > 0
fail = score_diffs <= 0
# succ = testres.get_truth2_prop()[0]['gt']['rank'][:, 0] == 0
# fail = testres.get_truth2_prop()[0]['gt']['rank'][:, 0] != 0
# fail = np.where(testres.get_truth2_prop()[0]['gt']['score'][:, 0] != 0)
width = 0.25
succ_max = max(2, score_diffs[succ].max())
fail_max = max(2, -score_diffs[fail].min())
num_succ_bins = int(succ_max / width)
num_fail_bins = int(fail_max / width)
succ_bins = np.linspace(0, succ_max, num_succ_bins + 1)
fail_bins = np.linspace(-fail_max, 0, num_fail_bins + 1)
logger.info('succ_bins = %r' % (succ_bins,))
logger.info('fail_bins = %r' % (fail_bins,))
succ_hist, succ_edges = np.histogram(score_diffs[succ], bins=succ_bins)
fail_hist, fail_edges = np.histogram(score_diffs[fail], bins=fail_bins)
# bin_max = (score_diffs.max() - score_diffs.min()) / 50
ydata_list = [succ_hist, fail_hist]
bins_list = [succ_bins, fail_bins]
# xdata_list = [(bins[:-1] + bins[1:]) / 2 for bins in bins_list]
xdata_list = [bins[:-1] for bins in bins_list]
width_list = [np.diff(bins)[0] for bins in bins_list]
# nbins = 8
# bin_width = (score_diffs.mean() + score_diffs.std()) / nbins
# bin_width = int(np.ceil((score_diffs.mean() + score_diffs.std() / 4) / nbins))
# bin_width = 1
# bins = np.arange(nbins) * bin_width
# succ_hist, succ_edges = np.histogram(score_diffs[succ], bins=bins)
# fail_hist, fail_edges = np.histogram(score_diffs[fail], bins=bins)
# hist, edges = np.histogram(score_diffs)
# logger.info('hist = %r' % (hist,))
# ymax = max(max(succ_hist), max(fail_hist)) * 1.1
species = ibs.get_dominant_species(testres.qaids)
species_nice = ibs.get_species_nice(
ibs.get_species_rowids_from_text(ibs.get_dominant_species(testres.qaids))
)
species_nice = {
'zebra_grevys': "Grevy's Zebras",
'zebra_plains': 'Plains Zebras',
'giraffe_masai': 'Masai Giraffes',
}.get(species, species_nice)
pt.multi_plot(
xdata_list,
ydata_list,
label_list=['correct match at rank 1', 'correct match above rank 1'],
width_list=width_list,
fnum=pt.next_fnum(),
color_list=[pt.TRUE_BLUE, pt.FALSE_RED],
kind='bar',
alpha=0.7,
stacked=True,
edgecolor='none',
xlabel='difference between the best correct score and the best incorrect score',
ylabel='frequency (number of query annotations)',
title='Score differences for %s' % (species_nice,),
xmin=-10,
xmax=30,
use_legend=True,
)
# fnum = pt.next_fnum()
# pt.plot_score_histograms(
# [score_diffs[succ], score_diffs[fail]],
# fnum=fnum,
# score_colors=[pt.TRUE_BLUE, pt.FALSE_RED],
# score_lbls=['correct', 'incorrect'],
# bin_width=.5,
# # histnorm='percent',
# )
# pt.draw_histogram(succ_edges, succ_hist, xlabel='1st - 2nd score',
# autolabel=False, color='blue', title='Success Cases',
# ymax=ymax, pnum=(1, 2, 1), fnum=fnum)
# pt.draw_histogram(fail_edges, fail_hist, xlabel='1st - 2nd score',
# autolabel=False, title='Failure Cases', ymax=ymax,
# pnum=(1, 2, 2), fnum=fnum)
from wbia.plottool.abstract_interaction import AbstractInteraction
class SortedScoreSupportInteraction(AbstractInteraction):
@staticmethod
def static_plot(fnum, pnum):
s = 20
pt.plt.scatter(
top_scores[succ], score_diffs[succ], marker='x', color='b', s=s
)
pt.plt.scatter(
top_scores[fail], score_diffs[fail], marker='x', color='r', s=s
)
flags = ibs.filterflags_annot_tags(
aid_list, has_any=['photobomb', 'scenerymatch']
)
pt.plt.scatter(
top_scores[succ * flags],
score_diffs[succ * flags],
marker='o',
color='y',
s=s * 2,
facecolors='none',
)
pt.plt.scatter(
top_scores[fail * flags],
score_diffs[fail * flags],
marker='o',
color='y',
s=s * 2,
facecolors='none',
)
pt.set_xlabel('top score')
pt.set_ylabel('score diff')
def on_click_inside(self, event, ex):
import vtool as vt
# ax = event.inaxes
# for l in ax.get_lines():
# logger.info(l.get_label())
pts = np.array([top_scores, score_diffs]).T
idx, dist = vt.closest_point(np.array([event.xdata, event.ydata]), pts)
logger.info('idx = %r' % (idx,))
logger.info('dist = %r' % (dist,))
aid = aid_list[idx]
logger.info('aid = %r' % (aid,))
if event.button == 3: # right-click
cm = cm_list[idx]
from wbia.gui import inspect_gui
qaid = aid
qreq_ = testres.cfgx2_qreq_[cfgx]
ibs = testres.ibs
if len(cm.daid_list) > 0:
daid = cm.get_top_aids()[0]
options = [
('Interact Analysis', lambda: cm.ishow_analysis(qreq_))
]
options += inspect_gui.get_aidpair_context_menu_options(
ibs, qaid, daid, cm, qreq_=qreq_
)
else:
logger.info(' no matches for this one')
# update_callback=self.show_page,
# backend_callback=None, aid_list=aid_list)
# from wbia.viz.interact import interact_chip
# options = interact_chip.build_annot_context_options(
# testres.ibs, aid, refresh_func=self.show_page, config2_=.extern_query_config2)
self.show_popup_menu(options, event)
# x = SortedScoreSupportInteraction()
# x.start()
# pt.interactions.zoom_factory()
# @devcmd('scores', 'score', 'namescore_roc')
[docs]def draw_annot_scoresep(ibs, testres, f=None, verbose=None):
"""
Draws the separation between true positive and true negative name scores.
TODO:
plot the difference between the top true score and the next best false score?
CommandLine:
ib
python -m wbia draw_annot_scoresep --show
python -m wbia draw_annot_scoresep --db PZ_MTEST --allgt -w --show --serial
python -m wbia draw_annot_scoresep -t scores --db PZ_MTEST --allgt --show
python -m wbia draw_annot_scoresep -t scores --db PZ_Master0 --allgt --show
python -m wbia draw_annot_scoresep --db PZ_Master1 -a timectrl -t best --show
python -m wbia draw_annot_scoresep --db PZ_Master1 -a timectrl -t best --show -f :without_tag=photobomb
Paper:
python -m wbia draw_annot_scoresep --dbdir lev/media/hdd/golden/GGR-IBEIS -a timectrl --save gz_scoresep.png
python -m wbia draw_annot_scoresep --dbdir lev/media/hdd/golden/GZGC -a timectrl:species=zebra_plains --save pz_scoresep.png
python -m wbia draw_annot_scoresep --dbdir lev/media/hdd/golden/GZGC -a timectrl1h:species=giraffe_masai --save girm_scoresep.png
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import * # NOQA
>>> from wbia.init import main_helpers
>>> defaultdb = 'PZ_MTEST'
>>> ibs, testres = main_helpers.testdata_expts(defaultdb, a=['timectrl'], t=['best'])
>>> f = ut.get_argval(('--filt', '-f'), type_=list, default=[''])
>>> draw_annot_scoresep(ibs, testres, f=f, verbose=ut.VERBOSE)
>>> ut.show_if_requested()
Ignore:
import IPython
IPython.get_ipython().magic('pylab qt4')
"""
import wbia.plottool as pt
import vtool as vt
from wbia.expt import cfghelpers
if ut.VERBOSE:
logger.info('[dev] draw_annot_scoresep')
if f is None:
f = ['']
filt_cfg = ut.flatten(cfghelpers.parse_cfgstr_list2(f, strict=False))[0]
logger.info('filt_cfg = %r' % (filt_cfg,))
# assert len(testres.cfgx2_qreq_) == 1, 'can only specify one config here'
test_qaids = testres.get_test_qaids()
# TODO: option to group configs with same pcfg and different acfg
def load_annot_scores(testres, cfgx, filt_cfg):
qaids = testres.cfgx2_qaids[cfgx]
gt_rawscore = testres.get_infoprop_mat('qx2_gt_raw_score', qaids).T[cfgx]
gf_rawscore = testres.get_infoprop_mat('qx2_gf_raw_score', qaids).T[cfgx]
gt_daid = testres.get_infoprop_mat('qx2_gt_aid', qaids).T[cfgx]
gf_daid = testres.get_infoprop_mat('qx2_gf_aid', qaids).T[cfgx]
# FIXME: may need to specify which cfg is used in the future
isvalid = testres.case_sample2(filt_cfg, qaids=qaids, return_mask=True).T[cfgx]
isvalid[np.isnan(gf_rawscore)] = False
isvalid[np.isnan(gt_rawscore)] = False
tp_nscores = gt_rawscore[isvalid]
tn_nscores = gf_rawscore[isvalid]
# ---
tn_qaids = tp_qaids = test_qaids[isvalid]
tn_daids = gf_daid[isvalid]
tp_daids = gt_daid[isvalid]
part_attrs = {
1: {'qaid': tp_qaids, 'daid': tn_daids},
0: {'qaid': tn_qaids, 'daid': tp_daids},
}
return tp_nscores, tn_nscores, part_attrs
join_acfgs = True
if join_acfgs:
groupxs = testres.get_cfgx_groupxs()
else:
groupxs = list(zip(range(len(testres.cfgx2_qreq_))))
grouped_qreqs = ut.apply_grouping(testres.cfgx2_qreq_, groupxs)
cfgx2_shortlbl = testres.get_short_cfglbls(join_acfgs=join_acfgs)
grouped_scores = []
for cfgxs in groupxs:
# testres.print_pcfg_info()
score_group = []
for cfgx in cfgxs:
logger.info('Loading cached chipmatches')
tp_scores, tn_scores, part_attrs = load_annot_scores(testres, cfgx, filt_cfg)
score_group.append((tp_scores, tn_scores, part_attrs))
grouped_scores.append(score_group)
def attr_callback(qaid):
logger.info('callback qaid = %r' % (qaid,))
testres.interact_individual_result(qaid)
reconstruct_str = (
'python -m wbia.dev -e cases '
+ testres.reconstruct_test_flags()
+ ' --qaid '
+ str(qaid)
+ ' --show'
)
logger.info('Independent reconstruct')
logger.info(reconstruct_str)
fpr = ut.get_argval('--fpr', type_=float, default=None)
tpr = ut.get_argval('--tpr', type_=float, default=None if fpr is not None else 0.85)
for score_group, lbl in zip(grouped_scores, cfgx2_shortlbl):
tp_nscores = np.hstack(ut.take_column(score_group, 0))
tn_nscores = np.hstack(ut.take_column(score_group, 1))
combine_attrs = ut.partial(
ut.dict_union_combine,
combine_op=ut.partial(ut.dict_union_combine, combine_op=np.append),
)
part_attrs = reduce(combine_attrs, ut.take_column(score_group, 2))
# encoder = vt.ScoreNormalizer(adjust=8, tpr=.85)
encoder = vt.ScoreNormalizer(
# adjust=8,
adjust=1.5,
# fpr=fpr, tpr=tpr,
monotonize=True,
verbose=verbose,
)
tp_scores = tp_nscores
tn_scores = tn_nscores
name_scores, labels, attrs = encoder._to_xy(tp_nscores, tn_nscores, part_attrs)
encoder.fit(name_scores, labels, attrs, verbose=verbose)
# encoder.visualize(figtitle='Learned Name Score Normalizer\n' + qreq_.get_cfgstr())
# encoder.visualize(figtitle='Learned Name Score Normalizer\n' + qreq_.get_cfgstr(), fnum=cfgx)
# pt.set_figsize(w=30, h=10, dpi=256)
plotname = ''
figtitle = testres.make_figtitle(plotname, filt_cfg=filt_cfg)
encoder.visualize(
figtitle=figtitle,
#
with_scores=False,
with_prebayes=False,
with_postbayes=False,
#
histnorm='percent',
histoverlay=False,
with_hist=True,
with_roc=False,
attr_callback=attr_callback,
# bin_width=.125,
# bin_width=.05,
# logscale=dict(linthreshx=1, linthreshy=1, basex=2, basey=0),
# score_range=(0, 14),
# score_range=(0, 10),
score_range=(0, 6),
# bin_width=.5,
bin_width=0.125,
score_lbls=('incorrect', 'correct'),
verbose=verbose,
)
icon = ibs.get_database_icon()
if False and icon is not None:
pt.overlay_icon(
icon,
coords=(1, 0),
bbox_alignment=(1, 0),
as_artist=1,
max_asize=(1000, 2000),
)
if ut.get_argflag('--contextadjust'):
pt.adjust_subplots(left=0.1, bottom=0.25, wspace=0.2, hspace=0.2)
pt.adjust_subplots(use_argv=True)
# pt.set_figsize(w=30, h=10, dpi=256)
pt.set_figtitle(ibs.get_dbname() + ' ' + lbl)
locals_ = locals()
return locals_
[docs]def draw_casetag_hist(
ibs, testres, f=None, with_wordcloud=not ut.get_argflag('--no-wordcloud')
):
r"""
Args:
ibs (wbia.IBEISController): wbia controller object
testres (TestResult): test result object
CommandLine:
wbia --tf -draw_casetag_hist --show
# Experiments I tagged
wbia --tf -draw_casetag_hist -a timectrl -t invarbest --db PZ_Master1 --show
wbia -e taghist -a timectrl -t best --db PZ_Master1 --show
wbia -e taghist -a timequalctrl -t invarbest --db PZ_Master1 --show
wbia -e taghist -a timequalctrl:minqual=good -t invarbest --db PZ_Master1 --show
wbia -e taghist -a timequalctrl:minqual=good -t invarbest --db PZ_Master1 --show --filt :fail=True
# Do more tagging
wbia -e cases -a timequalctrl:minqual=good -t invarbest --db PZ_Master1 \
--filt :orderby=gfscore,reverse=1,min_gtrank=1,max_gf_tags=0 --show
wbia -e print -a timequalctrl:minqual=good -t invarbest --db PZ_Master1 --show
wbia -e cases -a timequalctrl -t invarbest --db PZ_Master1 \
--filt :orderby=gfscore,reverse=1,max_gf_tags=0,:fail=True,min_gf_timedelta=12h --show
wbia -e cases -a timequalctrl -t invarbest --db PZ_Master1 \
--filt :orderby=gfscore,reverse=1,max_gf_tags=0,:fail=True,min_gf_timedelta=12h --show
python -m wbia -e taghist --db PZ_Master1 -a timectrl -t best \
--filt :fail=True --no-wordcloud --hargv=tags --prefix "Failure Case " --label PZTags --figsize=10,3 --left=.2
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import * # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_Master1', a=['timequalcontrolled'])
>>> f = ut.get_argval(('--filt', '-f'), type_=list, default=[''])
>>> draw_casetag_hist(ibs, testres, f=f)
>>> ut.show_if_requested()
"""
import wbia.plottool as pt
from wbia import tag_funcs
from wbia.expt import cfghelpers
# All unfiltered tags
all_tags = testres.get_all_tags()
if True:
# Remove gf tags below a thresh and gt tags above a thresh
gt_tags = testres.get_gt_tags()
gf_tags = testres.get_gf_tags()
truth2_prop, prop2_mat = testres.get_truth2_prop()
score_thresh = testres.find_score_thresh_cutoff()
logger.info('score_thresh = %r' % (score_thresh,))
# TODO: I want the point that the prob true is greater than prob false
gt_is_problem = truth2_prop['gt']['score'] < score_thresh
gf_is_problem = truth2_prop['gf']['score'] >= score_thresh
other_is_problem = ~np.logical_or(gt_is_problem, gf_is_problem)
def zipmask(_tags, _flags):
return [
[item if flag else [] for item, flag in zip(list_, flags)]
for list_, flags in zip(_tags, _flags)
]
def combinetags(tags1, tags2):
import utool as ut
return [ut.list_zipflatten(t1, t2) for t1, t2 in zip(tags1, tags2)]
gt_problem_tags = zipmask(gt_tags, gt_is_problem)
gf_problem_tags = zipmask(gf_tags, gf_is_problem)
other_problem_tags = zipmask(all_tags, other_is_problem)
all_tags = reduce(
combinetags, [gt_problem_tags, gf_problem_tags, other_problem_tags]
)
if not ut.get_argflag('--fulltag'):
all_tags = [tag_funcs.consolodate_annotmatch_tags(tags) for tags in all_tags]
# Get tags that match the filter
if f is None:
f = ['']
filt_cfg = ut.flatten(cfghelpers.parse_cfgstr_list2(f, strict=False))[0]
case_pos_list = testres.case_sample2(filt_cfg)
case_qx_list = ut.unique_ordered(case_pos_list.T[0])
selected_tags = ut.take(all_tags, case_qx_list)
flat_tags_list = list(map(ut.flatten, selected_tags))
WITH_NOTAGS = False
if WITH_NOTAGS:
flat_tags_list_ = [
tags if len(tags) > 0 else ['NoTag'] for tags in flat_tags_list
]
else:
flat_tags_list_ = flat_tags_list
WITH_TOTAL = False
if WITH_TOTAL:
total = [['Total']] * len(case_qx_list)
flat_tags_list_ += total
WITH_WEIGHTS = True
if WITH_WEIGHTS:
flat_weights_list = [
[] if len(tags) == 0 else [1.0 / len(tags)] * len(tags)
for tags in flat_tags_list_
]
flat_tags = list(map(str, ut.flatten(flat_tags_list_)))
if WITH_WEIGHTS:
weight_list = ut.flatten(flat_weights_list)
else:
weight_list = None
fnum = None
pnum_ = pt.make_pnum_nextgen(nRows=1, nCols=with_wordcloud + 1)
fnum = pt.ensure_fnum(fnum)
pt.word_histogram2(
flat_tags, weight_list=weight_list, fnum=fnum, pnum=pnum_(), xlabel='Case'
)
icon = ibs.get_database_icon()
if icon is not None:
pt.overlay_icon(icon, coords=(1, 1), bbox_alignment=(1, 1))
if with_wordcloud:
pt.wordcloud(' '.join(flat_tags), fnum=fnum, pnum=pnum_())
# figtitle = testres.make_figtitle('Tag Histogram', filt_cfg=filt_cfg)
figtitle = testres.make_figtitle('Case Histogram', filt_cfg=filt_cfg)
figtitle += ' #cases=%r' % (len(case_qx_list))
pt.set_figtitle(figtitle)
if ut.get_argflag('--contextadjust'):
# pt.adjust_subplots(left=.1, bottom=.25, wspace=.2, hspace=.2)
# pt.adjust_subplots(wspace=.01)
pt.adjust_subplots(use_argv=True, wspace=0.01, bottom=0.3)
[docs]def draw_rank_surface(ibs, testres, verbose=None, fnum=None):
r"""
Draws n dimensional data + a score / rank
The rank is always on the y axis.
The first dimension is on the x axis.
The second dimension is split over multiple plots.
The third dimension becomes multiple lines.
May need to clean this scheme up a bit.
Args:
ibs (wbia.IBEISController): wbia controller object
testres (TestResult): test result object
CommandLine:
wbia --tf draw_rank_surface --db PZ_Master1 -a varysize_td -t CircQRH_K --show
wbia --tf draw_rank_surface --show -t best -a varysize --db PZ_Master1 --show
wbia --tf draw_rank_surface --show -t CircQRH_K -a varysize_td --db PZ_Master1 --show
wbia --tf draw_rank_surface --show -t CircQRH_K -a varysize_td --db PZ_Master1 --show
wbia --tf draw_rank_surface --show -t candidacy_k -a varysize --db PZ_Master1 --show --param-keys=K,dcfg_sample_per_name,dcfg_sample_size
wbia --tf draw_rank_surface --show -t best \
-a varynannots_td varynannots_td:qmin_pername=3,dpername=2 \
--db PZ_Master1 --show --param-keys=dcfg_sample_per_name,dcfg_sample_size
wbia --tf draw_rank_surface --show -t best -a varynannots_td --db PZ_Master1 --show --param-keys=dcfg_sample_size
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import * # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> result = draw_rank_surface(ibs, testres)
>>> ut.show_if_requested()
>>> print(result)
"""
import wbia.plottool as pt
from wbia.expt import annotation_configs
if verbose is None:
verbose = ut.VERBOSE
# percent_le1_list = 100 * rank_le1_list / len(testres.qaids)
cfgx2_cumhist_percent, edges = testres.get_rank_percentage_cumhist(bins='dense')
percent_le1_list = cfgx2_cumhist_percent.T[0]
# testres.cfgx2_lbl
# testres.get_param_basis('dcfg_sample_per_name')
# testres.get_param_basis('dcfg_sample_size')
# K_basis = testres.get_param_basis('K')
# K_cfgx_lists = [testres.get_cfgx_with_param('K', K) for K in K_basis]
# param_key_list = testres.get_all_varied_params()
# Extract the requested keys
default_param_key_list = ['K', 'dcfg_sample_per_name', 'dcfg_sample_size']
param_key_list = ut.get_argval(
'--param-keys', type_=list, default=default_param_key_list
)
# param_key_list = ['K', 'dcfg_sample_per_name', 'len(daids)']
basis_dict = {}
cfgx_lists_dict = {}
for key in param_key_list:
_basis = testres.get_param_basis(key)
_cfgx_list = [testres.get_cfgx_with_param(key, val) for val in _basis]
# Grid of config indexes using param key as reference
cfgx_lists_dict[key] = _cfgx_list
basis_dict[key] = _basis
if verbose:
logger.info('basis_dict = ' + ut.repr2(basis_dict, nl=1, hack_liststr=True))
logger.info(
'e.g. cfgx_lists_dict[1] contains indicies of configs where K = basis_dict["K"][1]'
)
logger.info(
'cfx_lists_dict = ' + ut.repr2(cfgx_lists_dict, nl=2, hack_liststr=True)
)
# const_key = 'K'
if len(param_key_list) == 1:
const_key = None
const_basis = [None]
basis_dict[None] = [0]
# const_basis_cfgx_lists
# Create a single empty dimension for a single pnum
# correct, but not conceptually right
cfgx_lists_dict[None] = ut.list_transpose(cfgx_lists_dict[param_key_list[0]])
elif len(param_key_list) > 1:
# Hold a key constant if more than 1 subplot
const_key = param_key_list[1]
# const_key = 'dcfg_sample_per_name'
const_basis = basis_dict[const_key]
# pnum_ = pt.make_pnum_nextgen(*pt.get_square_row_cols(len(basis_dict[const_key]), max_cols=1))
# ymax = percent_le1_list.max()
# ymin = percent_le1_list.min()
else:
assert False
const_basis_cfgx_lists = cfgx_lists_dict[const_key]
if len(param_key_list) > 2:
# Use consistent markers and colors when varying a lot of params
# num_other_params = len(basis_dict[param_key_list[-1]])
# num_other_params = len(basis_dict[const_key])
num_other_params = len(basis_dict[ut.setdiff(param_key_list, const_key)[-1]])
color_list = pt.distinct_colors(num_other_params)
marker_list = pt.distinct_markers(num_other_params)
else:
color_list = pt.distinct_colors(1)
marker_list = pt.distinct_markers(1)
fnum = pt.ensure_fnum(fnum)
nd_labels_full = [key for key in param_key_list if key != const_key]
# setup args for all plots
pnum_ = pt.make_pnum_nextgen(*pt.get_square_row_cols(len(basis_dict[const_key])))
for const_idx, const_val in enumerate(const_basis):
pnum = pnum_()
if verbose:
logger.info('---- NEXT PNUM=%r --- ' % (pnum,))
logger.info('const_key = %r' % (const_key,))
logger.info('const_val = %r' % (const_val,))
logger.info('const_idx = %r' % (const_idx,))
const_basis_cfgx_list = const_basis_cfgx_lists[const_idx]
rank_list = ut.take(percent_le1_list, const_basis_cfgx_list)
# Figure out what the values are for other dimensions
agree_param_vals = dict(
[
(
key,
[
testres.get_param_val_from_cfgx(cfgx, key)
for cfgx in const_basis_cfgx_list
],
)
for key in nd_labels_full
]
)
# Make a list of points that need plotting
known_nd_data = np.array(list(agree_param_vals.values())).T
known_target_points = np.array(rank_list)
nd_labels_ = nd_labels_full[:]
if len(nd_labels_) == 1:
# hack for nonvaried params
empty_dim = np.zeros((len(known_nd_data), 1))
known_nd_data = np.hstack([known_nd_data, empty_dim])
nd_labels_ += [None]
# short ndlabels
nd_labels = [
annotation_configs.shorten_to_alias_labels(key) for key in nd_labels_
]
target_label = annotation_configs.shorten_to_alias_labels(key)
target_label = 'accuracy (%)'
# hack
ymin = 30 if known_target_points.min() > 30 and False else 0
num_yticks = 8 if ymin == 30 else 11
if const_key is None:
title = 'accuracy'
else:
title = (
'accuracy when '
+ annotation_configs.shorten_to_alias_labels(const_key)
+ '=%r' % (const_val,)
)
if verbose:
logger.info('title = %r' % (title,))
# logger.info('nd_labels = %r' % (nd_labels,))
logger.info('target_label = %r' % (target_label,))
logger.info('known_nd_data = %r' % (known_nd_data,))
# logger.info('known_target_points = %r' % (known_target_points,))
# PLOT3D = not ut.get_argflag('--no3dsurf')
# PLOT3D = ut.get_argflag('--no2dsurf')
PLOT3D = ut.get_argflag('--3dsurf')
if PLOT3D:
pt.plot_search_surface(
known_nd_data,
known_target_points,
nd_labels,
target_label,
title=title,
fnum=fnum,
pnum=pnum,
)
else:
# Convert known nd data into a multiplot-ish format
nonconst_basis_vals = np.unique(known_nd_data.T[1])
# Find which colors will not be used
nonconst_key = nd_labels_[1]
nonconst_basis = np.array(basis_dict[nonconst_key])
nonconst_covers_basis = np.in1d(nonconst_basis, nonconst_basis_vals)
# I dont remember what was trying to happen here
nonconst_color_list = ut.compress(color_list, nonconst_covers_basis)
nonconst_marker_list = ut.compress(marker_list, nonconst_covers_basis)
pt.plot_multiple_scores(
known_nd_data,
known_target_points,
nd_labels,
target_label,
title=title,
color_list=nonconst_color_list,
marker_list=nonconst_marker_list,
fnum=fnum,
pnum=pnum,
num_yticks=num_yticks,
ymin=ymin,
ymax=100,
ypad=0.5,
xpad=0.05,
legend_loc='lower right',
# **FONTKW
)
fig = pt.gcf()
ax = fig.axes[0]
pt.plt.sca(ax)
# pt.figure(fnum=fnum, pnum=pnum0)
icon = ibs.get_database_icon()
if icon is not None:
# pt.overlay_icon(icon, coords=(0, 0), bbox_alignment=(0, 0))
pt.overlay_icon(icon, coords=(0.001, 0.001), bbox_alignment=(0, 0))
nd_labels = [
annotation_configs.shorten_to_alias_labels(key) for key in nd_labels_full
]
plotname = 'Effect of ' + ut.conj_phrase(nd_labels, 'and') + ' on accuracy'
figtitle = testres.make_figtitle(plotname)
# hack
if 1 or verbose:
testres.print_unique_annot_config_stats()
# pt.set_figtitle(figtitle, size=14)
pt.set_figtitle(figtitle)
# HACK FOR FIGSIZE
fig = pt.gcf()
fig.set_size_inches(14, 4)
pt.adjust_subplots(
left=0.05, bottom=0.08, top=0.80, right=0.95, wspace=0.2, hspace=0.3
)
if ut.get_argflag('--contextadjust'):
pt.adjust_subplots(left=0.1, bottom=0.25, wspace=0.2, hspace=0.2)
pt.adjust_subplots(use_argv=True)
[docs]def temp_num_exmaples_cmc():
import wbia
ibs1, testres1 = wbia.testdata_expts(
dbdir=ut.truepath('~/lev/media/hdd/golden/GGR-IBEIS'),
t='Ell:K=1',
# t='best:prescore_method=nsum',
# a='timectrl'
a='timectrl:species=zebra_grevys,qmin_pername=3,dsample_per_name=[1,2],dsize=1939',
)
ibs2, testres2 = wbia.testdata_expts(
dbdir=ut.truepath('~/lev/media/hdd/golden/GZGC'),
# t='best:prescore_method=nsum',
t='CircQRH:K=3',
a='timectrl:species=zebra_plains,qmin_pername=3,dsample_per_name=[1,2],dsize=1187'
# a='timectrl:species=zebra_plains,sample_per_name=3'
)
key = 'qx2_gt_rank'
ydata_list = []
label_list = []
num_ranks = 5
testres_list = [testres1, testres2]
ibs_list = [ibs1, ibs2]
# ibs = ibs1
# testres = testres1
for ibs, testres in zip(ibs_list, testres_list):
cfgx2_cmc, edges = testres.get_rank_percentage_cumhist(
bins='dense', key=key, join_acfgs=True
)
species = ibs.get_dominant_species(testres.qaids)
species_nice = ibs.get_species_nice(
ibs.get_species_rowids_from_text(ibs.get_dominant_species(testres.qaids))
)
species_nice = {
'zebra_grevys': "Grevy's Zebras",
'zebra_plains': 'Plains Zebras',
'giraffe_masai': 'Masai Giraffes',
}.get(species, species_nice)
cmcs = cfgx2_cmc.T[:num_ranks].T
lbls = testres.get_varied_labels(shorten=True, join_acfgs=True)
# lbls = ['%6.2f%% @ rank #1' % (cmc[0],) + ' - ' + lbl + ' - ' + species_nice
# for lbl, cmc in zip(lbls, cmcs)]
lbls = [
species_nice + ' - ' + lbl.replace('dpername', 'exemplars')
for lbl, cmc in zip(lbls, cmcs)
]
ydata_list.extend(cmcs)
label_list.extend(lbls)
sortx = np.argsort(ut.take_column(ydata_list, 0))[::-1]
ydata_list = ut.take(ydata_list, sortx)
label_list = ut.take(label_list, sortx)
xdata = list(range(1, num_ranks + 1))
import wbia.plottool as pt
ymin = 30
xpad = 0.9 # if kind == 'plot' else .5
num_yticks = 8 if ymin == 30 else 11
# ymin = 30 if cfgx2_cumhist_percent.min() > 30 and False else 0
fig = pt.multi_plot(
xdata,
ydata_list,
marker_list=pt.distinct_markers(len(ydata_list)),
label_list=label_list,
num_xticks=num_ranks,
xlabel='rank',
ylabel='recognition rate (%)',
legend_loc='lower right',
num_yticks=num_yticks,
ymax=100,
ymin=ymin,
ypad=0.5,
xmin=xpad,
xmax=num_ranks + 1 - xpad,
use_legend=True,
title='Cumulative match characteristics',
legendsize=10,
titlesize=12,
labelsize=12,
)
fig.set_size_inches(*((1.0 * np.array([4, 3])).tolist()))
pt.adjust_subplots(bottom=0.15, top=0.9, left=0.2)
fpath = 'cmc-dpername-combined.png'
fig.savefig(fpath, transparent=True, edgecolor='none')
import vtool as vt
vt.clipwhite_ondisk(fpath, fpath, verbose=True)
ut.startfile(fpath)
[docs]def temp_multidb_cmc():
"""
Plots multiple database CMC curves in the same plot for the AI for social
good paper
"""
import wbia
ibs1, testres1 = wbia.testdata_expts(
dbdir=ut.truepath('~/lev/media/hdd/golden/GGR-IBEIS'),
t='Ell:K=4',
# t='best:prescore_method=nsum',
a='timectrl'
# a='timectrl:species=zebra_grevys,qmin_pername=3,dsample_per_name=2'
)
ibs2, testres2 = wbia.testdata_expts(
dbdir=ut.truepath('~/lev/media/hdd/golden/GZGC'),
# t='best:prescore_method=nsum',
t='CircQRH:K=3',
a='timectrl:species=zebra_plains'
# a='timectrl:species=zebra_plains,qmin_pername=3,dsample_per_name=2'
# a='timectrl:species=zebra_plains,sample_per_name=3'
)
ibs3, testres3 = wbia.testdata_expts(
dbdir=ut.truepath('~/lev/media/hdd/golden/GZGC'),
t='Ell:K=2',
a='timectrl1h:species=giraffe_masai'
# a='timectrl1h:species=giraffe_masai,qmin_pername=2,dsample_per_name=2'
)
testres_list = [testres1, testres2, testres3]
ibs_list = [ibs1, ibs2, ibs3]
key = 'qx2_gt_rank'
ydata_list = []
label_list = []
num_ranks = 5
for ibs, testres in zip(ibs_list, testres_list):
cfgx2_cmc, edges = testres.get_rank_percentage_cumhist(
bins='dense', key=key, join_acfgs=True
)
cmc = cfgx2_cmc[0][:num_ranks]
species = ibs.get_dominant_species(testres.qaids)
species_nice = ibs.get_species_nice(
ibs.get_species_rowids_from_text(ibs.get_dominant_species(testres.qaids))
)
species_nice = {
'zebra_grevys': "Grevy's Zebras",
'zebra_plains': 'Plains Zebras',
'giraffe_masai': 'Masai Giraffes',
}.get(species, species_nice)
label = ('%6.2f%% @ rank #1' % (cmc[0],)) + ' - ' + species_nice
label_list.append(label)
ydata_list.append(cmc)
sortx = np.argsort(ut.take_column(ydata_list, 0))[::-1]
ydata_list = ut.take(ydata_list, sortx)
label_list = ut.take(label_list, sortx)
xdata = list(range(1, num_ranks + 1))
import wbia.plottool as pt
ymin = 30
xpad = 0.9 # if kind == 'plot' else .5
num_yticks = 8 if ymin == 30 else 11
# ymin = 30 if cfgx2_cumhist_percent.min() > 30 and False else 0
fig = pt.multi_plot(
xdata,
ydata_list,
marker_list=pt.distinct_markers(len(ydata_list)),
label_list=label_list,
num_xticks=num_ranks,
xlabel='rank',
ylabel='recognition rate (%)',
legend_loc='lower right',
num_yticks=num_yticks,
ymax=100,
ymin=ymin,
ypad=0.5,
xmin=xpad,
xmax=num_ranks + 1 - xpad,
use_legend=True,
title='Cumulative match characteristics',
)
fig.set_size_inches(*((1.5 * np.array([4, 3])).tolist()))
pt.adjust_subplots(bottom=0.15)
fig.savefig('cmc-combined.png', transparent=True, edgecolor='none')
import vtool as vt
vt.clipwhite_ondisk('cmc-combined.png', 'cmc-combined.png', verbose=True)
ut.startfile('cmc-combined.png')
[docs]def draw_rank_cmc(
ibs,
testres,
verbose=False,
test_cfgx_slice=None,
group_queries=False,
draw_icon=True,
numranks=5,
kind='cmc',
cdfzoom=True,
**kwargs
):
# numranks=3, kind='bar', cdfzoom=False):
r"""
Args:
ibs (wbia.IBEISController): wbia controller object
testres (TestResult):
TODO:
# Cross-validated results with timectrl
python -m wbia draw_rank_cmc --db PZ_MTEST --show -a timectrl:xval=True -t invar --kind=cmc
CommandLine:
python -m wbia draw_rank_cmc
python -m wbia draw_rank_cmc --db PZ_MTEST --show -a timectrl -t default --kind=cmc
python -m wbia draw_rank_cmc --db PZ_MTEST --show -a :proot=smk,num_words=64000
python -m wbia draw_rank_cmc --db PZ_MTEST --show -a ctrl -t best:prescore_method=csum
python -m wbia draw_rank_cmc --db PZ_MTEST --show -a timectrl -t invar --kind=cmc --cdfzoom
python -m wbia draw_rank_cmc --db PZ_MTEST --show -a varypername_td -t CircQRH_ScoreMech:K=3
#wbia -e rank_cmc --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 --show
python -m wbia.dev -e draw_rank_cmc --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 --show
python -m wbia --tf draw_rank_cmc -t best -a timectrl --db PZ_Master1 --show
python -m wbia --tf draw_rank_cmc --db PZ_Master1 --show -t best \
-a timectrl:qhas_any=\(needswork,correctable,mildviewpoint\),qhas_none=\(viewpoint,photobomb,error:viewpoint,quality\) \
--acfginfo --veryverbtd
wbia --tf draw_match_cases --db GZ_ALL -a ctrl \
-t default:K=1,resize_dim=[width],dim_size=[700,750] \
-f :sortdsc=gfscore,without_tag=scenerymatch,disagree=True \
--show
wbia --tf autogen_ipynb --db GZ_ALL --ipynb -a ctrl \
-t default:K=1,resize_dim=[width],dim_size=[600,700,750] \
default:K=1,resize_dim=[area],dim_size=[450,550,600,650]
wbia draw_rank_cmc --db GZ_ALL -a ctrl -t default --show
wbia draw_match_cases --db GZ_ALL -a ctrl -t default -f :fail=True --show
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import * # NOQA
>>> from wbia.init import main_helpers
>>> #ibs, testres = main_helpers.testdata_expts(
>>> # 'seaturtles', a='default2:qhas_any=(left),sample_occur=True,occur_offset=[0,1,2,3,4,5,6,7,8],num_names=None')
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> kwargs = ut.argparse_funckw(draw_rank_cmc)
>>> result = draw_rank_cmc(ibs, testres, **kwargs)
>>> ut.show_if_requested()
>>> print(result)
"""
import wbia.plottool as pt
if group_queries:
key = 'qnx2_gt_name_rank'
target_label = 'accuracy (% per name)'
else:
key = 'qx2_gt_rank'
target_label = 'accuracy (% per annotation)'
join_acfgs = True
cfgx2_cumhist_percent, edges = testres.get_rank_percentage_cumhist(
bins='dense', key=key, join_acfgs=join_acfgs
)
# Do this twice, but no sep the first time
cfglbl_list = testres.get_varied_labels(shorten=True, join_acfgs=join_acfgs)
cfglbl_to_score = ut.dzip(cfglbl_list, cfgx2_cumhist_percent.T[0])
cfglbl_to_score = ut.sort_dict(cfglbl_to_score, part='vals')
logger.info(
'accuracy @rank1: ' + ut.repr4(cfglbl_to_score, strkeys=True, precision=4)
)
cfglbl_list = testres.get_varied_labels(
shorten=True, join_acfgs=join_acfgs, sep=kwargs.get('sep', '')
)
label_list = [
('%6.2f%%' % (percent,)) + ' - ' + label
for label, percent in zip(cfglbl_list, cfgx2_cumhist_percent.T[0])
]
cmap_seed = ut.get_argval('--prefix', type_=str, default=None)
color_list = pt.distinct_colors(len(label_list), cmap_seed=cmap_seed)
marker_list = pt.distinct_markers(len(label_list))
test_cfgx_slice = ut.get_argval(
'--test_cfgx_slice', type_='fuzzy_subset', default=test_cfgx_slice
)
if test_cfgx_slice is not None:
logger.info('test_cfgx_slice = %r' % (test_cfgx_slice,))
cfgx2_cumhist_percent = np.array(ut.take(cfgx2_cumhist_percent, test_cfgx_slice))
label_list = ut.take(label_list, test_cfgx_slice)
color_list = ut.take(color_list, test_cfgx_slice)
marker_list = ut.take(marker_list, test_cfgx_slice)
# Order cdf list by rank0
# sortx = cfgx2_cumhist_percent.T[0].argsort()[::-1]
sortx = vt.argsort_records(cfgx2_cumhist_percent.T)[::-1]
label_list = ut.take(label_list, sortx)
cfgx2_cumhist_percent = np.array(ut.take(cfgx2_cumhist_percent, sortx))
color_list = ut.take(color_list, sortx)
marker_list = ut.take(marker_list, sortx)
#
if verbose:
testres.print_unique_annot_config_stats(ibs)
numranks = ut.get_argval('--numranks', type_=int, default=numranks)
if numranks is None:
numranks = len(cfgx2_cumhist_percent.T)
maxpos = min(len(cfgx2_cumhist_percent.T), numranks)
cfgx2_cumhist_short = cfgx2_cumhist_percent[:, 0:maxpos]
edges_short = edges[0 : min(len(edges), numranks + 1)]
# twoplots = not ut.get_argflag('--cdfzoom')
twoplots = not cdfzoom
pnum_ = pt.make_pnum_nextgen(nRows=twoplots + 1, nCols=1)
fnum = pt.ensure_fnum(None)
# target_label = '% groundtrue matches ≤ rank'
ymin = 30 if cfgx2_cumhist_percent.min() > 30 and False else 0
num_yticks = 8 if ymin == 30 else 11
kind = ut.get_argval('--kind', default=kind)
if kind is None:
kind = 'bar'
elif kind == 'cmc':
kind = 'plot'
plotname = 'Cumulative Match Characteristic (CMC)'
plotname = ut.get_argval('--plotname', default=plotname)
figtitle = testres.make_figtitle(plotname)
xpad = 0.9 if kind == 'plot' else 0.5
cumhistkw = dict(
xlabel='rank',
ylabel=target_label,
color_list=color_list,
marker_list=marker_list,
fnum=fnum,
# legend_loc='lower right',
legend_loc='lower right',
num_yticks=num_yticks,
ymax=100,
ymin=ymin,
ypad=0.5,
xmin=xpad,
kind=kind,
title=figtitle,
# figtitle=figtitle,
# xpad=.05,
# **FONTKW
)
cumhistkw.update(kwargs)
# if not twoplots:
minval = numranks if numranks <= 10 else 10
pt.plot_rank_cumhist(
cfgx2_cumhist_short,
edges=edges_short,
label_list=label_list,
num_xticks=max(5, min(minval, (numranks // 20) + 2)),
# legend_alpha=.85,
legend_alpha=0.92,
# legendsize=12,
xmax=numranks + 1 - xpad,
use_legend=not twoplots,
pnum=pnum_(),
**cumhistkw
)
if twoplots:
del cumhistkw['title']
numranks2 = len(cfgx2_cumhist_percent.T)
ax1 = pt.gca()
pt.plot_rank_cumhist(
cfgx2_cumhist_percent,
edges=edges,
label_list=label_list,
num_xticks=numranks2,
use_legend=twoplots,
pnum=pnum_(),
xmax=numranks2 + 1 - xpad,
**cumhistkw
)
ax2 = pt.gca()
# pt.zoom_effect01(ax1, ax2, 1, numranks2, fc='w')
# pt.zoom_effect01(ax1, ax2, 1, numranks, fc='w')
pt.zoom_effect01(ax1, ax2, 1, numranks, ec='k', fc='w')
icon = ibs.get_database_icon()
# logger.info('draw_icon = %r' % (draw_icon,))
if draw_icon and icon is not None:
# ax = pt.gca()
# ax.get_xlim()
pt.overlay_icon(icon, bbox_alignment=(0, 0), as_artist=True)
# pt.overlay_icon(icon, bbox_alignment=(0, 0), as_artist=False, max_asize=(.5, .5))
# pt.overlay_icon(icon, bbox_alignment=(0, 1), as_artist=False)
# max_asize=(2, 4))
pass
# ax.get_ylim()
# ax = pt.gca()
# ax.grid(True)
# fig = pt.gcf()
# import utool as ut
# HACK FOR FIGSIZE
# fig.set_size_inches(15, 7)
if ut.get_argflag('--contextadjust') or True:
pt.adjust_subplots(left=0.05, bottom=0.08, wspace=0.0, hspace=0.15)
pt.adjust_subplots(use_argv=True)
[docs]@profile
def draw_case_timedeltas(ibs, testres, falsepos=None, truepos=None, verbose=False):
r"""
CommandLine:
python -m wbia.dev -e draw_case_timedeltas --show
python -m wbia.dev -e draw_case_timedeltas --show -t default \
-a unctrl:num_names=1,name_offset=[1,2]
python -m wbia.dev -e draw_case_timedeltas --show -t default \
-a unctrl:num_names=1,name_offset=[1,2],joinme=1
python -m wbia.dev -e draw_case_timedeltas --show -t default \
-a unctrl:num_names=1,name_offset=[1,2] \
unctrl:num_names=1,name_offset=[3,0]
python -m wbia.dev -e timedelta_hist --show -t baseline \
-a unctrl ctrl:force_const_size=True unctrl:force_const_size=True \
--consistent --db PZ_MTEST
# Testing
python -m wbia.dev -e timedelta_hist --show -t baseline \
-a unctrl ctrl:force_const_size=True unctrl:force_const_size=True \
--consistent --db PZ_Master1
python -m wbia.dev -e timedelta_hist --show -t baseline \
-a unctrl ctrl:sample_rule_ref=max_timedelta --db PZ_Master1 \
--aidcfginfo
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import * # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> draw_case_timedeltas(ibs, testres)
>>> ut.show_if_requested()
"""
import wbia.plottool as pt
import datetime
plotkw = {}
plotkw['markersize'] = 12
plotkw['marker_list'] = []
# plotkw['linestyle'] = '--'
if verbose:
testres.print_unique_annot_config_stats(ibs)
join_acfgs = True
# cfgx2_shortlbl = testres.get_short_cfglbls()
cfgx2_shortlbl = testres.get_short_cfglbls(join_acfgs=join_acfgs)
if falsepos is None:
falsepos = ut.get_argflag('--falsepos')
if truepos is None:
truepos = ut.get_argflag('--truepos')
X_data_list = []
X_label_list = []
truth2_prop, prop2_mat = testres.get_truth2_prop(join_acfg=join_acfgs)
# is_failure = prop2_mat['is_failure']
is_success = prop2_mat['is_success']
for cfgx, lbl in enumerate(cfgx2_shortlbl):
gt_timedelta = truth2_prop['gt']['timedelta'].T
gf_timedelta = truth2_prop['gf']['timedelta'].T
# gt_f_td = gt_timedelta[cfgx][is_failure.T[cfgx]]
# gf_f_td = gf_timedelta[cfgx][is_failure.T[cfgx]]
gt_s_td = gt_timedelta[cfgx][is_success.T[cfgx]]
gf_s_td = gf_timedelta[cfgx][is_success.T[cfgx]]
if not falsepos or truepos:
X_data_list += [
gt_s_td,
# gf_s_td
]
X_label_list += [
'TP ' + lbl,
# 'FP ' + lbl
]
if falsepos:
X_data_list += [gf_s_td]
X_label_list += ['FP ' + lbl]
plotkw['marker_list'] += pt.distinct_markers(
1, style='polygon', offset=cfgx, total=len(cfgx2_shortlbl)
)
numnan_list = [(~np.isfinite(X)).sum() for X in X_data_list]
xdata_list = [X[~np.isnan(X)] for X in X_data_list]
max_score = max([0 if len(xdata) == 0 else xdata.max() for xdata in xdata_list])
bins = [
datetime.timedelta(seconds=0).total_seconds(),
datetime.timedelta(minutes=1).total_seconds(),
datetime.timedelta(hours=1).total_seconds(),
datetime.timedelta(days=1).total_seconds(),
datetime.timedelta(weeks=1).total_seconds(),
datetime.timedelta(days=356).total_seconds(),
# np.inf,
max(datetime.timedelta(days=356 * 10).total_seconds(), max_score + 1),
]
# HISTOGRAM
# if False:
freq_list = [np.histogram(xdata, bins)[0] for xdata in xdata_list]
timedelta_strs = [
ut.get_timedelta_str(datetime.timedelta(seconds=b), exclude_zeros=True)
for b in bins
]
bin_labels = [
length + ' - ' + height for length, height in ut.iter_window(timedelta_strs)
]
bin_labels[-1] = '> 1 year'
bin_labels[0] = '< 1 minute'
WITH_NAN = True
if WITH_NAN:
freq_list = [
np.append(freq, [numnan]) for freq, numnan in zip(freq_list, numnan_list)
]
bin_labels += ['nan']
# Make PIE chart
fnum = None
fnum = pt.ensure_fnum(fnum)
pt.figure(fnum=fnum)
pnum_ = pt.make_pnum_nextgen(*pt.get_square_row_cols(len(freq_list)))
bin_labels[0]
# python -m wbia.dev -e timedelta_hist -t baseline -a
# ctrl:force_const_size=True uncontrolled:force_const_size=True
# --consistent --db GZ_ALL --show
colors = pt.distinct_colors(len(bin_labels))
if WITH_NAN:
colors[-1] = pt.GRAY
for count, freq in enumerate(freq_list):
pt.figure(fnum=fnum, pnum=pnum_())
mask = freq > 0
masked_freq = freq.compress(mask, axis=0)
masked_lbls = ut.compress(bin_labels, mask)
masked_colors = ut.compress(colors, mask)
explode = [0] * len(masked_freq)
size = masked_freq.sum()
masked_percent = masked_freq * 100 / size
pt.plt.pie(
masked_percent,
explode=explode,
autopct='%1.1f%%',
labels=masked_lbls,
colors=masked_colors,
)
ax = pt.gca()
ax.set_xlabel(X_label_list[count] + '\nsize=%d' % (size,))
ax.set_aspect('equal')
if ut.get_argflag('--contextadjust'):
pt.adjust_subplots(left=0.08, bottom=0.1, top=0.9, wspace=0.3, hspace=0.1)
pt.adjust_subplots(use_argv=True)
[docs]@profile
def draw_match_cases(
ibs,
testres,
metadata=None,
f=None,
show_in_notebook=False,
annot_modes=None,
figsize=None,
case_pos_list=None,
verbose=None,
interact=None,
figdir=None,
**kwargs
):
r"""
Args:
ibs (wbia.IBEISController): wbia controller object
testres (TestResult): test result object
metadata (None): (default = None)
CommandLine:
python -m wbia --tf draw_match_cases
python -m wbia.dev -e draw_match_cases --figdir=figure
python -m wbia.dev -e draw_match_cases --db PZ_Master1 -a ctrl \
-t default --filt :fail=True,min_gtrank=5,gtrank_lt=20 --render
# Shows the best results
python -m wbia.dev -e cases --db PZ_Master1 \
-a timectrl -t invarbest
--filt :sortasc=gtscore,success=True,index=200:201 --show
# Shows failures sorted by gt score
python -m wbia.dev -e cases --db PZ_Master1 \
-a timectrl -t invarbest \
--filt :sortdsc=gfscore,min_gtrank=1 --show
# Find the untagged photobomb and scenery cases
python -m wbia.dev -e cases --db PZ_Master1 -a timectrl \
-t invarbest --show --filt \
:orderby=gfscore,reverse=1,min_gtrank=1,max_gf_td=24h,max_gf_tags=0
# Find untagged failures
python -m wbia.dev -e cases --db PZ_Master1 -a timectrl \
-t invarbest \
--filt :orderby=gfscore,reverse=1,min_gtrank=1,max_gf_tags=0 --show
# Show disagreement cases
wbia --tf draw_match_cases --db PZ_MTEST -a default:size=20 \
-t default:K=[1,4] \
--filt :disagree=True,index=0:4 --show
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import * # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> filt_cfg = main_helpers.testdata_filtcfg()
>>> metadata = None
>>> figdir = ut.get_argval(('--figdir', '--dpath'), type_=str, default=None)
>>> analysis_fpath_list = draw_match_cases(ibs, testres, metadata,
>>> f=filt_cfg, figdir=figdir)
>>> ut.show_if_requested()
"""
import wbia.plottool as pt
if ut.NOT_QUIET:
ut.colorprint('[expt] Drawing individual results', 'yellow')
if True:
import matplotlib as mpl
from wbia.scripts.thesis import TMP_RC
mpl.rcParams.update(TMP_RC)
# ### ARGUMENT PARSING AND RECTIFICATION ###
cmdaug = ut.get_argval('--cmdaug', type_=str, help_='candhack')
show = ut.get_argflag('--show')
if interact is None:
interact = ut.get_argflag('--interact')
if figdir is None:
figdir = ibs.get_fig_dir()
show_kwargs = {'N': 3, 'ori': True, 'ell_alpha': 0.9}
# show analysis
show_kwargs['show_query'] = False
show_kwargs['viz_name_score'] = kwargs.get('viz_name_score', True)
show_kwargs['show_timedelta'] = True
show_kwargs['show_gf'] = True
show_kwargs['colorbar_'] = False
show_kwargs['with_figtitle'] = show_in_notebook
show_kwargs['fastmode'] = True
if annot_modes is None:
annot_modes = ut.get_argval('--annotmodes', default=[0])
filt_cfg = f
if case_pos_list is None:
case_pos_list = testres.case_sample2(filt_cfg, verbose=verbose) # NOQA
#########################
# ### DIRECTORY SETUP ###
figdir = ut.truepath(figdir)
case_figdir = join(figdir, 'cases_' + ibs.get_dbname())
ut.ensuredir(case_figdir)
if ut.get_argflag(('--view-fig-directory', '--vf')):
ut.view_directory(case_figdir)
# Common directory
indiv_results_figdir = ut.ensuredir((case_figdir, 'indiv_results'))
top_rank_analysis_dir = ut.ensuredir((case_figdir, 'top_rank_analysis'))
dump = False
DO_COPY_QUEUE = True
if DO_COPY_QUEUE:
cpq = draw_helpers.IndividualResultsCopyTaskQueue()
if ut.NOT_QUIET:
logger.info('case_figdir = %r' % (case_figdir,))
##########################
# ### INTERACTIVE SETUP ###
def toggle_annot_mode():
for ix in range(len(annot_modes)):
# See viz_qres.py for more annot_mode info
# TODO: use viz_qres to get the number of possible annot modes.
annot_modes[ix] = annot_modes[ix] + 1 % 4
def toggle_fast_mode():
show_kwargs['fastmode'] = not show_kwargs['fastmode']
logger.info("show_kwargs['fastmode'] = %r" % (show_kwargs['fastmode'],))
custom_actions = [
('present', ['s'], 'present', pt.present),
('toggle_annot_mode', ['a'], 'toggle_annot_mode', toggle_annot_mode),
(
'toggle_fast_mode',
['f'],
'toggle_fast_mode',
toggle_fast_mode,
'Fast mode lowers drwaing quality',
),
]
##########################
cfgx2_qreq_ = testres.cfgx2_qreq_
qaids = testres.get_test_qaids()
cfgx2_shortlbl = testres.get_short_cfglbls()
qx_list, cfgx_list = case_pos_list.T
# Get configs needed for each query
qx2_cfgxs = ut.group_items(cfgx_list, qx_list)
if show_in_notebook:
cfg_colors = pt.distinct_colors(len(testres.cfgx2_qreq_))
unique_qx = ut.unique(qx_list)
if interact:
_iter = ut.InteractiveIter(
unique_qx, enabled=interact, custom_actions=custom_actions
)
else:
_iter = ut.ProgIter(unique_qx, lbl='drawing cases')
fnum = pt.ensure_fnum(None)
fpaths_list = []
analysis_fpath_list = []
for count, qx in enumerate(_iter):
cfgxs = qx2_cfgxs[qx]
qreq_list = ut.take(cfgx2_qreq_, cfgxs)
# TODO: try to get away with not reloading query results or loading
# them in batch if possible
# It actually doesnt take that long. the drawing is what hurts
# TODO: be able to load old results even if they are currently invalid
qaid = qaids[qx]
cm_list = [qreq_.execute(qaids=[qaid])[0] for qreq_ in qreq_list]
fpaths_list.append([])
truth2_prop, prop2_mat = testres.get_truth2_prop()
if 0 or ut.VERBOSE:
logger.info('=== QUERY INFO ===')
logger.info('=== QUERY INFO ===')
logger.info('qaid = %r' % (qaid,))
logger.info('qx = %r' % (qx,))
logger.info('cfgxs = %r' % (cfgxs,))
# print testres info about this item
take_cfgs = ut.partial(ut.take, index_list=cfgxs)
take_qx = ut.partial(ut.take, index_list=qx)
truth_cfgs = ut.hmap_vals(take_qx, truth2_prop)
truth_item = ut.hmap_vals(take_cfgs, truth_cfgs, max_depth=1)
prop_cfgs = ut.hmap_vals(take_qx, prop2_mat)
prop_item = ut.hmap_vals(take_cfgs, prop_cfgs, max_depth=0)
logger.info('truth2_prop[item] = ' + ut.repr3(truth_item, nl=2))
logger.info('prop2_mat[item] = ' + ut.repr3(prop_item, nl=1))
if show_in_notebook:
# hack to show vertical line in notebook separate configs
fnum = fnum + 1
pt.imshow(np.zeros((1, 200), dtype=np.uint8), fnum=fnum)
for count2, (cfgx, cm, qreq_) in enumerate(zip(cfgxs, cm_list, qreq_list)):
if show_in_notebook:
fnum = fnum + 1
else:
fnum = cfgx if interact else 1
# cm = cm.extend_results(qreq_)
# Get row and column index
cfgstr = testres.get_cfgstr(cfgx)
query_lbl = cfgx2_shortlbl[cfgx]
qres_dpath = 'qaid={qaid}'.format(qaid=cm.qaid)
individ_results_dpath = join(indiv_results_figdir, qres_dpath)
ut.ensuredir(individ_results_dpath)
# Draw Result
# try to shorten query labels a bit
query_lbl = query_lbl.replace(' ', '').replace("'", '')
_query_lbl = query_lbl
qres_fname = query_lbl + '.png'
analysis_fpath = join(individ_results_dpath, qres_fname)
if show or interact or show_in_notebook:
needs_draw = True
else:
needs_draw = not (dump and ut.checkpath(analysis_fpath))
if needs_draw:
bar_label = 'Case: Query %r / %r, Config %r / %r --- qaid=%d, cfgx=%r' % (
count + 1,
len(qx_list),
count2 + 1,
len(cfgxs),
qaid,
cfgx,
)
logger.info('bar_label = %r' % (bar_label,))
if show_in_notebook:
# hack to show vertical line in notebook
if len(cfg_colors) > 0:
bar = np.zeros((1, 400, 3), dtype=np.uint8) + (
np.array(cfg_colors[cfgx]) * 255
)
fnum = fnum + 1
fig, ax = pt.imshow(bar, fnum=fnum)
pt.set_xlabel(bar_label, ax=ax)
pt.plt.show() # Need to show when doing notebook
for annot_mode in annot_modes:
show_kwargs['annot_mode'] = annot_mode
if show_in_notebook:
fnum = fnum + 1
if interact:
cm.ishow_analysis(
qreq_, figtitle=_query_lbl, fnum=fnum, **show_kwargs
)
else:
cm.show_analysis(
qreq_, figtitle=_query_lbl, fnum=fnum, **show_kwargs
)
if show_in_notebook:
_query_lbl = '' # only show the query label once
if figsize is not None:
fig = pt.gcf()
fig.set_size_inches(*figsize)
fig.set_dpi(256)
pt.plt.show()
if cmdaug is not None:
# Hack for candidacy
analysis_fpath = join(figdir, 'figuresC/case_%s.png' % (cmdaug,))
logger.info('analysis_fpath = %r' % (analysis_fpath,))
if show_in_notebook:
pt.plt.show()
# elif not show:
if False:
fig = pt.gcf()
logger.info('analysis_fpath = %r' % (analysis_fpath,))
fig.savefig(analysis_fpath)
vt.clipwhite_ondisk(
analysis_fpath, analysis_fpath, verbose=ut.VERBOSE
)
if DO_COPY_QUEUE:
if cmdaug is None:
cpq.append_copy_task(analysis_fpath, top_rank_analysis_dir)
analysis_fpath_list.append(analysis_fpath)
fpaths_list[-1].append(analysis_fpath)
if metadata is not None:
metadata.set_global_data(
cfgstr, cm.qaid, 'analysis_fpath', analysis_fpath
)
# if some condition of of batch sizes
if DO_COPY_QUEUE:
flush_freq = 4
if count % flush_freq == (flush_freq - 1):
cpq.flush_copy_tasks()
if DO_COPY_QUEUE:
# Copy summary images to query_analysis folder
cpq.flush_copy_tasks()
return analysis_fpath_list