wbia.expt package

Submodules

wbia.expt.annotation_configs module

Definitions for common aid configurations

Rename to annot_cfgdef

wbia.expt.annotation_configs.apply_qualcontrol(acfg)

wbia.expt.annotation_configs.apply_timecontrol(acfg, min_timedelta='6h', require_timestamp=True)

wbia.expt.annotation_configs.compress_acfg_list_for_printing(acfg_list)

CommandLine:

python -m wbia –tf compress_acfg_list_for_printing

Ignore:

>>> from wbia.expt.annotation_configs import *  # NOQA
>>> qcfg_list = [{'f': 1, 'b': 1}, {'f': 2, 'b': 1}, {'f': 3, 'b': 1, 'z': 4}]
>>> acfg_list = [{'qcfg': qcfg} for qcfg in qcfg_list]
>>> nonvaried_dict, varied_dicts = compress_acfg_list_for_printing(acfg_list)
>>> result = ('varied_dicts = %s\n' % (ut.repr2(varied_dicts),))
>>> result += ('nonvaried_dict = %s' % (ut.repr2(nonvaried_dict),))
>>> print(result)

wbia.expt.annotation_configs.compress_aidcfg(acfg, filter_nones=False, filter_empty=False, force_noncommon=[])

Idea is to add a third subconfig named common that is the intersection of qcfg and dcfg.

Parameters:acfg (dict) – Returns:acfg Return type:dict

CommandLine:

# python -m wbia –tf compress_aidcfg python -m wbia.expt.annotation_configs –exec-compress_aidcfg –show

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.annotation_configs import *  # NOQA
>>> acfg = default
>>> acfg = compress_aidcfg(acfg)
>>> result = ('acfg = %s' % (ut.repr2(acfg),))
>>> print(default)
>>> print(result)

wbia.expt.annotation_configs.ctrl = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 1, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg –db PZ_Master1 -a timectrl

wbia.expt.annotation_configs.default2 = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': None, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': None, 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': None, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': None, 'view': None, 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': None, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': None, 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': None, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': None, 'view': None, 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg –db PZ_Master1 -a unctrl

wbia.expt.annotation_configs.flatten_acfg_list(acfg_list)

Returns a new config where subconfig params are prefixed by subconfig keys

wbia.expt.annotation_configs.get_varied_acfg_labels(acfg_list, mainkey='_cfgname', checkname=False)

>>> from wbia.expt.annotation_configs import *  # NOQA

wbia.expt.annotation_configs.partition_acfg_list(acfg_list)

wbia.expt.annotation_configs.print_acfg(acfg, expanded_aids=None, ibs=None, **kwargs)

wbia.expt.annotation_configs.print_acfg_list(acfg_list, expanded_aids_list=None, ibs=None, combined=False, only_summary=False, **kwargs)

Parameters:

• acfg_list (list) –
• expanded_aids_list (list) – (default = None)
• ibs (IBEISController) – wbia controller object(default = None)
• combined (bool) – (default = False)

CommandLine:

python -m wbia.expt.annotation_configs –exec-print_acfg_list

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.annotation_configs import *  # NOQA
>>> import wbia
>>> ibs = wbia.opendb('testdb1')
>>> a = ['default']
>>> acfg_list, expanded_aids_list = wbia.expt.experiment_helpers.get_annotcfg_list(
>>>     ibs, acfg_name_list=a, verbose=0)
>>> combined = False
>>> result = print_acfg_list(acfg_list, expanded_aids_list, ibs, combined)
>>> print(result)

wbia.expt.annotation_configs.shorten_to_alias_labels(key)

wbia.expt.annotation_configs.timectrlL = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 1, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': False, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': False, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg –db PZ_Master1 -a timequalctrl

wbia.expt.annotation_configs.timectrlhard = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 1, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': ('needswork', 'correctable', 'mildviewpoint'), 'has_none': ('viewpoint', 'photobomb', 'error:viewpoint', 'quality'), 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg -a viewdiff –db PZ_Master1 –verbtd –nocache –per_vp=True wbia -e print_acfg -a viewdiff_td –db PZ_Master1 –verbtd –nocache –per_vp=True

wbia.expt.annotation_configs.unctrl_comp = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': None, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': None, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 1, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg –db PZ_Master1 -a ctrl wbia -e print_acfg –db PZ_Master1 -a unctrl ctrl::unctrl:qpername=1,qview_ext=0 wbia -e print_acfg –db PZ_Master1 -a unctrl ctrl::unctrl_comp

wbia.expt.annotation_configs.unflatten_acfgdict(flat_dict, prefix_list=['dcfg', 'qcfg'])

wbia.expt.annotation_configs.varynannots_td1h = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 1, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '1h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': [1], 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': [0.0, 0.01, 0.05, 0.125, 0.25, 0.375, 0.5, 0.75, 0.875, 0.95, 1.0], 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '1h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg -a viewpoint_compare –db PZ_Master1 –verbtd –nocache python -m wbia –tf parse_acfg_combo_list -a viewpoint_compare python -m wbia –tf get_annotcfg_list –db PZ_Master1 -a viewpoint_compare –verbtd # Check composition of names per viewpoint python -m wbia –tf group_annots_by_multi_prop –db PZ_Master1 –props=yaw_texts,name_rowids –keys1 frontleft python -m wbia –tf get_annot_stats_dict –db PZ_Master1 –per_name_vpedge=True

TODO: Need to explicitly setup the common config I think? wbia -e print_acfg -a viewdiff:min_timedelta=1h –db PZ_Master1 –verbtd –nocache-aid wbia –tf get_annotcfg_list -a viewdiff:min_timedelta=1h –db PZ_Master1 –verbtd –nocache-aid

wbia.expt.annotation_configs.varypername2_td = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 1, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': [1, 2], 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 3, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg –db PZ_Master1 -a ctrl2 wbia -e print_acfg –db PZ_Master1 -a timectrl2 wbia -e rank_cmc –db PZ_Master1 -a timectrl2 -t invarbest

wbia.expt.annotation_configs.varypername_td1h = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '1h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': [1, 2, 3], 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 4, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '1h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg –db PZ_Master1 -a varypername_tdqual

wbia.expt.annotation_configs.varypername_tdqual = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': True, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': [1, 2, 3], 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 4, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '6h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': True, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

python -m wbia –tf get_num_annots_per_name –db PZ_Master1 wbia -e print_acfg -a varysize2 –db PZ_Master1 –verbtd –nocache wbia -e print_acfg -a varysize2 –db NNP_MasterGIRM_core –verbtd –nocache

wbia.expt.annotation_configs.varysize = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 1, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': [1, 2, 3], 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': [0.25, 0.5, 0.75], 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': None, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 4, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': None, 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': None, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': None}}

wbia -e print_acfg -a varysize2_td –db PZ_Master1 –verbtd –nocache

wbia.expt.annotation_configs.viewdiff_td1h = {'dcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': True, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 1, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '1h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': 1, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': ['primary'], 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': '#primary>0&#primary1>0'}, 'qcfg': {'any_endswith': None, 'any_match': None, 'any_startswith': None, 'been_adjusted': None, 'contributor_contains': None, 'crossval_enc': None, 'exclude_ref_contact': None, 'exclude_reference': None, 'force_const_size': True, 'hackerrors': False, 'has_all': None, 'has_any': None, 'has_none': None, 'index': None, 'is_exemplar': None, 'is_known': True, 'joinme': None, 'logic': None, 'max_num': None, 'max_numfeat': None, 'max_pername': None, 'max_pername_global': None, 'max_timestamp': None, 'max_unixtime': None, 'min_num': None, 'min_numfeat': None, 'min_pername': 2, 'min_pername_global': None, 'min_spacedelta': None, 'min_spacetimedelta': None, 'min_timedelta': '1h', 'min_unixtime': None, 'minqual': 'ok', 'multiple': None, 'name_offset': None, 'none_match': None, 'num_names': None, 'occur_offset': None, 'require_gps': None, 'require_quality': None, 'require_timestamp': True, 'require_viewpoint': None, 'reviewed': None, 'same_imageset': None, 'sample_occur': None, 'sample_offset': None, 'sample_per_name': 1, 'sample_per_ref_name': None, 'sample_rule': 'random', 'sample_rule_ref': 'random', 'sample_seed': 0, 'sample_size': None, 'shuffle': False, 'species': 'primary', 'view': 'primary1', 'view_ext': 0, 'view_ext1': None, 'view_ext2': None, 'view_pername': '#primary>0&#primary1>0'}}

qhas_any=(query,),dpername=2,exclude_reference=True –acfginfo –verbtd –veryverbtd wbia get_annotcfg_list –db Oxford -a oxford –acfginfo (‘_QSUUIDS((55)qxlgljvomqpdvlny)’, ‘_DSUUIDS((4240)vhtqsdkrwetbftis)’),

wbia draw_rank_cmc –db Oxford –save oxfordccm.png -p :proot=smk,num_words=[64000],nAssign=[1],sv_on=[False] -a oxford

Type:wbia get_annotcfg_list –db Oxford -a default

wbia.expt.cfghelpers module

Helper module that helps expand parameters for grid search

DEPRICATE: Most of this can likely be replaced by util_gridsearch TODO: rectify with versions in util_gridsearch

It turns out a lot of the commandlines made possible here can be generatd by using bash brace expansion. http://www.linuxjournal.com/content/bash-brace-expansion

wbia.expt.cfghelpers.customize_base_cfg(cfgname, cfgopt_strs, base_cfg, cfgtype, alias_keys=None, valid_keys=None, offset=0, strict=True)

DEPRICATE

wbia.expt.cfghelpers.parse_argv_cfg(argname, default=[''], named_defaults_dict=None, valid_keys=None)

simple configs

Parameters:

• argname –
• default (list) – (default = [])
• named_defaults_dict (dict) – (default = None)
• valid_keys (None) – (default = None)

Returns:

cfg_list

Return type:

list

CommandLine:

python -m wbia.expt.cfghelpers –exec-parse_argv_cfg –filt :foo=bar python -m wbia.expt.cfghelpers –test-parse_argv_cfg

Example

>>> # ENABLE_DOCTET
>>> from wbia.expt.cfghelpers import *  # NOQA
>>> argname = '--filt'
>>> cfg_list = parse_argv_cfg(argname)
>>> result = ('cfg_list = %s' % (str(cfg_list),))
>>> print(result)

wbia.expt.cfghelpers.parse_cfgstr_list2(cfgstr_list, named_defaults_dict=None, cfgtype=None, alias_keys=None, valid_keys=None, expand_nested=True, strict=True, special_join_dict=None, is_nestedcfgtype=False, metadata=None)

Parses config strings. By looking up name in a dict of configs

DEPRICATE

Parameters:

• cfgstr_list (list) –
• named_defaults_dict (dict) – (default = None)
• cfgtype (None) – (default = None)
• alias_keys (None) – (default = None)
• valid_keys (None) – (default = None)
• expand_nested (bool) – (default = True)
• strict (bool) – (default = True)
• - used for annot configs so special joins arent geometrically combined (is_nestedcfgtype) –

Note

Normal Case:

–flag name

Custom Arugment Cases:

–flag name:custom_key1=custom_val1,custom_key2=custom_val2

Multiple Config Case:

–flag name1:custom_args1 name2:custom_args2

Multiple Config (special join) Case:

(here name2 and name3 have some special interaction) –flag name1:custom_args1 name2:custom_args2::name3:custom_args3

Varied Argument Case:

–flag name:key1=[val1,val2]

Returns:cfg_combos_list Return type:list

CommandLine:

python -m wbia.expt.cfghelpers –exec-parse_cfgstr_list2 python -m wbia.expt.cfghelpers –test-parse_cfgstr_list2

Example

>>> # ENABLE_DOCTET
>>> from wbia.expt.cfghelpers import *  # NOQA
>>> cfgstr_list = ['name', 'name:f=1', 'name:b=[1,2]', 'name1:f=1::name2:f=1,b=2']
>>> #cfgstr_list = ['name', 'name1:f=1::name2:f=1,b=2']
>>> named_defaults_dict = None
>>> cfgtype = None
>>> alias_keys = None
>>> valid_keys = None
>>> expand_nested = True
>>> strict = False
>>> special_join_dict = {'joined': True}
>>> cfg_combos_list = parse_cfgstr_list2(cfgstr_list, named_defaults_dict,
>>>                                      cfgtype, alias_keys, valid_keys,
>>>                                      expand_nested, strict,
>>>                                      special_join_dict)
>>> print('cfg_combos_list = %s' % (ut.repr2(cfg_combos_list, nl=2),))
>>> print(ut.depth_profile(cfg_combos_list))
>>> cfg_list = ut.flatten(cfg_combos_list)
>>> cfg_list = ut.flatten([cfg if isinstance(cfg, list) else [cfg] for cfg in cfg_list])
>>> result = ut.repr2(ut.get_varied_cfg_lbls(cfg_list))
>>> print(result)
['name:', 'name:f=1', 'name:b=1', 'name:b=2', 'name1:f=1,joined=True', 'name2:b=2,f=1,joined=True']

wbia.expt.cfghelpers.remove_prefix_hack(cfg, cfgtype, cfg_options, alias_keys)

wbia.expt.draw_helpers module

class wbia.expt.draw_helpers.IndividualResultsCopyTaskQueue

Bases: object

append_copy_task(fpath_orig, dstdir=None)

helper which copies a summary figure to root dir

flush_copy_tasks()

wbia.expt.draw_helpers.make_individual_latex_figures(ibs, fpaths_list, flat_case_labels, cfgx2_shortlbl, case_figdir, analysis_fpath_list)

wbia.expt.experiment_configs module

In this file dicts specify all possible combinations of the varied parameters and lists specify the union of parameters

Rename to pipe_cfgdef

wbia.expt.experiment_configs.apply_CircQRH(cfg)

wbia.expt.experiment_configs.apply_Ell(cfg)

wbia.expt.experiment_configs.apply_EllQRH(cfg)

wbia.expt.experiment_configs.apply_k(cfg)

wbia.expt.experiment_configs.apply_knorm(cfg)

wbia.expt.experiment_configs.apply_param(cfg, **kwargs)

wbia.expt.experiment_configs.augbase(basedict, updatedict)

wbia.expt.experiment_configs.best(metadata)

Infer the best pipeline config based on the metadata

wbia.expt.experiment_configs.get_candidacy_dbnames()

wbia.expt.experiment_drawing module

./dev.py -t custom:affine_invariance=False,adapteq=True,fg_on=False –db Elephants_drop1_ears –allgt –index=0:10 –guiview # NOQA

wbia.expt.experiment_drawing.draw_annot_scoresep(ibs, testres, f=None, verbose=None)

Draws the separation between true positive and true negative name scores.

Todo

plot the difference between the top true score and the next best false score?

CommandLine:

ib python -m wbia draw_annot_scoresep –show python -m wbia draw_annot_scoresep –db PZ_MTEST –allgt -w –show –serial python -m wbia draw_annot_scoresep -t scores –db PZ_MTEST –allgt –show python -m wbia draw_annot_scoresep -t scores –db PZ_Master0 –allgt –show python -m wbia draw_annot_scoresep –db PZ_Master1 -a timectrl -t best –show python -m wbia draw_annot_scoresep –db PZ_Master1 -a timectrl -t best –show -f :without_tag=photobomb

Paper:

python -m wbia draw_annot_scoresep –dbdir lev/media/hdd/golden/GGR-IBEIS -a timectrl –save gz_scoresep.png python -m wbia draw_annot_scoresep –dbdir lev/media/hdd/golden/GZGC -a timectrl:species=zebra_plains –save pz_scoresep.png python -m wbia draw_annot_scoresep –dbdir lev/media/hdd/golden/GZGC -a timectrl1h:species=giraffe_masai –save girm_scoresep.png

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import *  # NOQA
>>> from wbia.init import main_helpers
>>> defaultdb = 'PZ_MTEST'
>>> ibs, testres = main_helpers.testdata_expts(defaultdb, a=['timectrl'], t=['best'])
>>> f = ut.get_argval(('--filt', '-f'), type_=list, default=[''])
>>> draw_annot_scoresep(ibs, testres, f=f, verbose=ut.VERBOSE)
>>> ut.show_if_requested()

Ignore:

import IPython IPython.get_ipython().magic(‘pylab qt4’)

wbia.expt.experiment_drawing.draw_case_timedeltas(ibs, testres, falsepos=None, truepos=None, verbose=False)

CommandLine:

python -m wbia.dev -e draw_case_timedeltas –show python -m wbia.dev -e draw_case_timedeltas –show -t default

-a unctrl:num_names=1,name_offset=[1,2]

python -m wbia.dev -e draw_case_timedeltas –show -t default

-a unctrl:num_names=1,name_offset=[1,2],joinme=1

python -m wbia.dev -e draw_case_timedeltas –show -t default

-a unctrl:num_names=1,name_offset=[1,2]

unctrl:num_names=1,name_offset=[3,0]

python -m wbia.dev -e timedelta_hist –show -t baseline

-a unctrl ctrl:force_const_size=True unctrl:force_const_size=True –consistent –db PZ_MTEST

# Testing python -m wbia.dev -e timedelta_hist –show -t baseline

-a unctrl ctrl:force_const_size=True unctrl:force_const_size=True –consistent –db PZ_Master1

python -m wbia.dev -e timedelta_hist –show -t baseline

-a unctrl ctrl:sample_rule_ref=max_timedelta –db PZ_Master1 –aidcfginfo

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> draw_case_timedeltas(ibs, testres)
>>> ut.show_if_requested()

wbia.expt.experiment_drawing.draw_casetag_hist(ibs, testres, f=None, with_wordcloud=True)

Parameters:

• ibs (wbia.IBEISController) – wbia controller object
• testres (TestResult) – test result object

CommandLine:

wbia –tf -draw_casetag_hist –show

# Experiments I tagged wbia –tf -draw_casetag_hist -a timectrl -t invarbest –db PZ_Master1 –show

wbia -e taghist -a timectrl -t best –db PZ_Master1 –show

wbia -e taghist -a timequalctrl -t invarbest –db PZ_Master1 –show wbia -e taghist -a timequalctrl:minqual=good -t invarbest –db PZ_Master1 –show wbia -e taghist -a timequalctrl:minqual=good -t invarbest –db PZ_Master1 –show –filt :fail=True

# Do more tagging wbia -e cases -a timequalctrl:minqual=good -t invarbest –db PZ_Master1

–filt :orderby=gfscore,reverse=1,min_gtrank=1,max_gf_tags=0 –show

wbia -e print -a timequalctrl:minqual=good -t invarbest –db PZ_Master1 –show wbia -e cases -a timequalctrl -t invarbest –db PZ_Master1

–filt :orderby=gfscore,reverse=1,max_gf_tags=0,:fail=True,min_gf_timedelta=12h –show

wbia -e cases -a timequalctrl -t invarbest –db PZ_Master1

–filt :orderby=gfscore,reverse=1,max_gf_tags=0,:fail=True,min_gf_timedelta=12h –show

python -m wbia -e taghist –db PZ_Master1 -a timectrl -t best

–filt :fail=True –no-wordcloud –hargv=tags –prefix “Failure Case ” –label PZTags –figsize=10,3 –left=.2

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_Master1', a=['timequalcontrolled'])
>>> f = ut.get_argval(('--filt', '-f'), type_=list, default=[''])
>>> draw_casetag_hist(ibs, testres, f=f)
>>> ut.show_if_requested()

wbia.expt.experiment_drawing.draw_match_cases(ibs, testres, metadata=None, f=None, show_in_notebook=False, annot_modes=None, figsize=None, case_pos_list=None, verbose=None, interact=None, figdir=None, **kwargs)

Parameters:

• ibs (wbia.IBEISController) – wbia controller object
• testres (TestResult) – test result object
• metadata (None) – (default = None)

CommandLine:

python -m wbia –tf draw_match_cases python -m wbia.dev -e draw_match_cases –figdir=figure python -m wbia.dev -e draw_match_cases –db PZ_Master1 -a ctrl

-t default –filt :fail=True,min_gtrank=5,gtrank_lt=20 –render

# Shows the best results python -m wbia.dev -e cases –db PZ_Master1

-a timectrl -t invarbest –filt :sortasc=gtscore,success=True,index=200:201 –show

# Shows failures sorted by gt score python -m wbia.dev -e cases –db PZ_Master1

-a timectrl -t invarbest –filt :sortdsc=gfscore,min_gtrank=1 –show

# Find the untagged photobomb and scenery cases python -m wbia.dev -e cases –db PZ_Master1 -a timectrl

-t invarbest –show –filt :orderby=gfscore,reverse=1,min_gtrank=1,max_gf_td=24h,max_gf_tags=0

# Find untagged failures python -m wbia.dev -e cases –db PZ_Master1 -a timectrl

-t invarbest –filt :orderby=gfscore,reverse=1,min_gtrank=1,max_gf_tags=0 –show

# Show disagreement cases wbia –tf draw_match_cases –db PZ_MTEST -a default:size=20

-t default:K=[1,4] –filt :disagree=True,index=0:4 –show

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> filt_cfg = main_helpers.testdata_filtcfg()
>>> metadata = None
>>> figdir = ut.get_argval(('--figdir', '--dpath'), type_=str, default=None)
>>> analysis_fpath_list = draw_match_cases(ibs, testres, metadata,
>>>                                        f=filt_cfg, figdir=figdir)
>>> ut.show_if_requested()

wbia.expt.experiment_drawing.draw_rank_cmc(ibs, testres, verbose=False, test_cfgx_slice=None, group_queries=False, draw_icon=True, numranks=5, kind='cmc', cdfzoom=True, **kwargs)

Parameters:

• ibs (wbia.IBEISController) – wbia controller object
• testres (TestResult) –

Todo

# Cross-validated results with timectrl python -m wbia draw_rank_cmc –db PZ_MTEST –show -a timectrl:xval=True -t invar –kind=cmc

CommandLine:

python -m wbia draw_rank_cmc python -m wbia draw_rank_cmc –db PZ_MTEST –show -a timectrl -t default –kind=cmc

python -m wbia draw_rank_cmc –db PZ_MTEST –show -a :proot=smk,num_words=64000 python -m wbia draw_rank_cmc –db PZ_MTEST –show -a ctrl -t best:prescore_method=csum python -m wbia draw_rank_cmc –db PZ_MTEST –show -a timectrl -t invar –kind=cmc –cdfzoom python -m wbia draw_rank_cmc –db PZ_MTEST –show -a varypername_td -t CircQRH_ScoreMech:K=3 #wbia -e rank_cmc –db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 –show

python -m wbia.dev -e draw_rank_cmc –db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 –show

python -m wbia –tf draw_rank_cmc -t best -a timectrl –db PZ_Master1 –show

python -m wbia –tf draw_rank_cmc –db PZ_Master1 –show -t best

-a timectrl:qhas_any=(needswork,correctable,mildviewpoint),qhas_none=(viewpoint,photobomb,error:viewpoint,quality) –acfginfo –veryverbtd

wbia –tf draw_match_cases –db GZ_ALL -a ctrl

-t default:K=1,resize_dim=[width],dim_size=[700,750] -f :sortdsc=gfscore,without_tag=scenerymatch,disagree=True –show

wbia –tf autogen_ipynb –db GZ_ALL –ipynb -a ctrl

-t default:K=1,resize_dim=[width],dim_size=[600,700,750]

default:K=1,resize_dim=[area],dim_size=[450,550,600,650]

wbia draw_rank_cmc –db GZ_ALL -a ctrl -t default –show wbia draw_match_cases –db GZ_ALL -a ctrl -t default -f :fail=True –show

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import *  # NOQA
>>> from wbia.init import main_helpers
>>> #ibs, testres = main_helpers.testdata_expts(
>>> #    'seaturtles', a='default2:qhas_any=(left),sample_occur=True,occur_offset=[0,1,2,3,4,5,6,7,8],num_names=None')
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> kwargs = ut.argparse_funckw(draw_rank_cmc)
>>> result = draw_rank_cmc(ibs, testres, **kwargs)
>>> ut.show_if_requested()
>>> print(result)

wbia.expt.experiment_drawing.draw_rank_surface(ibs, testres, verbose=None, fnum=None)

Draws n dimensional data + a score / rank The rank is always on the y axis.

The first dimension is on the x axis. The second dimension is split over multiple plots. The third dimension becomes multiple lines. May need to clean this scheme up a bit.

Parameters:

• ibs (wbia.IBEISController) – wbia controller object
• testres (TestResult) – test result object

CommandLine:

wbia –tf draw_rank_surface –db PZ_Master1 -a varysize_td -t CircQRH_K –show

wbia –tf draw_rank_surface –show -t best -a varysize –db PZ_Master1 –show

wbia –tf draw_rank_surface –show -t CircQRH_K -a varysize_td –db PZ_Master1 –show wbia –tf draw_rank_surface –show -t CircQRH_K -a varysize_td –db PZ_Master1 –show

wbia –tf draw_rank_surface –show -t candidacy_k -a varysize –db PZ_Master1 –show –param-keys=K,dcfg_sample_per_name,dcfg_sample_size wbia –tf draw_rank_surface –show -t best

-a varynannots_td varynannots_td:qmin_pername=3,dpername=2 –db PZ_Master1 –show –param-keys=dcfg_sample_per_name,dcfg_sample_size

wbia –tf draw_rank_surface –show -t best -a varynannots_td –db PZ_Master1 –show –param-keys=dcfg_sample_size

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST')
>>> result = draw_rank_surface(ibs, testres)
>>> ut.show_if_requested()
>>> print(result)

wbia.expt.experiment_drawing.scorediff(ibs, testres, f=None, verbose=None)

Parameters:

• ibs (wbia.IBEISController) – image analysis api
• testres (wbia.TestResult) – test result object
• f (None) – (default = None)
• verbose (bool) – verbosity flag(default = None)

CommandLine:

python -m wbia.expt.experiment_drawing scorediff –db PZ_Master1 -a timectrl -t best –show python -m wbia.expt.experiment_drawing scorediff –db PZ_MTEST -a default -t best –show

python -m wbia.expt.experiment_drawing scorediff –db humpbacks_fb

-a default:has_any=hasnotch,mingt=2 -t default:proot=BC_DTW,decision=max,crop_dim_size=500,crop_enabled=True,use_te_scorer=False,manual_extract=True,ignore_notch=True,te_net=annot_simple –show

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_drawing import *  # NOQA
>>> from wbia.init import main_helpers
>>> defaultdb = 'PZ_MTEST'
>>> ibs, testres = main_helpers.testdata_expts(defaultdb, a=['timectrl'], t=['best'])
>>> f = ut.get_argval(('--filt', '-f'), type_=list, default=[''])
>>> scorediff(ibs, testres, f=f, verbose=ut.VERBOSE)
>>> ut.show_if_requested()

wbia.expt.experiment_drawing.temp_multidb_cmc()

Plots multiple database CMC curves in the same plot for the AI for social good paper

wbia.expt.experiment_drawing.temp_num_exmaples_cmc()

wbia.expt.experiment_helpers module

Helper module that helps expand parameters for grid search TODO: move into custom pipe_cfg and annot_cfg modules

wbia.expt.experiment_helpers.filter_duplicate_acfgs(expanded_aids_list, acfg_list, acfg_name_list, verbose=None)

Removes configs with the same expanded aids list

CommandLine:

# The following will trigger this function: wbia -m wbia get_annotcfg_list:0 -a timectrl timectrl:view=left –db PZ_MTEST

wbia.expt.experiment_helpers.get_annotcfg_list(ibs, acfg_name_list, filter_dups=True, qaid_override=None, daid_override=None, initial_aids=None, use_cache=None, verbose=None)

For now can only specify one acfg name list

TODO: move to filter_annots

Parameters:annot_cfg_name_list (list) –

CommandLine:

python -m wbia get_annotcfg_list:0 python -m wbia get_annotcfg_list:1 python -m wbia get_annotcfg_list:2

wbia get_annotcfg_list:0 –ainfo wbia get_annotcfg_list:0 –db NNP_Master3 -a viewpoint_compare –nocache-aid –verbtd wbia get_annotcfg_list:0 –db PZ_ViewPoints -a viewpoint_compare –nocache-aid –verbtd wbia get_annotcfg_list:0 –db PZ_MTEST -a unctrl ctrl::unctrl –ainfo –nocache-aid wbia get_annotcfg_list:0 –db testdb1 -a : –ainfo –nocache-aid wbia get_annotcfg_list:0 –db Oxford -a :qhas_any=query –ainfo –nocache-aid wbia get_annotcfg_list:0 –db Oxford -a :qhas_any=query,dhas_any=distractor –ainfo –nocache-aid

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_helpers import *  # NOQA
>>> import wbia
>>> from wbia.expt import annotation_configs
>>> ibs = wbia.opendb(defaultdb='PZ_MTEST')
>>> filter_dups = not ut.get_argflag('--nofilter-dups')
>>> acfg_name_list = testdata_acfg_names()
>>> _tup = get_annotcfg_list(ibs, acfg_name_list, filter_dups)
>>> acfg_list, expanded_aids_list = _tup
>>> print('\n PRINTING TEST RESULTS')
>>> result = ut.repr2(acfg_list, nl=3)
>>> print('\n')
>>> #statskw = ut.parse_func_kwarg_keys(ibs.get_annot_stats_dict, with_vals=False)
>>> printkw = dict(combined=True, per_name_vpedge=None,
>>>                per_qual=False, per_vp=False, case_tag_hist=False)
>>> annotation_configs.print_acfg_list(
>>>     acfg_list, expanded_aids_list, ibs, **printkw)

Example

>>> # ENABLE_DOCTEST
>>> from wbia.expt.experiment_helpers import *  # NOQA
>>> import wbia
>>> from wbia.init import main_helpers
>>> from wbia.expt import annotation_configs
>>> ibs = wbia.opendb(defaultdb='PZ_MTEST')
>>> aids = ibs.get_valid_aids()
>>> main_helpers.monkeypatch_encounters(ibs, aids, days=50)
>>> a = ['default:crossval_enc=True,require_timestamp=True']
>>> acfg_name_list = testdata_acfg_names(a)
>>> acfg_list, expanded_aids_list = get_annotcfg_list(ibs, acfg_name_list)
>>> annotation_configs.print_acfg_list(acfg_list, expanded_aids_list)
>>> # Restore state
>>> main_helpers.unmonkeypatch_encounters(ibs)

wbia.expt.experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=None, verbose=None)

Builds a list of varied query configurations. Only custom configs depend on an ibs object. The order of the output is not gaurenteed to aggree with input order.

FIXME:

This breaks if you proot=BC_DTW and ibs is None

Parameters:

• test_cfg_name_list (list) – list of strs
• ibs (wbia.IBEISController) – wbia controller object (optional)

Returns:

(cfg_list, cfgx2_lbl) -

cfg_list (list): list of config objects cfgx2_lbl (list): denotes which parameters are being varied.

If there is just one config then nothing is varied

Return type:

tuple

CommandLine:

python -m wbia get_pipecfg_list:0 python -m wbia get_pipecfg_list:1 –db humpbacks python -m wbia get_pipecfg_list:2

Example

>>> # ENABLE_DOCTEST
>>> from wbia.expt.experiment_helpers import *  # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='testdb1')
>>> #test_cfg_name_list = ['best', 'custom', 'custom:sv_on=False']
>>> #test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> test_cfg_name_list = ['default', 'default:sv_on=False', 'best']
>>> # execute function
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> # verify results
>>> assert pipecfg_list[0].sv_cfg.sv_on is True
>>> assert pipecfg_list[1].sv_cfg.sv_on is False
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.repr2(pipecfg_lbls))
>>> print(result)
pipecfg_lbls = ['default:', 'default:sv_on=False']

Example

>>> # DISABLE_DOCTEST
>>> import wbia_flukematch.plugin
>>> from wbia.expt.experiment_helpers import *  # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='humpbacks')
>>> test_cfg_name_list = ['default:pipeline_root=BC_DTW,decision=average,crop_dim_size=[960,500]', 'default:K=[1,4]']
>>> (pcfgdict_list, pipecfg_list) = get_pipecfg_list(test_cfg_name_list, ibs)
>>> pipecfg_lbls = get_varied_pipecfg_lbls(pcfgdict_list)
>>> result = ('pipecfg_lbls = '+ ut.repr2(pipecfg_lbls))
>>> print(result)
>>> print_pipe_configs(pcfgdict_list, pipecfg_list)

wbia.expt.experiment_helpers.get_varied_pipecfg_lbls(cfgdict_list, pipecfg_list=None)

wbia.expt.experiment_helpers.parse_acfg_combo_list(acfg_name_list)

Parses the name list into a list of config dicts

Parameters:acfg_name_list (list) – a list of annotation config strings Returns:acfg_combo_list Return type:list

CommandLine:

python -m wbia parse_acfg_combo_list:0 python -m wbia parse_acfg_combo_list:1 python -m wbia parse_acfg_combo_list:2

Example

>>> # ENABLE_DOCTEST
>>> from wbia.expt.experiment_helpers import *  # NOQA
>>> import wbia
>>> from wbia.expt import annotation_configs
>>> acfg_name_list = testdata_acfg_names(['default', 'uncontrolled'])
>>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list)
>>> acfg_list = ut.flatten(acfg_combo_list)
>>> printkw = dict()
>>> annotation_configs.print_acfg_list(acfg_list, **printkw)
>>> result = ut.repr2(sorted(acfg_list[0].keys()))
>>> print(result)
['dcfg', 'qcfg']

Example

>>> # ENABLE_DOCTEST
>>> from wbia.expt.experiment_helpers import *  # NOQA
>>> import wbia
>>> from wbia.expt import annotation_configs
>>> # double colon :: means expand consistently and force const size
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'ctrl::unctrl'])
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::unctrl'])
>>> acfg_name_list = testdata_acfg_names(['unctrl', 'varysize', 'ctrl::varysize', 'ctrl::unctrl'])
>>> acfg_combo_list = parse_acfg_combo_list(acfg_name_list)
>>> acfg_list = ut.flatten(acfg_combo_list)
>>> printkw = dict()
>>> annotation_configs.print_acfg_list(acfg_list, **printkw)

wbia.expt.experiment_helpers.print_pipe_configs(cfgdict_list, pipecfg_list)

wbia.expt.experiment_helpers.testdata_acfg_names(default_acfg_name_list=['default'])

wbia.expt.experiment_printres module

displays results from harness

TODO: save a testres variable so reloading and regenration becomes easier.

wbia.expt.experiment_printres.get_diffmat_str(rank_mat, qaids, nConfig)

wbia.expt.experiment_printres.get_diffranks(rank_mat, qaids)

Find rows which scored differently over the various configs FIXME: duplicated

wbia.expt.experiment_printres.print_latexsum(ibs, testres, verbose=True)

Parameters:

• ibs (IBEISController) – wbia controller object
• testres –

CommandLine:

python -m wbia.expt.experiment_printres –exec-print_latexsum python -m wbia.scripts.gen_cand_expts –exec-gen_script

python -m wbia –tf print_latexsum -t candidacy –db PZ_Master0 -a controlled –rank-lt-list=1,5,10,100 python -m wbia –tf print_latexsum -t candidacy –db PZ_MTEST -a controlled –rank-lt-list=1,5,10,100

Example

>>> # SCRIPT
>>> from wbia.expt.experiment_printres import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts()
>>> tabular_str2 = print_latexsum(ibs, testres)

wbia.expt.experiment_printres.print_results(ibs, testres, **kwargs)

Prints results from an experiment harness run. Rows store different qaids (query annotation ids) Cols store different configurations (algorithm parameters)

TODO: join acfgs

Parameters:

• ibs (IBEISController) – wbia controller object
• testres (test_result.TestResult) –

CommandLine:

python dev.py -e print –db PZ_MTEST

-a default:dpername=1,qpername=[1,2] -t default:fg_on=False

python dev.py -e print -t best –db seals2 –allgt –vz python dev.py -e print –db PZ_MTEST –allgt -t custom

–print-confusion-stats

python dev.py -e print –db PZ_MTEST –allgt –noqcache

–index 0:10:2 -t custom:rrvsone_on=True –print-confusion-stats

python dev.py -e print –db PZ_MTEST –allgt –noqcache –qaid4

-t custom:rrvsone_on=True –print-confusion-stats

python -m wbia print_results -t default –db PZ_MTEST -a ctrl python -m wbia print_results -t default –db PZ_MTEST -a ctrl python -m wbia print_results –db PZ_MTEST -a default

-t default:lnbnn_on=True default:lnbnn_on=False,bar_l2_on=True default:lnbnn_on=False,normonly_on=True

CommandLine:

python -m wbia.expt.experiment_printres –test-print_results utprof.py -m wbia.expt.experiment_printres –test-print_results

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.experiment_printres import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts(
>>>     'pz_mtest', a='default:dpername=1,qpername=[1,2]',
>>>     t='default:fg_on=false')
>>> result = print_results(ibs, testres)
>>> print(result)

wbia.expt.experiment_printres.rankscore_str(thresh, nLess, total, withlbl=True)

wbia.expt.harness module

Runs many queries and keeps track of some results

wbia.expt.harness.make_single_testres(ibs, qaids, daids, pipecfg_list, cfgx2_lbl, cfgdict_list, lbl, testnameid, use_cache=None, subindexer_partial=<class 'utool.util_progress.ProgIter'>)

CommandLine:

python -m wbia run_expt

wbia.expt.harness.run_expt(ibs, acfg_name_list, test_cfg_name_list, use_cache=None, qaid_override=None, daid_override=None, initial_aids=None)

Loops over annot configs.

Try and use this function as a starting point to clean up this module. The code is getting too untenable.

CommandLine:

python -m wbia.expt.harness run_expt –acfginfo python -m wbia.expt.harness run_expt –pcfginfo python -m wbia.expt.harness run_expt

Ignore:

test_cfg_name_list = [p]

Example

>>> # SLOW_DOCTEST
>>> from wbia.expt.harness import *  # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='PZ_MTEST')
>>> default_acfgstrs = ['ctrl:qsize=20,dpername=1,dsize=10',
>>>                     'ctrl:qsize=20,dpername=10,dsize=20']
>>> acfg_name_list = default_acfgstrs
>>> test_cfg_name_list = ['default:proot=smk', 'default']
>>> #test_cfg_name_list = ['custom', 'custom:fg_on=False']
>>> use_cache = False
>>> testres_list = run_expt(ibs, acfg_name_list, test_cfg_name_list, use_cache)

wbia.expt.old_storage module

wbia.expt.test_result module

class wbia.expt.test_result.TestResult(cfg_list, cfgx2_lbl, cfgx2_cmsinfo, cfgx2_qreq_)

Bases: utool.util_dev.NiceRepr

CommandLine:

export SMK_PIPE=”smk:nwords=[64000],sv=[False]” wbia TestResult –db PZ_MTEST -a ctrl -p $SMK_PIPE wbia TestResult –db Oxford -a oxford -p $SMK_PIPE

Example

>>> # Script
>>> from wbia.init import main_helpers
>>> import utool as ut
>>> ibs, testres = main_helpers.testdata_expts()
>>> testres.help()
>>> actions = testres.get_actions()
>>> testres.map_score()
>>> ut.qtensure()
>>> prompt = ut.InteractivePrompt(actions)
>>> prompt.loop()

case_sample2(filt_cfg, qaids=None, return_mask=False, verbose=None)

Filters individual test result cases based on how they performed, what tags they had, and various other things.

Parameters:filt_cfg (dict) – Returns:case_pos_list (list of (qx, cfgx)) or isvalid mask Return type:list

CommandLine:

python -m wbia TestResult.case_sample2 python -m wbia TestResult.case_sample2:0 python -m wbia TestResult.case_sample2:1 –db GZ_ALL –filt :min_tags=1 python -m wbia TestResult.case_sample2:1 –db PZ_Master1 –filt :min_gf_tags=1

python -m wbia TestResult.case_sample2:2 –db PZ_Master1

Example

>>> # DISABLE_DOCTEST
>>> # The same results is achievable with different filter config settings
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> verbose = True
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])
>>> filt_cfg1 = {'fail': True}
>>> case_pos_list1 = testres.case_sample2(filt_cfg1)
>>> filt_cfg2 = {'min_gtrank': 1}
>>> case_pos_list2 = testres.case_sample2(filt_cfg2)
>>> filt_cfg3 = {'min_gtrank': 0}
>>> case_pos_list3 = testres.case_sample2(filt_cfg3)
>>> filt_cfg4 = {}
>>> case_pos_list4 = testres.case_sample2(filt_cfg4)
>>> assert np.all(case_pos_list1 == case_pos_list2), 'should be equiv configs'
>>> assert np.any(case_pos_list2 != case_pos_list3), 'should be diff configs'
>>> assert np.all(case_pos_list3 == case_pos_list4), 'should be equiv configs'
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:sv_on=[True,False]'])
>>> filt_cfg5 = filt_cfg1.copy()
>>> mask5 = testres.case_sample2(filt_cfg5, return_mask=True)
>>> case_pos_list5 = testres.case_sample2(filt_cfg5, return_mask=False)
>>> assert len(mask5.shape) == 2
>>> assert np.all(mask5.T[0] == mask5.T[1])
>>> filt_cfg6 = {'fail': True, 'allcfg': True}
>>> mask6 = testres.case_sample2(filt_cfg6, return_mask=True)
>>> assert np.all(mask6.T[0] == mask6.T[1])
>>> print(mask5)
>>> print(case_pos_list5)
>>> filt_cfg = filt_cfg7 = {'disagree': True}
>>> case_pos_list7 = testres.case_sample2(filt_cfg7, verbose=verbose)
>>> print(case_pos_list7)

Example

>>> # SCRIPT
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])
>>> filt_cfg = main_helpers.testdata_filtcfg()
>>> case_pos_list = testres.case_sample2(filt_cfg)
>>> result = ('case_pos_list = %s' % (str(case_pos_list),))
>>> print(result)
>>> # Extra stuff
>>> all_tags = testres.get_all_tags()
>>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])
>>> print('selcted_tags = %r' % (selcted_tags,))

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:K=[1,2,3]'])
>>> ut.exec_funckw(testres.case_sample2, globals())
>>> filt_cfg = {'fail': True, 'min_gtrank': 1, 'max_gtrank': None, 'min_gf_timedelta': '24h'}
>>> ibs, testres = main_helpers.testdata_expts('humpbacks_fb', a=['default:has_any=hasnotch,mingt=2,qindex=0:300,dindex=0:300'], t=['default:proot=BC_DTW,decision=max,crop_dim_size=500,crop_enabled=True,manual_extract=False,use_te_scorer=True,ignore_notch=True,te_net=annot_simple', 'default:proot=vsmany'], qaid_override=[12])
>>> filt_cfg = ':disagree=True,index=0:8,min_gtscore=.00001,require_all_cfg=True'
>>> #filt_cfg = cfghelpers.parse_argv_cfg('--filt')[0]
>>> case_pos_list = testres.case_sample2(filt_cfg, verbose=True)
>>> result = ('case_pos_list = %s' % (str(case_pos_list),))
>>> print(result)
>>> # Extra stuff
>>> all_tags = testres.get_all_tags()
>>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])
>>> print('selcted_tags = %r' % (selcted_tags,))

logger.info(‘qaid = %r’ % (qaid,)) logger.info(‘qx = %r’ % (qx,)) logger.info(‘cfgxs = %r’ % (cfgxs,)) # print testres info about this item take_cfgs = ut.partial(ut.take, index_list=cfgxs) take_qx = ut.partial(ut.take, index_list=qx) truth_cfgs = ut.hmap_vals(take_qx, truth2_prop) truth_item = ut.hmap_vals(take_cfgs, truth_cfgs, max_depth=1) prop_cfgs = ut.hmap_vals(take_qx, prop2_mat) prop_item = ut.hmap_vals(take_cfgs, prop_cfgs, max_depth=0) logger.info(‘truth2_prop[item] = ‘ + ut.repr3(truth_item, nl=2)) logger.info(‘prop2_mat[item] = ‘ + ut.repr3(prop_item, nl=1))

cfgx2_daids

cfgx2_qaids

draw_failure_cases(**kwargs)

>>> from wbia.other.dbinfo import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts(defaultdb='PZ_MTEST', a='timectrl:qsize=2', t='invar:AI=[False],RI=False', use_cache=False)

draw_match_cases(**kwargs)

Wrapper

draw_rank_cmc()

Wrapper

draw_score_diff_disti()

CommandLine:

python -m wbia –tf TestResult.draw_score_diff_disti –show -a varynannots_td -t best –db PZ_Master1 python -m wbia –tf TestResult.draw_score_diff_disti –show -a varynannots_td -t best –db GZ_Master1 python -m wbia –tf TestResult.draw_score_diff_disti –show -a varynannots_td1h -t best –db GIRM_Master1

python -m wbia –tf TestResult.draw_score_diff_disti –show -a varynannots_td:qmin_pername=3,dpername=2 -t best –db PZ_Master1

python -m wbia –tf get_annotcfg_list -a varynannots_td -t best –db PZ_Master1 13502 python -m wbia –tf draw_match_cases –db PZ_Master1 -a varynannots_td:dsample_size=.01 -t best –show –qaid 13502 python -m wbia –tf draw_match_cases –db PZ_Master1 -a varynannots_td -t best –show

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('PZ_Master1', a=['varynannots_td'], t=['best'])
>>> result = testres.draw_score_diff_disti()
>>> print(result)
>>> ut.show_if_requested()

embed_testres()

CommandLine:

python -m wbia TestResults.embed_testres

Example

>>> # SCRIPT
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts(defaultdb='PZ_MTEST')
>>> embed_testres(testres)

find_score_thresh_cutoff()

FIXME DUPLICATE CODE rectify with experiment_drawing

classmethod from_cms(cm_list, qreq_)

get_X_LIST()

DEPRICATE or refactor

get_actions()

get_all_qaids()

get_all_tags()

CommandLine:

python -m wbia –tf TestResult.get_all_tags –db PZ_Master1 –show –filt : python -m wbia –tf TestResult.get_all_tags –db PZ_Master1 –show –filt :min_gf_timedelta=24h python -m wbia –tf TestResult.get_all_tags –db PZ_Master1 –show –filt :min_gf_timedelta=24h,max_gt_rank=5

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_Master1', a=['timectrl'])
>>> filt_cfg = main_helpers.testdata_filtcfg()
>>> case_pos_list = testres.case_sample2(filt_cfg)
>>> all_tags = testres.get_all_tags()
>>> selected_tags = ut.take(all_tags, case_pos_list.T[0])
>>> flat_tags = list(map(str, ut.flatten(ut.flatten(selected_tags))))
>>> print(ut.repr2(ut.dict_hist(flat_tags), key_order_metric='val'))
>>> ut.quit_if_noshow()
>>> import wbia.plottool as pt
>>> pt.word_histogram2(flat_tags, fnum=1, pnum=(1, 2, 1))
>>> pt.wordcloud(' '.join(flat_tags), fnum=1, pnum=(1, 2, 2))
>>> pt.set_figtitle(ut.get_cfg_lbl(filt_cfg))
>>> ut.show_if_requested()

get_all_varied_params()

Returns the parameters that were varied between different configurations in this test

Returns:varied_params Return type:list

CommandLine:

python -m wbia TestResult.get_all_varied_params

Example

>>> # ENABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> testres = wbia.testdata_expts(
>>>     'PZ_MTEST', t='default:K=[1,2]')[1]
>>> varied_params = sorted(testres.get_all_varied_params())
>>> result = ('varied_params = %s' % (ut.repr2(varied_params),))
>>> print(result)
varied_params = ['K', '_cfgindex']

get_annotcfg_args()

CommandLine:

# TODO: More robust fix # To reproduce the error wbia -e rank_cmc –db humpbacks_fb -a default:mingt=2,qsize=10,dsize=100 default:qmingt=2,qsize=10,dsize=100 -t default:proot=BC_DTW,decision=max,crop_dim_size=500,crop_enabled=True,manual_extract=False,use_te_scorer=True,ignore_notch=True,te_score_weight=0.5 –show

get_cfgstr(cfgx)

just dannots and config_str

get_cfgx_groupxs()

Returns the group indices of configurations specified to be joined.

Ignore:

a = [ ‘default:minqual=good,require_timestamp=True,view=left,crossval_enc=True,joinme=1’, ‘default:minqual=good,require_timestamp=True,view=right,crossval_enc=True,joinme=1’, ‘default:minqual=ok,require_timestamp=True,view=left,crossval_enc=True,joinme=2’, ‘default:minqual=ok,require_timestamp=True,view=right,crossval_enc=True,joinme=2’, ] >>> a = [ >>> ‘default:minqual=good,require_timestamp=True,view=left,crossval_enc=True,joinme=1’, >>> ‘default:minqual=good,require_timestamp=True,view=right,crossval_enc=True,joinme=1’, >>> ‘default:minqual=ok,require_timestamp=True,view=left,crossval_enc=True,joinme=2’, >>> ‘default:minqual=ok,require_timestamp=True,view=right,crossval_enc=True,joinme=2’, >>> ] >>> from wbia.init import main_helpers >>> #a = ‘default:minqual=good,require_timestamp=True,crossval_enc=True,view=[right,left]’ >>> t = ‘default:K=[1]’ >>> ibs, testres = main_helpers.testdata_expts(‘WWF_Lynx_Copy’, a=a, t=t) >>> testres.get_cfgx_groupxs()

ut.lmap(sum, ut.apply_grouping([len(ut.unique(ibs.annots(aids).nids)) for aids in testres.cfgx2_qaids], testres.get_cfgx_groupxs())) ut.lmap(sum, ut.apply_grouping([len(ut.unique(ibs.annots(aids))) for aids in testres.cfgx2_qaids], testres.get_cfgx_groupxs()))

Example

>>> # xdoctest: +REQUIRES(--slow)
>>> # ENABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts(
>>>    'PZ_MTEST',
>>>     a=['default:qnum_names=1,qname_offset=[0,1],joinme=1,dpername=1',
>>>        'default:qsize=1,dpername=[1,2]'],
>>>     t=['default:K=[1,2]'])
>>> groupxs = testres.get_cfgx_groupxs()
>>> result = groupxs
>>> print(result)
[[6], [4], [0, 2], [7], [5], [1, 3]]

get_cfgx_with_param(key, val)

Gets configs where the given parameter is held constant

get_common_qaids()

get_fname_aug(**kwargs)

get_full_cfgstr(cfgx)

both qannots and dannots included

get_gf_tags()

Returns:case_pos_list Return type:list

CommandLine:

python -m wbia –tf TestResult.get_gf_tags –db PZ_Master1 –show

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('PZ_Master1', a=['timectrl'])
>>> filt_cfg = main_helpers.testdata_filtcfg()
>>> case_pos_list = testres.case_sample2(filt_cfg)
>>> gf_tags = testres.get_gf_tags()

get_gt_annot_tags()

get_gt_tags()

get_gtquery_annot_tags()

get_infoprop_list(key, qaids=None)

key = ‘qx2_gt_rank’ key = ‘qx2_gt_rank’ qaids = testres.get_test_qaids()

get_infoprop_mat(key, qaids=None)

key = ‘qx2_gf_raw_score’ key = ‘qx2_gt_raw_score’

get_nLessX_dict()

Build a (histogram) dictionary mapping X (as in #ranks < X) to a list of cfg scores

get_options()

get_param_basis(key)

Returns what a param was varied between over all tests key = ‘K’ key = ‘dcfg_sample_size’

get_param_val_from_cfgx(cfgx, key)

get_pipecfg_args()

get_query_annot_tags()

get_rank_histogram_bins()

easy to see histogram bins

get_rank_histograms(bins=None, key=None, join_acfgs=False)

Ignore:

testres.get_infoprop_mat(‘qnx2_gt_name_rank’) testres.get_infoprop_mat(‘qnx2_gf_name_rank’) testres.get_infoprop_mat(‘qnx2_qnid’)

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts('testdb1', a=['default'])
>>> bins = 'dense'
>>> key = 'qnx2_gt_name_rank'
>>> config_hists = testres.get_rank_histograms(bins, key=key)

get_rank_mat(qaids=None)

get_rank_percentage_cumhist(bins='dense', key=None, join_acfgs=False)

Parameters:

• bins (unicode) – (default = u’dense’)
• key (None) – (default = None)
• join_acfgs (bool) – (default = False)

Returns:

(config_cdfs, edges)

Return type:

tuple

CommandLine:

python -m wbia –tf TestResult.get_rank_percentage_cumhist python -m wbia –tf TestResult.get_rank_percentage_cumhist

-t baseline -a unctrl ctrl

python -m wbia –tf TestResult.get_rank_percentage_cumhist

–db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 –show –cmd

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts(
>>>     'testdb1', a=['default:num_names=1,name_offset=[0,1]'])
>>> bins = u'dense'
>>> key = None
>>> (config_cdfs, edges) = testres.get_rank_percentage_cumhist(bins)
>>> result = ('(config_cdfs, edges) = %s' % (str((config_cdfs, edges)),))
>>> print(result)

get_short_cfglbls(join_acfgs=False)

Labels for published tables

cfg_lbls = [‘baseline:nRR=200+default:’, ‘baseline:+default:’]

CommandLine:

python -m wbia –tf TestResult.get_short_cfglbls

Example

>>> # SLOW_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('PZ_MTEST', a=['ctrl:size=10'],
>>>                                     t=['default:dim_size=[450,550]'])
>>> cfg_lbls = testres.get_short_cfglbls()
>>> result = ('cfg_lbls = %s' % (ut.repr2(cfg_lbls),))
>>> print(result)
cfg_lbls = [
    'default:dim_size=450+ctrl',
    'default:dim_size=550+ctrl',
]

get_sorted_config_labels()

helper

get_test_qaids()

get_title_aug(with_size=True, with_db=True, with_cfg=True, friendly=False)

Parameters:with_size (bool) – (default = True) Returns:title_aug Return type:str

CommandLine:

python -m wbia –tf TestResult.get_title_aug –db PZ_Master1 -a timequalctrl::timectrl

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('PZ_MTEST')
>>> with_size = True
>>> title_aug = testres.get_title_aug(with_size)
>>> res = u'title_aug = %s' % (title_aug,)
>>> print(res)

get_total_num_varied_params()

get_truth2_prop(qaids=None, join_acfg=False)

Returns:(truth2_prop, prop2_mat) Return type:tuple

CommandLine:

python -m wbia.expt.test_result –exec-get_truth2_prop –show

Example

>>> # xdoctest: +REQUIRES(--slow)
>>> # ENABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('PZ_MTEST', a=['ctrl'])
>>> (truth2_prop, prop2_mat) = testres.get_truth2_prop()
>>> result = '(truth2_prop, prop2_mat) = %s' % str((truth2_prop, prop2_mat))
>>> print(result)
>>> ut.quit_if_noshow()
>>> import wbia.plottool as pt
>>> ut.show_if_requested()

get_varied_labels(shorten=False, join_acfgs=False, sep='')

Returns labels indicating only the parameters that have been varied between different annot/pipeline configurations.

Helper for consistent figure titles

CommandLine:

python -m wbia –tf TestResult.make_figtitle –prefix “Seperability ” –db GIRM_Master1 -a timectrl -t Ell:K=2 –hargv=scores python -m wbia –tf TestResult.make_figtitle python -m wbia TestResult.get_varied_labels

Example

>>> # SLOW_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts(
>>>     'PZ_MTEST', t='default:K=[1,2]',
>>>     #a=['timectrl:qsize=[1,2],dsize=[3,4]']
>>>     a=[
>>>        'default:qsize=[1,2],dsize=2,joinme=1,view=left',
>>>        'default:qsize=2,dsize=3,joinme=1,view=primary',
>>>        'default:qsize=[3,2],dsize=4,joinme=2,view=left',
>>>        'default:qsize=4,dsize=5,joinme=2,view=primary',
>>>       ]
>>> )
>>> # >>> ibs, testres = wbia.testdata_expts(
>>> # >>>     'WWF_Lynx_Copy', t='default:K=1',
>>> # >>>     a=[
>>> # >>>         'default:minqual=good,require_timestamp=True,view=left,dcrossval_enc=1,joinme=1',
>>> # >>>         'default:minqual=good,require_timestamp=True,view=left,dcrossval_enc=2,joinme=2',
>>> # >>>         #'default:minqual=good,require_timestamp=True,view=left,dcrossval_enc=3,joinme=3',
>>> # >>>         'default:minqual=good,require_timestamp=True,view=right,dcrossval_enc=1,joinme=1',
>>> # >>>         'default:minqual=good,require_timestamp=True,view=right,dcrossval_enc=2,joinme=2',
>>> # >>>         #'default:minqual=good,require_timestamp=True,view=right,dcrossval_enc=3,joinme=3',
>>> # >>>       ]
>>> # >>> )
>>> varied_lbls = testres.get_varied_labels(shorten=False, join_acfgs=True)
>>> result = ('varied_lbls = %s' % (ut.repr2(varied_lbls, strvals=True, nl=2),))
>>> print(result)

varied_lbls = [u’K=1+qsize=1’, u’K=2+qsize=1’, u’K=1+qsize=2’, u’K=2+qsize=2’]

get_worst_possible_rank()

has_constant_daids()

has_constant_length_daids()

has_constant_length_qaids()

has_constant_qaids()

help()

ibs

interact_individual_result(qaid, cfgx=0)

make_figtitle(plotname='', filt_cfg=None)

Helper for consistent figure titles

CommandLine:

python -m wbia –tf TestResult.make_figtitle –prefix “Seperability ” –db GIRM_Master1 -a timectrl -t Ell:K=2 –hargv=scores python -m wbia –tf TestResult.make_figtitle

Example

>>> # ENABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('PZ_MTEST')
>>> plotname = ''
>>> figtitle = testres.make_figtitle(plotname)
>>> result = ('figtitle = %r' % (figtitle,))
>>> print(result)

map_score()

For each query compute a precision recall curve. Then, for each query compute the average precision. Then take the mean of all average precisions to obtain the mAP.

Script:

>>> #ibs = wbia.opendb('Oxford')
>>> #ibs, testres = wbia.testdata_expts('Oxford', a='oxford', p='smk:nWords=[64000],nAssign=[1],SV=[False,True]')
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('Oxford', a='oxford', p='smk:nWords=[64000],nAssign=[1],SV=[False,True],can_match_sameimg=True')
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('Oxford', a='oxford', p='smk:nWords=[64000],nAssign=[1],SV=[False],can_match_sameimg=True')

nConfig

nQuery

print_acfg_info(**kwargs)

Prints verbose information about the annotations used in each test configuration

CommandLine:

python -m wbia –tf TestResult.print_acfg_info

Kwargs:

see ibs.get_annot_stats_dict hashid, per_name, per_qual, per_vp, per_name_vpedge, per_image, min_name_hourdist

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> ibs, testres = wbia.testdata_expts('PZ_MTEST',
>>>                                     a=['ctrl::unctrl_comp'],
>>>                                     t=['candk:K=[1,2]'])
>>> ibs = None
>>> result = testres.print_acfg_info()
>>> print(result)

print_config_overlap(with_plot=True)

print_pcfg_info()

Prints verbose information about each pipeline configuration

>>> from wbia.expt.test_result import *  # NOQA

print_percent_identification_success()

Prints names identified (at rank 1) / names queried. This combines results over multiple queries of a particular name using max

OLD, MAYBE DEPRIATE

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA

print_results(**kwargs)

CommandLine:

python -m wbia –tf TestResult.print_results

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.expt import harness
>>> ibs, testres = harness.testdata_expts('PZ_MTEST')
>>> result = testres.print_results()
>>> print(result)

print_unique_annot_config_stats(ibs=None)

Parameters:ibs (IBEISController) – wbia controller object(default = None)

CommandLine:

python -m wbia TestResult.print_unique_annot_config_stats

Example

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> testres = wbia.testdata_expts('PZ_MTEST', a=['ctrl::unctrl_comp'])
>>> ibs = None
>>> result = testres.print_unique_annot_config_stats(ibs)
>>> print(result)

qaids

rank_mat

reconstruct_test_flags()

report()

rrr(verbose=True, reload_module=True)

special class reloading function This function is often injected as rrr of classes

unique_pcfgs

wbia.expt.test_result.build_cmsinfo(cm_list, qreq_)

Helper function to report results over multiple queries (chip matches). Basically given a group of queries of the same name, we only care if one of them is correct. This emulates encounters.

Runs queries of a specific configuration returns the best rank of each query.

Parameters:

• cm_list (list) – list of chip matches
• qreq (QueryRequest) – request that computed the chip matches.

Returns:

cmsinfo - info about multiple chip matches cm_list

Return type:

dict

CommandLine:

python -m wbia get_query_result_info python -m wbia get_query_result_info:0 –db lynx

-a :qsame_imageset=True,been_adjusted=True,excluderef=True -t :K=1

python -m wbia get_query_result_info:0 –db lynx

-a :qsame_imageset=True,been_adjusted=True,excluderef=True -t :K=1 –cmd

Example

>>> # ENABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> import wbia
>>> qreq_ = wbia.main_helpers.testdata_qreq_(a=[':qindex=0:3,dindex=0:5'])
>>> cm_list = qreq_.execute()
>>> cmsinfo = build_cmsinfo(cm_list, qreq_)
>>> print(ut.repr2(cmsinfo))

Ignore:

wbia -e rank_cmc –db humpbacks -a :has_any=hasnotch,mingt=2

-t :proot=BC_DTW –show –nocache-big

wbia -e rank_cmc –db humpbacks -a :is_known=True,mingt=2

-t :pipeline_root=BC_DTW

wbia -e rank_cmc –db humpbacks -a :is_known=True

-t :pipeline_root=BC_DTW –qaid=1,9,15,16,18 –daid-override=1,9,15,16,18,21,22 –show –debug-depc

–clear-all-depcache

wbia.expt.test_result.combine_testres_list(ibs, testres_list)

combine test results over multiple annot configs

The combination of pipeline and annotation config is indexed by cfgx. A cfgx corresponds to a unique query request

CommandLine:

python -m wbia –tf combine_testres_list

python -m wbia –tf -draw_rank_cmc –db PZ_MTEST –show python -m wbia –tf -draw_rank_cmc –db PZ_Master1 –show python -m wbia –tf -draw_rank_cmc –db PZ_MTEST –show -a varysize -t default python -m wbia –tf -draw_rank_cmc –db PZ_MTEST –show -a varysize -t default

>>> # DISABLE_DOCTEST
>>> from wbia.expt.test_result import *  # NOQA
>>> from wbia.expt import harness
>>> ibs, testres = harness.testdata_expts('PZ_MTEST', ['varysize'])

Module contents

wbia.expt.IMPORT_TUPLES = [('experiment_configs', None), ('harness', None), ('experiment_helpers', None), ('experiment_printres', None), ('results_all', None), ('results_analyzer', None), ('results_organizer', None)]

cd /home/joncrall/code/wbia/wbia.expt makeinit.py

Type:Regen Command

wbia.expt.reassign_submodule_attributes(verbose=True)

why reloading all the modules doesnt do this I don’t know

wbia.expt.reload_subs(verbose=True)

Reloads wbia.expt and submodules

wbia.expt.rrrr(verbose=True)

Reloads wbia.expt and submodules