Skip to content

14. Evaluation attributes in SDK

When run results are uploaded to Foretify Manager, root-level custom data keys become evaluation attributes in the database. For details on how to set custom data, see Custom data.

Reference evaluation attributes by their internal name — the database key — when filtering, sorting, or displaying run data through the Python SDK.

The clip_id and original_clipgt_url attributes are populated automatically during S3 ClipGT import. For details on S3 batch import, see [Batch import ClipGT files from S3 using the Python SDK][setting-up-s3].

The following example shows how to reference and use evaluation attributes by their internal database key names when filtering, sorting, and displaying run data through the Python SDK.

Python: evaluation attributes in SDK
from argparse import ArgumentParser
from ftx._common.utils import add_common_arguments

def parse():
    parser = ArgumentParser(description="Getting arguments")
    add_common_arguments(parser)
    parser.add_argument("--test_run", nargs="?", help="test run id")
    parser.add_argument("--workspace", nargs="?", help="workspace id")
    parser.add_argument("--triage_view", nargs="?", help="triage view id")
    return parser.parse_args()

args = parse()
globals().update(vars(args))
test_run_id = args.test_run
workspace_id = args.workspace
triage_view_id = args.triage_view

from ftx.shell import client, test_runs, workspaces, attributes
from ftx.model.search import Filter, FilterContainer

client.login(host=host, user=user, port=port, https=https)


# Any reference to evaluation attributes (UPLOAD) should be through their internal name,
# which is the key for the attribute value in the database

# Get all the 'UPLOAD' attributes (globally):
upload_attributes = [attr for attr in attributes.get_all() if attr.get('creationMethod') == 'UPLOAD']

# Display the internal name of each attribute:
for attr in upload_attributes:
    print(f"{attr.get('displayName')}: \t{attr.get('name')}")

# Filter and sort by evaluation attributes
order_by_custom = {"properties": ["evaluationAttributesNumeric.customDouble"], "direction": "DESC"}
custom_filter = FilterContainer.of(Filter.any("evaluationAttributes.customEnum EQ COMPLETED", "evaluationAttributesNumeric.customDuration EQ 10500.0"))
test_runs.page(filt=custom_filter, pagination={"page": 0, "size": 20}, orderBy=order_by_custom)['evaluationAttributesNumeric.customDouble']
test_runs.get(filt=custom_filter, orderBy=order_by_custom)['evaluationAttributesNumeric.customDouble']

# Display the custom attributes of a specific run
test_run_data = test_runs.get_by_id(test_run_id, True)

columns = ['evaluationAttributes.customString', 'evaluationAttributes.customLink', 'evaluationAttributes.customTimestamp', 'evaluationAttributes.customEnum', 'evaluationAttributesNumeric.customDouble']
for col in columns:
    print(f"{col}: {test_run_data.get('customAttributes').get(col)}")

# Filter workspace runs
custom_filter = Filter.all("evaluationAttributes.customEnum EQ COMPLETED", "evaluationAttributesNumeric.customDuration EQ 120500.0")
workspaces.filter_in(workspace_id, filt=custom_filter)
filtered_test_runs = test_runs.get_by_workspace_test_run_filter(workspace_id)

columns_to_keep = ['testName', 'seed', 'evaluationAttributes.customEnum', 'evaluationAttributesNumeric.customDuration']
filtered_test_runs[columns_to_keep]

# Display runs in triage view:
test_runs.get_runs_in_triage(triage_view_id)[columns_to_keep]