How to use the PigeonSuperModel DataSet

Download this notebook in the upper right corner as .ipynb file and start it from within your DeepLabCut environment. If you need to install DeepLabCut first, please check here.

This notebook will show you how to create a pretrained project with the PigeonSuperModel and how to download and merge the pre-labeled PigeonSuperModel dataset to increase your own training data:

  • Create a new PigeonSuperModel Project

  • Extract new Frames

  • Label your own Data

  • Merge PigeonSuperModel DataSet

  • Re-Train the PigeonSuperModel

  • Evaluate your new PigeonSuperModel

  • Contribute to

import os
import deeplabcut
from tkinter import filedialog

print(f'Using DeepLabCut version: {deeplabcut.__version__}')

Create a new project

See Notebook before

video_path = r'F:\PigeonSuperModel_local\ForAnalysisVideoData\StopSignal'
video_list = os.listdir(video_path)
projectpath = r'F:\PigeonSuperModel_local\DeepLabCutModels'
def monkeypatch():
    This function locates a file named 'pose_estimation_tensorflow/models/pretrained/pretrained_model_urls.yaml' in your local DeepLabCut
    installation and adds the new URLs below to reach the PigeonSUperModel repository.
    In a last step, the names of the PigeonSuperModels are added to the '' Modeloptions as valid model names.
    from deeplabcut.create_project import modelzoo

    # locate the `pretrained_model_urls.yaml` file in your local DeepLabCut installation (hardcoded based on deeplabcut dir structure)
    neturls_path = os.path.join(deeplabcut.auxiliaryfunctions.get_deeplabcut_path(), 'pose_estimation_tensorflow', 'models', 'pretrained', 'pretrained_model_urls.yaml')

    edits = {'PigeonSuperModel_resnet_50':  '',
             'PigeonSuperModel_resnet_101': '', 
             'PigeonSuperModel_resnet_152': ''}
    # add new model to the neturls file     
    deeplabcut.auxiliaryfunctions.edit_config(neturls_path, edits)

    # add new model to locally loaded Modeloptions from
    modelzoo.Modeloptions.extend(['PigeonSuperModel_resnet_50', 'PigeonSuperModel_resnet_101', 'PigeonSuperModel_resnet_152'])
# you may need admin rights on your computer to edit the files above
def create_dummy_videos(target_dir):
    import urllib.request
    import cv2

    # create dummy video
    logolink = ''
    logo = os.path.join(target_dir, 'logo.png')
    urllib.request.urlretrieve(logolink, logo);

    dummyvideo = os.path.join(target_dir, 'logo.mp4')
    img_array = []
    for i in range(20):
        img = cv2.imread(logo)
        height, width, layers = img.shape
    out = cv2.VideoWriter(dummyvideo, cv2.VideoWriter_fourcc(*'mp4v'), 20, (width, height))
    for i in range(len(img_array)):

    return dummyvideo

# create dummy video to use copy=True
video_path = create_dummy_videos(projectpath)
# create pre-trained model from PigeonSuperModel
config_path, _ = deeplabcut.create_pretrained_project(
sign = {'PigeonSuperModel': f'The pretrained PigeonSuperModel was added to this project. (C)'}
deeplabcut.auxiliaryfunctions.edit_config(config_path, sign)

Copying the videos may take very long, and creating hyperlinks does not always work on windows. Alternatively, we can add the video paths to the config.yaml file while leaving our video data on a separate disk.

# patch: add video links to config.yaml
def add_videopaths(config, video_list):
    from pathlib import Path
    from deeplabcut.utils.auxfun_videos import VideoReader

    videos = video_list
    original_video_sets = deeplabcut.auxiliaryfunctions.read_config(config)['video_sets']
    projectpath = deeplabcut.auxiliaryfunctions.read_config(config)['project_path']
    # create new dict for video_sets
    video_sets = {}
    for video in videos:
        print(f'Adding video: {video}')
            rel_video_path = str(Path.resolve(Path(video)))
            vid = VideoReader(rel_video_path)
            video_sets[rel_video_path] = {"crop": ", ".join(map(str, vid.get_bbox()))}
            print(f'Could not read video, writing fake frame dimensions ...')
            rel_video_path = str(Path.resolve(Path(video)))
            video_sets[rel_video_path] = {"crop": ", ".join(map(str, [0, 1920, 0, 1080]))} # how bad is this
        # create labels directory
        labeldir = os.path.basename(video).split('.')[0]
        if os.path.isdir(os.path.join(projectpath,'labeled-data', labeldir)):
            print('Labels directory already exists!')
            print(f'Creating directoy: {labeldir}')
            os.mkdir(os.path.join(projectpath,'labeled-data', labeldir))

    # expand existing video_sets
    if original_video_sets is None:
        new_video_sets = video_sets
        new_video_sets = {**original_video_sets, **video_sets}

    # add videos to project
    edit = {'video_sets': new_video_sets}
    deeplabcut.auxiliaryfunctions.edit_config(config, edit);


# add video paths
#video_list = filedialog.askopenfilenames(title = 'Choose video paths to include to your project')
add_videopaths(config_path, video_list)

Alternatively you can just load a previously created model:

# load new project (see Notebook before to create a new project)
config = filedialog.askopenfilenames(title='Choose the config.yaml file of your DeepLabCut project:')
config_path = config[0]

print(f'Using project path: {config_path}')

Optional: Create 3D project to match synchronous frames

# create 3d config
config_3d = deeplabcut.create_new_project_3d('StopSignal','3D',num_cameras = 2, working_directory=deeplabcut.auxiliaryfunctions.read_config(config_path)['project_path'])

# change settings
edits = {'camera_names': ['camR', 'camL']}
deeplabcut.auxiliaryfunctions.edit_config(config_3d, edits);
config_3d = r'F:\PigeonSuperModel_local\DeepLabCutModels\StopSignal-PigeonSuperModel-2022-07-08\StopSignal-3D-2022-07-08-3d\config.yaml'

Extract New Frames

# select videos for primary camera
video_sets = deeplabcut.auxiliaryfunctions.read_config(config_path)['video_sets']
primary_camera = deeplabcut.auxiliaryfunctions.read_config(config_3d)['camera_names'][0]
videos_primary = [str(video) for video in video_sets if primary_camera in video]

# change the number of frames to extract
edits = {'numframes2pick': 10}

# write changes to config.yaml
deeplabcut.auxiliaryfunctions.edit_config(config_path, edits);
# Extract frames
# Note: videos_list parameter is only available since dlc==2.2.1
deeplabcut.extract_frames(config_path, mode="automatic", algo="uniform", userfeedback = False, videos_list = videos_primary)

Optional: Extract Matching Frames in 3D projects

def patch_mp4toavi(config):
    import cv2
    import shutil

    # get videos from config.yaml
    video_sets = deeplabcut.auxiliaryfunctions.read_config(config)['video_sets'].keys()
    projectpath = deeplabcut.auxiliaryfunctions.read_config(config)['project_path']
    original_videos = [video for video in video_sets]
    # patch videos for matched frame extraction se here:
    patched_videos = [os.path.join(projectpath, 'videos', os.path.basename(video).split('.')[0] + '.avi') for video in original_videos]
    # change video extension and move
    for old, new in zip(original_videos, patched_videos):
        # test and close videos
        cap = cv2.VideoCapture(video)
            # read first frame
            # break after first frame
        # Closes all the frames
        # rename
        if os.path.isfile(new):
            print(f'File {new} already exists')
            shutil.move(old, new)
            #os.rename(old, new)
            print(f'Video: {old} changed to {new}')
    return original_videos, patched_videos
def patch_reverse_mp4toavi(original_videos, patched_videos):
    import shutil
    for old, new in zip(original_videos, patched_videos):
        # reverse video patch to original
        shutil.move(new, old)
        print(f'Video: {new} reversed to {old}')
# patch videos for matched frames see here:
original_videos, patched_videos = patch_mp4toavi(config_path)
# Overwrite matched synchronous frames with mode="match" and config3d file
deeplabcut.extract_frames(config_path, mode="match", config3d=config_3d, extracted_cam=0, userfeedback = False)
# reverse patched videos
patch_reverse_mp4toavi(original_videos, patched_videos)

Label your own Data first

# manually label frames
# check labels

Merge the PigeonSuperModel DataSet

How to use the PigeonSuperModel Dataset. Once you have your own DeepLabCut project for single Pigeon tracking, why not spiking your dataset with an extra 1000 pre-labeled frames? The cells below will guide you through this process.

# make a single function
def merge_psm_dataset(config):

    import urllib.request
    import tarfile
    import pandas as pd
    from pathlib import Path
    # read config.yaml
    target_dir = deeplabcut.auxiliaryfunctions.read_config(config)['project_path']
    scorer = deeplabcut.auxiliaryfunctions.read_config(config)['scorer']
    original_video_sets = deeplabcut.auxiliaryfunctions.read_config(config)['video_sets']

    # download
    print('Downloading DataSet from: ...')
    dataset_url = ''
    filename, _ = urllib.request.urlretrieve(dataset_url)
    statement = 'The PigeonSuperModel Dataset was added to this project to increase the number of labeled frames. (C)'
    # Extract .tar
    print(f'Extracting tar file to: {target_dir} ...')
    with, mode="r:gz") as tar:

    dataset_path = [os.path.join(target_dir, path) for path in os.listdir(target_dir) if 'pigeonsupermodel' in path][0]

    # scrap pre-labeled dataset
    print(f'Merging Dataset to: {os.path.join(target_dir, "labeled-data")} ...')
    for root, dirs, files in os.walk(dataset_path):
        for f in files:
            if '.h5' in f:
                h5file = os.path.join(root, f)
                # rename scorer in .h5
                df = pd.read_hdf(h5file)
                df.columns = df.columns.set_levels([scorer], level=0)
                # rename .h5 file
                rename = os.path.join(root, f'CollectedData_{scorer}.h5')
                df.to_hdf(rename, key = "df_with_missing", mode="w")

                # move frame set to labeled-data
                src = root
                dst = os.path.join(target_dir, 'labeled-data', os.path.basename(root))
                os.rename(src, dst)

    # add dummy videopaths to config.yaml files
    print('Adding video paths to "video_sets" in config.yaml ...')
    framesets = os.listdir(os.path.join(target_dir, 'labeled-data'))
    videos = [os.path.join(target_dir, 'videos',frameset+'.avi') for frameset in framesets]

    # create new dict for video_sets
    video_sets = {}
    for video in videos:
        rel_video_path = str(Path.resolve(Path(video)))
        video_sets[rel_video_path] = {"crop": ", ".join(map(str, [0, 1920, 0, 1080]))} # how bad is this?
    # expand existing video_sets
    if original_video_sets is None:
        new_video_sets = video_sets
        new_video_sets = {**original_video_sets, **video_sets}

    # add video paths to project
    edit = {'video_sets': new_video_sets, 
            'PigeonSuperModel_data': statement}
    deeplabcut.auxiliaryfunctions.edit_config(config, edit);

    print('The PigeonSuperModel Dataset has been successfully merged to your project! Enjoy')


Now you are ready to create a training dataset and train your own model

Re-Train the PigeonSuperModel

# create training set
deeplabcut.create_training_dataset(config_path, num_shuffles=1, net_type='resnet_50', augmenter_type='imgaug')
# find pose_cfg.yaml
for root, dirs, files in os.walk(deeplabcut.auxiliaryfunctions.read_config(config)['project_path']):
    for file in files:
        if 'pose_cfg.yaml' in file:
            pose_cfg = os.path.join(root, file)

# edit relevant training parameters
edits = {'multi_step': [[0.005, 10000], [0.02, 430000], [0.002, 730000], [0.001, 1030000]], # change this to train over 1M iterations
        'init_weights': path_to_lastsnapshot, # change this to use pre-trained model
        'max_input_size': 2000, # change this to largest frame dimension in dataset
        'global_scale': 0.4, # change this to rescale frames to reduce GPU load
deeplabcut.auxiliaryfunctions.edit_config(pose_cfg, edits)
# train
deeplabcut.train_network(config_path, max_snapshots_to_keep=None, displayiters=1000, saveiters=10000, maxiters=1000000, allow_growth=True)

Evaluate your new PigeonSuperModel

# change config.yaml file to evaluate all snapshots
edits = {'snapshotindex': 'all'}
deeplabcut.auxiliaryfunctions.edit_config(config_path, edits)

# evaluate snapshots
deeplabcut.evaluate_network(config_path, plotting=False)
def check_evaluation_results(config):
    import pandas as pd
    import numpy as np
    import matplotlib.pyplot as plt

    # read config
    config = deeplabcut.auxiliaryfunctions.read_config(config)
    projectpath = config['project_path']
    iteration = config['iteration']
    modelname = config['Task']
    pcutoff = config['pcutoff']
    videores = list(config['video_sets'].items())[0][1]['crop'].split(',')[1:4:2]
    # read evaluation results
    evals = os.path.join(projectpath, 'evaluation-results', f'iteration-{iteration}', 'CombinedEvaluation-results.csv')
    df = pd.read_csv(evals)
    evaluation_results = df.loc[:, ~df.columns.isin(['Unnamed: 0', '%Training dataset', 'Shuffle number', 'p-cutoff used'])]

    # plot evaluation results
    evaluation_results.plot.line(x = 'Training iterations:', 
                title = f"Evaluation results of DLC Model {modelname}", 
                ylabel = f"Pixel Error in {videores[0]}:{videores[1]} resolution frames",
                xlabel = f"Training Iterations in DLC [p-cutoff = {pcutoff}]",
                xticks = np.arange(0, 1000001, 40000),
                rot = 45, figsize = (15,10))
    plt.ticklabel_format(axis='both', style='plain')

    # check evaluation results
    return evaluation_results

# show evaluation results
# export model
deeplabcut.export_model(config_path, shuffle=0, iteration=0, snapshotindex=-1, make_tar=True, modelprefix='')

Contribute to

Do you have an improved version of the PigeonSuperModel?

Share your labeled data and your new trained model with us to contribute to the

Get in touch at or at