Reducing .slp file to contain only human labeled frames

[catubc]

Hello

We have an .slp file that contains 1000 human labeled frames from 12 videos, total 300,000+ frames. We'd like to compress this file significantly, i.e remove all non-human labeled frames and save an slp file containing just the labeled frames. In parallel we've made a short (chimera) video from the 12 vids containing just those 1000 frames that were labeled.

We are close (see also a screen grab), but can't change the video filenames in the .slp files. For some reason the original video filenames are still being searched for during the save. The code is below and the 2 files (hybrid .mp4 and original .slp file they are here: https://drive.google.com/drive/folders/1GhK8D1YslYhtR6Ut5xRYi6wZoAeCu8jK?usp=share_link)

I think this should be trivial, but I"m out of ideas... the videonames are really sticky!?

#
fname = "/home/cat/data/dan/merge_hybrid_slp/cohort2_day.1000.slp"
merged_video = sleap.load_video('/home/cat/data/dan/merge_hybrid_slp/hybrid.mp4')

#
labels = sleap.load_file(fname)
![Screenshot from 2022-12-22 10-41-04](https://user-images.githubusercontent.com/4267452/209131195-ddcdae74-8f7f-40f7-a882-1cd0706c8950.png)


#
labeled_frames_from_video = labels.user_labeled_frames

#
lf_indices = np.arange(len(labeled_frames_from_video))

# Create a new `Labels` object containing only the matched `LabeledFrame`s
labels_keep = labels.extract(lf_indices, copy=True)
   
# loop over all the frames to keep and
frame_idxs = []
fnames = []
ctr=0
for lk in tqdm(labels_keep):
    lk.frame_idx = ctr

    frame_idxs.append(lk.frame_idx)
    fnames.append(lk.video.filename)
    lk.video.filename = '/home/cat/data/dan/merge_hybrid_slp/hybrid.mp4'
    lf.video = merged_video

    for p in range(len(lk.instances)):
        lk.instances[p].video.filename = '/home/cat/data/dan/merge_hybrid_slp/hybrid.mp4'
   
    ctr+=1

# overwrite labels videos with the merged video
for k in range(len(labels_keep.videos)):
    #print (labels.videos[k])
    labels_keep.videos[k] = merged_video  

# Save the modified slp
fname_out = fname[:-4] + "_deleted.slp"  # Use a different path from the current slp to be safe

#
labels_keep.save(fname_out)

#
np.save(fname_out[:-4]+"_frame_idx.npy", frame_idxs)
np.save(fname_out[:-4]+"_fnames.npy", fnames)

ERROR:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[192], line 50
     47 fname_out = fname[:-4] + "_deleted.slp"  # Use a different path from the current slp to be safe
     49 #
---> 50 labels_keep.save(fname_out)
     52 #
     53 np.save(fname_out[:-4]+"_frame_idx.npy", frame_idxs)

File ~/.local/lib/python3.10/site-packages/sleap/io/dataset.py:2008, in Labels.save(self, filename, with_images, embed_all_labeled, embed_suggested)
   2006 if os.path.splitext(filename)[1].lower() != ".slp":
   2007     filename = filename + ".slp"
-> 2008 Labels.save_file(
   2009     self,
   2010     filename,
   2011     save_frame_data=with_images,
   2012     all_labeled=embed_all_labeled,
   2013     suggested=embed_suggested,
   2014 )

File ~/.local/lib/python3.10/site-packages/sleap/io/dataset.py:1978, in Labels.save_file(cls, labels, filename, default_suffix, *args, **kwargs)
   1974 os.makedirs(os.path.dirname(filename), exist_ok=True)
   1976 from .format import write
-> 1978 write(filename, labels, *args, **kwargs)

File ~/.local/lib/python3.10/site-packages/sleap/io/format/main.py:162, in write(filename, source_object, as_format, *args, **kwargs)
    160 if hasattr(source_object, "labeled_frames"):
    161     disp.register_list(default_labels_adaptors)
--> 162     return disp.write(filename, source_object, *args, **kwargs)
    164 raise NotImplementedError(
    165     f"No adaptors for object type {type(source_object)} ({as_format})."
    166 )

File ~/.local/lib/python3.10/site-packages/sleap/io/format/dispatch.py:78, in Dispatch.write(self, filename, source_object, *args, **kwargs)
     76 for adaptor in self._adaptors:
     77     if adaptor.can_write_filename(filename):
---> 78         return adaptor.write(filename, source_object, *args, **kwargs)
     80 raise TypeError("No file format adaptor could write this file.")

File ~/.local/lib/python3.10/site-packages/sleap/io/format/hdf5.py:244, in LabelsV1Adaptor.write(cls, filename, source_object, append, save_frame_data, frame_data_format, all_labeled, suggested, progress_callback)
    241     os.unlink(filename)
    243 # Serialize all the meta-data to JSON.
--> 244 d = labels.to_dict(skip_labels=True)
    246 if save_frame_data:
    247     new_videos = labels.save_frame_data_hdf5(
    248         filename,
    249         format=frame_data_format,
   (...)
    253         progress_callback=progress_callback,
    254     )

File ~/.local/lib/python3.10/site-packages/sleap/io/dataset.py:1919, in Labels.to_dict(self, skip_labels)
   1910 track_cattr = cattr.Converter(unstruct_strat=cattr.UnstructureStrategy.AS_TUPLE)
   1912 # Serialize the skeletons, videos, and labels
   1913 dicts = {
   1914     "version": LABELS_JSON_FILE_VERSION,
   1915     "skeletons": skeleton_cattr.unstructure(self.skeletons),
   1916     "nodes": cattr.unstructure(self.nodes),
   1917     "videos": Video.cattr().unstructure(self.videos),
   1918     "tracks": track_cattr.unstructure(self.tracks),
-> 1919     "suggestions": label_cattr.unstructure(self.suggestions),
   1920     "negative_anchors": label_cattr.unstructure(self.negative_anchors),
   1921     "provenance": label_cattr.unstructure(self.provenance),
   1922 }
   1924 if not skip_labels:
   1925     dicts["labels"] = label_cattr.unstructure(self.labeled_frames)

File ~/.local/lib/python3.10/site-packages/cattr/converters.py:142, in Converter.unstructure(self, obj)
    141 def unstructure(self, obj):
--> 142     return self._unstructure_func.dispatch(obj.__class__)(obj)

File ~/.local/lib/python3.10/site-packages/cattr/converters.py:232, in Converter._unstructure_seq(self, seq)
    230 # We can reuse the sequence class, so tuples stay tuples.
    231 dispatch = self._unstructure_func.dispatch
--> 232 return seq.__class__(dispatch(e.__class__)(e) for e in seq)

File ~/.local/lib/python3.10/site-packages/cattr/converters.py:232, in <genexpr>(.0)
    230 # We can reuse the sequence class, so tuples stay tuples.
    231 dispatch = self._unstructure_func.dispatch
--> 232 return seq.__class__(dispatch(e.__class__)(e) for e in seq)

File ~/.local/lib/python3.10/site-packages/cattr/converters.py:212, in Converter.unstructure_attrs_asdict(self, obj)
    210     name = a.name
    211     v = getattr(obj, name)
--> 212     rv[name] = dispatch(v.__class__)(v)
    213 return rv

File ~/.local/lib/python3.10/site-packages/sleap/io/dataset.py:1896, in Labels.to_dict.<locals>.<lambda>(x)
   1891 label_cattr = make_instance_cattr()
   1892 label_cattr.register_unstructure_hook(
   1893     Skeleton, lambda x: str(self.skeletons.index(x))
   1894 )
   1895 label_cattr.register_unstructure_hook(
-> 1896     Video, lambda x: str(self.videos.index(x))
   1897 )
   1898 label_cattr.register_unstructure_hook(Node, lambda x: str(self.nodes.index(x)))
   1899 label_cattr.register_unstructure_hook(
   1900     Track, lambda x: str(self.tracks.index(x))
   1901 )

ValueError: Video(backend=MediaVideo(filename='/home/cat/data/dan/merge_hybrid_slp/2020_07_30_11_30_11_732262_compressed_corrected.mp4', grayscale=False, bgr=True, dataset='', input_format='')) is not in list

[roomrys]

Hi @catubc,

So the problem stems from setting filename of the wrapper Video class not the backend stored in Video.backend:

lk.video.filename = '/home/cat/data/dan/merge_hybrid_slp/hybrid.mp4'

You should instead be setting the filename on the Video.backend:

lk.video.backend.filename = '/home/cat/data/dan/merge_hybrid_slp/hybrid.mp4'

We use Video as an abstraction to the other backends, and, while we have an implementation for getting attributes through Video,

sleap/sleap/io/video.py

Lines 1023 to 1029 in c4861e3

	backend: Union[
	HDF5Video, NumpyVideo, MediaVideo, ImgStoreVideo, SingleImageVideo, DummyVideo
	] = attr.ib()
	# Delegate to the backend
	def __getattr__(self, item):
	return getattr(self.backend, item)

we do not currently any setter methods for setting backend specific attributes through Video. But, we could definitely add this (I too have run into this trap when working with the Video class).

Let us know if this helps get rid of those sticky videos!

Thanks,
Liezl

[catubc]

Thanks so much Liezl.

I've spent several hours past 2 days working on this and fortunately I was able to get it work now - but I had to revert to Talmo's method which just makes a new label file - rather than editing the existing. Below are the 2 functions that we used if anyone else needs help.

We will next work on merging this shortened human labeled .slp file with the id-switch .slp file which we made before. We may still require some help with this merge, so I will start a separate issue if that's the case.

Thank you again for everything (Talmo too).
catubc

This function makes the compressed .slp file:

#
def make_hybrid_slp(fname_merged_video,
                    fname_slp):

    ''' This function reduces an .slp file containing human labels and other frames
        to a much shroter file containing only human labeled frames. 
        It also saves the frame indexes and the video names so that the required hybrid.avi file
        can also be made to match the shorter .slp file
    
    '''
    
    #
    merged_video = sleap.load_video(fname_merged_video)

    # Load saved labels.
    labels = sleap.load_file(slp_file, match_to=first_labels)
    print ("Labels: ", labels)

    # loop over 
    new_frames = []
    frames_idx = []
    fnames = []
    ctr1 = 0
    for lf in labels:

        # Catalin added
        if lf.has_user_instances==False:
            continue

        # save indexes in order to make the hybrid videos
        frames_idx.append(lf.frame_idx)
        fnames.append(lf.video.backend.filename)

        # Update reference to merged video.
        lf.video = merged_video
        lf.video.backend.filename= fname_merged_video

        # Update frame index to the frame number within the merged video.
        lf.frame_idx = ctr1

        # Append the labeled frame to the list of frames we're keeping from these labels.
        new_frames.append(lf)

        #
        ctr1+=1

    # format labels and save them
    merged_labels = sleap.Labels(new_frames)
    merged_labels.save(os.path.split(fname_merged_video)[0]+'/hybrid.slp')

    # save indexes and video names to make hybrid video
    np.save(os.path.split(fname_merged_video)[0]+'/hybrid_frames_idx.npy' ,frames_idx)
    np.save(os.path.split(fname_merged_video)[0]+'/hybrid_fnames.npy' , fnames)

    #
    print ("...done...")

# load hybrid video (or placeholder until video is made)
fname_merged_video = '/home/cat/data/dan/merge_hybrid_slp/hybrid.avi'

# slp filename
fname_slp = '/home/cat/data/dan/merge_hybrid_slp/cohort2_day.1000.slp'

#
make_hybrid_slp(fname_merged_video,
                fname_slp)
    
    

This function makes the compressed .avi video to accompany the above .slp file.

def make_hybrid_human_labeled_video(idxs,
                                    fnames_videos):
    
    ''' Function makes a hybrid video from a list of video names and frame indexes.
        This video accompanies the compressed .slp file made that contains only human labeled
        frames.
    
    '''
    
    
    #
    root_dir = os.path.split(fnames_videos[0])[0] + '/'
    
    # make new video video settings
    size_vid = np.int32(np.array([900, 700]))
    fps_out = 1
    dot_size = 4
    thickness = -1
    window = 100
    fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
    #fourcc = cv2.VideoWriter_fourcc(*'mp4v')
    #fourcc = cv2.VideoWriter_fourcc(*'X264')
    # #fourcc = cv2.VideoWriter_fourcc(*args["codec"])
    #fourcc = cv2.VideoWriter_fourcc(*'XVID')
    #fourcc = cv2.VideoWriter_fourcc('p', 'n', 'g', '')

    # load videos
    fname_out = root_dir + '/hybrid.avi'

    #
    print("Fname hybrid movie: ", fname_out)

    # make new video video settings
    size_vid = np.int32(np.array([900, 700]))
    fps_out = 1

    #
    video_out = cv2.VideoWriter(fname_out,
                                fourcc,
                                fps_out,
                                (size_vid[0], size_vid[1]),
                                True)

    # loop over each slp file
    ctr=0
    for idx, fname_video in tqdm(zip(idxs, fnames_videos)):

        # load current vid
        original_vid = cv2.VideoCapture(fname_video)
        original_vid.set(cv2.CAP_PROP_POS_FRAMES, idx)
        
        #
        ret, img_out = original_vid.read()

        #
        video_out.write(img_out)
    
        # fname_video_old = fname_video
        original_vid.release()
        
        ctr+=1
    
    video_out.release()
    
    print ("done...")
    
#
fname_idx = np.load('/home/cat/data/dan/merge_hybrid_slp/hybrid_frames_idx.npy')
fnames_videos = np.load('/home/cat/data/dan/merge_hybrid_slp/hybrid_fnames.npy')

# make hybrid video from indexes
make_hybrid_human_labeled_video(fname_idx, 
                                fnames_videos)