How to extract movie frames in natural size?

Add-Ons
1

Hi,

I currently use this code to extract frames from a movie file:

if AAssetReader=nil then Return 'AVAssetReaderMBS object

if AAssetReader.status=AAssetReader.AVAssetReaderStatusReading then
  Var nts As CGSizeMBS=TrackOutput.track.naturalSize 'TrackOutput is an AVAssetReaderTrackOutputMBS
  Var bf As CMSampleBufferMBS=TrackOutput.NextSampleBuffer
  
  if bf<>nil then
    Var ib As CVImageBufferMBS=bf.ImageBuffer
    
    if ib<>nil then
      ResultNSImage=ib.NSImage
      Return
    end if
  end if
end if

The resulting image has a unexpected size, however. The movie’s real size is 1048x576 (in the code above, the “nts” variable correctly reports this), but the returned NSImage has a size of 720x576.

Checking in QuickTime Player, I get both versions (the “Resolution” line), although the player shows the correct size of 1048x576:
image

I’d like to retrieve the image in the natural size (1048x576) rather than the “current” size (I don’t know where this size is coming from, by the way :man_shrugging:).

The MBS documentation mentions “The MBS Plugin implements a subset of what’s available. If you need more, please do not hesitate to contact us.”. I’m therefore wondering whether the function I need could be added (either specify a given size to an existing class or be able to retrieve the NSImage in a wanted size). Or perhaps I’m missing an existing way?

For now, I’m using “ResultNSImage.setSize(nts.width,nts.height)” to resize the picture after getting it, but I’m concerned about losing quality.

2

So your video has 720x576, which is currently shown as 1049 x 576, so it’s basically compressed horizontally.

I tried this code:

Dim file As FolderItem = New FolderItem("/Users/cs/Movies/Familie/Trampolin.mov", FolderItem.PathModes.Native)
Dim asset As AVAssetMBS = AVAssetMBS.assetWithFile(File)
dim error as NSErrorMBS
Dim AssetReader As AVAssetReaderMBS = AVAssetReaderMBS.assetReaderWithAsset(asset, error)

Dim VideoTracks() As AVAssetTrackMBS = asset.tracksWithMediaType(AVFoundationMBS.AVMediaTypeVideo)
Dim VideoTrack As AVAssetTrackMBS = VideoTracks(0)
Dim naturalSize As CGSizeMBS = VideoTrack.naturalSize

// setup track output
Dim d As New Dictionary
d.Value(CVPixelBufferMBS.kCVPixelBufferPixelFormatTypeKey) = CVPixelBufferMBS.kCVPixelFormatType_32ARGB

' you can request whatever size you like
'd.Value(CVPixelBufferMBS.kCVPixelBufferWidthKey) = 1024
'd.Value(CVPixelBufferMBS.kCVPixelBufferHeightKey) = 576

Dim TrackOutput As New AVAssetReaderTrackOutputMBS(VideoTrack, d)
AssetReader.addOutput TrackOutput

// start it!
AssetReader.startReading

If AssetReader.status = AssetReader.AVAssetReaderStatusReading Then
  
  Var bf As CMSampleBufferMBS = TrackOutput.NextSampleBuffer
  
  If bf<>Nil Then
    Var ib As CVImageBufferMBS=bf.ImageBuffer
    
    If ib<>Nil Then
      Dim ResultNSImage As NSImageMBS = ib.NSImage
      
      Backdrop = ResultNSImage.CopyPicture
      Break
    End If
  End If
End If
2 Likes
3

Thank you, this works like a charm! (and I’ve learnt a good trick with your example)

Do you happen to know whether this horizontal compression makes sense? The video came from a video camera.
Does it mean the compression reduce the file size, so the picture is actually poorer than it should?
Thanks again.