Render subtitles

Author: WyattBlue

.gitignore

@@ -11,6 +11,7 @@ tests/*
 *_ALTERED*
 *https*
 *.v3
+*.srt
 
 # OS Files
 .DS_Store

src/render/format.nim

@@ -12,6 +12,7 @@ import ../av
 import ../util/[bar, rules]
 import video
 import audio
+import subtitle
 
 type Priority = object
   index: float64
@@ -145,6 +146,32 @@ proc makeMedia*(args: mainArgs, tl: v3, outputPath: string, rules: Rules, bar: B
     for aEncCtx in audioEncoders:
       avcodec_free_context(addr aEncCtx)
 
+  # Setup subtitle streams
+  var subtitleStreams: seq[ptr AVStream] = @[]
+  var subtitleSources: seq[string] = @[]
+
+  if not args.sn and tl.s.len > 0:
+    for i in 0..<tl.s.len:
+      if tl.s[i].len > 0:
+        # Get source file and stream index from first clip
+        let firstClip = tl.s[i].c[0]
+        let sourcePath = firstClip.src[]
+        let streamIdx = firstClip.stream
+
+        # Open source container to get subtitle stream info
+        let srcContainer = av.open(sourcePath)
+        if streamIdx >= srcContainer.subtitle.len:
+          error &"Subtitle stream {streamIdx} not found in {sourcePath}"
+
+        let srcStream = srcContainer.subtitle[streamIdx]
+
+        # Add subtitle stream to output by copying from template
+        let sOutStream = output.addStreamFromTemplate(srcStream)
+        subtitleStreams.add(sOutStream)
+        subtitleSources.add(sourcePath)
+
+        srcContainer.close()
+
   var outPacket = av_packet_alloc()
   if outPacket == nil:
     error "Could not allocate output packet"
@@ -161,6 +188,10 @@ proc makeMedia*(args: mainArgs, tl: v3, outputPath: string, rules: Rules, bar: B
   for aEncCtx in audioEncoders:
     let name = aEncCtx.codec.canonicalName
     encoderTitles.add (if noColor: name else: &"\e[96m{name}")
+  for sStream in subtitleStreams:
+    let name = $sStream.name()
+    if name != "":
+      encoderTitles.add (if noColor: name else: &"\e[93m{name}")
 
   if noColor:
     title &= encoderTitles.join("+")
@@ -275,4 +306,12 @@ proc makeMedia*(args: mainArgs, tl: v3, outputPath: string, rules: Rules, bar: B
       output.mux(outPacket[])
       av_packet_unref(outPacket)
 
+  # Process subtitle streams
+  if not args.sn and subtitleStreams.len > 0:
+    for i in 0..<subtitleStreams.len:
+      let layer = tl.s[i]
+      let sourcePath = subtitleSources[i]
+      let outputStream = subtitleStreams[i]
+      remuxSubtitles(sourcePath, layer, outputStream, output, tl.tb)
+
   output.close()

src/render/subtitle.nim

@@ -0,0 +1,89 @@
+import ../timeline
+import ../ffmpeg
+import ../log
+import ../av
+
+# Simple subtitle remuxing: copy subtitle packets from source, adjusting timestamps
+# Note: This approach works well for text-based subtitles (SRT, ASS, WebVTT, etc.)
+# For bitmap subtitles (DVD/PGS), the timestamps are adjusted but the visual data remains unchanged
+proc remuxSubtitles*(sourcePath: string, layer: ClipLayer, outputStream: ptr AVStream,
+    output: var OutputContainer, timelineTb: AVRational) =
+  if layer.c.len == 0:
+    return
+
+  # Open source container for each remux operation
+  let srcContainer = av.open(sourcePath)
+  defer: srcContainer.close()
+
+  let formatCtx = srcContainer.formatContext
+  let outTb = outputStream.time_base
+
+  for clip in layer.c:
+    if clip.stream >= srcContainer.subtitle.len:
+      continue
+
+    let streamIndex = srcContainer.subtitle[clip.stream].index
+    let stream = formatCtx.streams[streamIndex]
+    let srcTb = stream.time_base
+
+    # Seek to the clip's offset position in source timebase
+    # Note: timelineTb is actually the frame rate, so the actual timebase is 1/timelineTb
+    let seekPts = int64(float64(clip.offset) / (float64(timelineTb) * float64(srcTb)))
+
+    # Calculate the end position in source timebase
+    let clipEndInSrcTb = int64(float64(clip.offset + clip.dur) / (float64(timelineTb) * float64(srcTb)))
+
+    # Seek to start of clip
+    if seekPts > 0:
+      srcContainer.seek(seekPts, backward = true, stream = stream)
+
+    var packet = av_packet_alloc()
+    if packet == nil:
+      error "Could not allocate subtitle packet"
+    defer: av_packet_free(addr packet)
+
+    # Read and copy packets for this clip
+    while av_read_frame(formatCtx, packet) >= 0:
+      defer: av_packet_unref(packet)
+
+      if packet.stream_index == streamIndex:
+        # Check if packet is within the clip range
+        if packet.pts != AV_NOPTS_VALUE and packet.pts >= seekPts and packet.pts < clipEndInSrcTb:
+          # Calculate new timestamp in output timebase
+          # Step 1: Get relative position in source timebase
+          let relativeStartInSrcTb = packet.pts - seekPts
+          # Step 2: Convert to timeline units (frames)
+          let relativeStartInFrames = int64(float64(relativeStartInSrcTb) * float64(timelineTb) * float64(srcTb))
+          # Step 3: Calculate absolute position in frames
+          let absoluteFramePos = clip.start + relativeStartInFrames
+          # Step 4: Convert from frames to output timebase
+          let newPts = int64(float64(absoluteFramePos) / (float64(timelineTb) * float64(outTb)))
+
+          # Create output packet with adjusted timestamps
+          var outPacket: AVPacket
+          outPacket.time_base = outTb
+          outPacket.stream_index = outputStream.index
+
+          if av_packet_ref(addr outPacket, packet) < 0:
+            error "Failed to reference subtitle packet"
+
+          # Convert timestamps to timeline timebase
+          outPacket.pts = newPts
+          if packet.dts != AV_NOPTS_VALUE:
+            let relativeDtsInSrcTb = packet.dts - seekPts
+            let relativeDtsInFrames = int64(float64(relativeDtsInSrcTb) * float64(timelineTb) * float64(srcTb))
+            let absoluteDtsFramePos = clip.start + relativeDtsInFrames
+            outPacket.dts = int64(float64(absoluteDtsFramePos) / (float64(timelineTb) * float64(outTb)))
+          else:
+            outPacket.dts = AV_NOPTS_VALUE
+
+          if packet.duration != AV_NOPTS_VALUE:
+            outPacket.duration = int64(float64(packet.duration) * float64(timelineTb) * float64(srcTb) / (float64(timelineTb) * float64(outTb)))
+
+          # Mux the packet
+          output.mux(outPacket)
+          av_packet_unref(addr outPacket)
+
+        elif packet.pts != AV_NOPTS_VALUE and packet.pts >= clipEndInSrcTb:
+          # We've passed the end of this clip
+          break

src/timeline.nim

@@ -52,6 +52,7 @@ type v3* = object
   res*: (int, int)
   v*: seq[ClipLayer]
   a*: seq[ClipLayer]
+  s*: seq[ClipLayer]
   effects*: seq[Action]
   clips2*: Option[seq[Clip2]]  # Optional because tl might be non-linear.
 
@@ -144,6 +145,7 @@ proc initLinearTimeline*(src: ptr string, tb: AvRational, bg: RGBColor, mi: Medi
 
   var vspace: seq[ClipLayer] = @[]
   var aspace: seq[ClipLayer] = @[]
+  var sspace: seq[ClipLayer] = @[]
 
   if mi.v.len > 0:
     var vlayer = ClipLayer(lang: mi.v[0].lang, c: @[])
@@ -161,7 +163,15 @@ proc initLinearTimeline*(src: ptr string, tb: AvRational, bg: RGBColor, mi: Medi
       alayer.c.add(audioClip)
     aspace.add(alayer)
 
-  result = v3(tb: tb, v: vspace, a: aspace, bg: bg, effects: effects)
+  for i in 0 ..< mi.s.len:
+    var slayer = ClipLayer(lang: mi.s[i].lang, c: @[])
+    for clip in clips:
+      var subtitleClip = clip
+      subtitleClip.stream = i.int32
+      slayer.c.add(subtitleClip)
+    sspace.add(slayer)
+
+  result = v3(tb: tb, v: vspace, a: aspace, s: sspace, bg: bg, effects: effects)
 
   if result.timelineIsEmpty:
     error "Timeline is empty, nothing to do."
@@ -217,6 +227,7 @@ proc toNonLinear*(src: ptr string, tb: AvRational, bg: RGBColor, mi: MediaInfo,
 
   var vspace: seq[ClipLayer] = @[]
   var aspace: seq[ClipLayer] = @[]
+  var sspace: seq[ClipLayer] = @[]
 
   if mi.v.len > 0:
     var vlayer = ClipLayer(lang: mi.v[0].lang, c: @[])
@@ -234,7 +245,15 @@ proc toNonLinear*(src: ptr string, tb: AvRational, bg: RGBColor, mi: MediaInfo,
       alayer.c.add(audioClip)
     aspace.add(alayer)
 
-  result = v3(tb: tb, v: vspace, a: aspace, bg: bg, clips2: some(clips2))
+  for i in 0 ..< mi.s.len:
+    var slayer = ClipLayer(lang: mi.s[i].lang, c: @[])
+    for clip in clips:
+      var subtitleClip = clip
+      subtitleClip.stream = i.int32
+      slayer.c.add(subtitleClip)
+    sspace.add(slayer)
+
+  result = v3(tb: tb, v: vspace, a: aspace, s: sspace, bg: bg, clips2: some(clips2))
   result.effects = effects
 
   if result.timelineIsEmpty:
@@ -275,6 +294,7 @@ proc toNonLinear2*(src: ptr string, tb: AVRational, bg: RGBColor, mi: MediaInfo,
 
   var vspace: seq[ClipLayer] = @[]
   var aspace: seq[ClipLayer] = @[]
+  var sspace: seq[ClipLayer] = @[]
 
   if mi.v.len > 0:
     var vlayer = ClipLayer(lang: mi.v[0].lang, c: @[])
@@ -292,7 +312,15 @@ proc toNonLinear2*(src: ptr string, tb: AVRational, bg: RGBColor, mi: MediaInfo,
       alayer.c.add(audioClip)
     aspace.add(alayer)
 
-  result = v3(tb: tb, v: vspace, a: aspace, bg: bg, clips2: some(clips2))
+  for i in 0 ..< mi.s.len:
+    var slayer = ClipLayer(lang: mi.s[i].lang, c: @[])
+    for clip in clips:
+      var subtitleClip = clip
+      subtitleClip.stream = i.int32
+      slayer.c.add(subtitleClip)
+    sspace.add(slayer)
+
+  result = v3(tb: tb, v: vspace, a: aspace, s: sspace, bg: bg, clips2: some(clips2))
   result.effects = effects
 
   if result.timelineIsEmpty:

tests/ffwrapper.py

@@ -41,7 +41,6 @@ class AudioStream:
 @dataclass(slots=True, frozen=True)
 class SubtitleStream:
     codec: str
-    ext: str
     lang: str | None
 
 
@@ -130,16 +129,11 @@ def init(self, path: str, log: Log) -> FileInfo:
 
         for s in cont.streams.subtitles:
             codec = s.codec_context.name
-            sub_exts = {"mov_text": "srt", "ass": "ass", "webvtt": "vtt"}
-            ext = sub_exts.get(codec, "vtt")
-            subtitles += (SubtitleStream(codec, ext, s.language),)
+            subtitles += (SubtitleStream(codec, s.language),)
 
         bitrate = 0 if cont.bit_rate is None else cont.bit_rate
         dur = 0 if cont.duration is None else cont.duration / av.time_base
 
         cont.close()
 
         return FileInfo(Path(path), bitrate, dur, videos, audios, subtitles)
-
-    def __repr__(self) -> str:
-        return f"@{self.path.name}"

tests/test.py

@@ -332,17 +332,33 @@ def test_premiere_named_export(self) -> None:
         self.main(["example.mp4"], ["--export", 'premiere:name="Foo Bar"'])
 
     def test_export_subtitles(self) -> None:
-        raise SkipTest()  # TODO
-        # cn = fileinfo(self.main(["resources/mov_text.mp4"], [], "movtext_out.mp4"))
+        # Test mov_text subtitle stream handling
+        cn = fileinfo(self.main(["resources/mov_text.mp4"], [], "movtext_out.mp4"))
 
-        # assert len(cn.videos) == 1
-        # assert len(cn.audios) == 1
-        # assert len(cn.subtitles) == 1
+        assert len(cn.videos) == 1, "Should have exactly 1 video stream"
+        assert len(cn.audios) == 1, "Should have exactly 1 audio stream"
+        assert len(cn.subtitles) == 1, "Should have exactly 1 subtitle stream"
+        assert cn.subtitles[0].codec == "mov_text", f"Expected mov_text codec, got {cn.subtitles[0].codec}"
 
+        # Test WebVTT subtitle stream handling
         cn = fileinfo(self.main(["resources/webvtt.mkv"], [], "webvtt_out.mkv"))
-        assert len(cn.videos) == 1
-        assert len(cn.audios) == 1
-        assert len(cn.subtitles) == 1
+        assert len(cn.videos) == 1, "Should have exactly 1 video stream"
+        assert len(cn.audios) == 1, "Should have exactly 1 audio stream"
+        assert len(cn.subtitles) == 1, "Should have exactly 1 subtitle stream"
+        assert cn.subtitles[0].codec == "webvtt", f"Expected webvtt codec, got {cn.subtitles[0].codec}"
+
+    def test_sn(self) -> None:
+        # Test that -sn flag properly disables subtitle streams
+        cn = fileinfo(self.main(["resources/mov_text.mp4"], ["-sn"], "movtext_nosub.mp4"))
+        assert len(cn.videos) == 1, "Should have exactly 1 video stream"
+        assert len(cn.audios) == 1, "Should have exactly 1 audio stream"
+        assert len(cn.subtitles) == 0, "Should have 0 subtitle streams when -sn flag is used"
+
+        # Test that -sn flag works with webvtt too
+        cn = fileinfo(self.main(["resources/webvtt.mkv"], ["-sn"], "webvtt_nosub.mkv"))
+        assert len(cn.videos) == 1, "Should have exactly 1 video stream"
+        assert len(cn.audios) == 1, "Should have exactly 1 audio stream"
+        assert len(cn.subtitles) == 0, "Should have 0 subtitle streams when -sn flag is used"
 
     def test_scale(self) -> None:
         cn = fileinfo(self.main(["example.mp4"], ["--scale", "1.5"], "scale.mp4"))