使用streamlink将流处理为单独的帧

Question

我正在尝试使用streamlink将流作为单独的帧来处理

args = ['streamlink', stream_url, "best", "-O"]
process = subprocess.Popen(args, stdout=subprocess.PIPE)
while True:
frame_size = width * height * 3
in_frame = streamlink.stdout.read(frame_size)
if in_frame is None:
break
#cv2.imwrite(f'frames/{i}.jpg', in_frame)
#do anything with in_frame

但让图片看起来像白噪声。我认为这是因为流也以字节形式包含音频。然后我尝试将其管道到ffmpeg，但无法从ffmpeg

中获得解码字节

args = (
ffmpeg.input('pipe:')
.filter('fps', fps=1)
.output('pipe:', vframes=1, format='image2', vcodec='mjpeg')
.compile()
)
process2 = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=None)
buff = process1.stdout.read(n)
process2.stdin.write(buff)
frame = process2.stdout.read(n)

当我试着这样写的时候，我所有的脚本都挂起了，等待着什么。如何正确地处理来自streamlink的流作为单个帧。以字节形式获取帧还是其他形式?谢谢你。

Answer 1

您可以将URL作为输入参数传递给FFmpeg，而不是将数据管道传递给FFmpeg。

• 从WEB站点URL获取流URL:

  def stream_to_url(url, quality='best'):
  session = Streamlink()
  streams = session.streams(url)
  return streams[quality].to_url()
  

• 使用FFprobe获取视频分辨率(如果需要):

  p = ffmpeg.probe(stream_url, select_streams='v');
  width = p['streams'][0]['width']
  height = p['streams'][0]['height']
  

• 以URL作为输入和原始(BGR)输出格式执行FFmpeg子进程:

  process = (
  ffmpeg
  .input(stream_url)
  .video
  .output('pipe:', format='rawvideo', pix_fmt='bgr24')
  .run_async(pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:FFmpegbinffmpeg.exe')
  )
  

• 从PIPE中读取帧，转换为NumPy数组，重塑并显示:

  ...
  in_bytes = process.stdout.read(width * height * 3)
  frame = np.frombuffer(in_bytes, np.uint8).reshape([height, width, 3])
  cv2.imshow('frame', frame)
  ...
  

---

完整的代码示例:

from streamlink import Streamlink
import numpy as np
import cv2
import ffmpeg
def stream_to_url(url, quality='best'):
""" Get URL, and return streamlink URL """
session = Streamlink()
streams = session.streams(url)
if streams:
return streams[quality].to_url()
else:
raise ValueError('Could not locate your stream.')

url = 'https://www.twitch.tv/riotgames'  # Login to twitch TV before starting (the URL is for a random live stream).
quality='best'
stream_url = stream_to_url(url, quality)
# Use FFprobe to get video frames resolution (required in case resolution is unknown).
###############################################
p = ffmpeg.probe(stream_url, select_streams='v');
width = p['streams'][0]['width']
height = p['streams'][0]['height']
###############################################
# Execute FFmpeg sub-process with URL as input and raw (BGR) output format.
process = (
ffmpeg
.input(stream_url)
.video
.output('pipe:', format='rawvideo', pix_fmt='bgr24')
.run_async(pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:FFmpegbinffmpeg.exe')
)

# Read decoded video (frame by frame), and display each frame (using cv2.imshow)
while True:
# Read raw video frame from stdout as bytes array.
in_bytes = process.stdout.read(width * height * 3)
if not in_bytes:
break
# Transform the byte read into a NumPy array
frame = np.frombuffer(in_bytes, np.uint8).reshape([height, width, 3])
# Display the frame
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
process.stdout.close()
process.wait()
cv2.destroyAllWindows()

---

有一个更简单的解决方案，使用cv2.VideoCapture:

stream_url = stream_to_url(url, quality)
cap = cv2.VideoCapture(stream_url)
while True:
success, frame = cap.read()
if not success:
break
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()

---

更新:

从Streamlink子进程到FFmpeg子进程的管道:

假设您必须从Streamlink的标准输出管道读取流并将其写入FFmpeg的标准输出管道:

• 启动Streamlink子进程(使用-O参数设置管道):

  streamlink_args = [r'c:Program Files (x86)Streamlinkbinstreamlink.exe', stream_url, "best", "-O"]  # Windows executable downloaded from: https://github.com/streamlink/streamlink/releases/tag/2.4.0
  streamlink_process = sp.Popen(streamlink_args, stdout=sp.PIPE)  # Execute Streamlink as sub-process
  

• 实现一个线程从Streamlink的标准输出管道中读取数据块并写入FFmpeg标准输入管道:

  def writer(streamlink_proc, ffmpeg_proc):
  while (not streamlink_proc.poll()) and (not ffmpeg_proc.poll()):
  try:
  chunk = streamlink_proc.stdout.read(1024)
  ffmpeg_proc.stdin.write(chunk)
  except (BrokenPipeError, OSError) as e:
  pass
  

• 使用输入管道和输出管道执行FFmpeg子进程:

  ffmpeg_process = (
  ffmpeg
  .input('pipe:')
  .video
  .output('pipe:', format='rawvideo', pix_fmt='bgr24')
  .run_async(pipe_stdin=True, pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:FFmpegbinffmpeg.exe')
  )
  

• 创建并启动线程:

  thread = threading.Thread(target=writer, args=(streamlink_process, ffmpeg_process))
  thread.start()
  

---

完整的代码示例:

import numpy as np
import subprocess as sp
import threading
import cv2
import ffmpeg
#stream_url = 'https://www.nimo.tv/v/v-1712291636586087045'
stream_url = 'https://www.twitch.tv/esl_csgo'
# Assume video resolution is known.
width, height = 1920, 1080

# Writer thread (read from streamlink and write to FFmpeg in chunks of 1024 bytes).
def writer(streamlink_proc, ffmpeg_proc):
while (not streamlink_proc.poll()) and (not ffmpeg_proc.poll()):
try:
chunk = streamlink_proc.stdout.read(1024)
ffmpeg_proc.stdin.write(chunk)
except (BrokenPipeError, OSError) as e:
pass

streamlink_args = [r'c:Program Files (x86)Streamlinkbinstreamlink.exe', stream_url, "best", "-O"]  # Windows executable downloaded from: https://github.com/streamlink/streamlink/releases/tag/2.4.0
streamlink_process = sp.Popen(streamlink_args, stdout=sp.PIPE)  # Execute streamlink as sub-process

# Execute FFmpeg sub-process with URL as input and raw (BGR) output format.
ffmpeg_process = (
ffmpeg
.input('pipe:')
.video
.output('pipe:', format='rawvideo', pix_fmt='bgr24')
.run_async(pipe_stdin=True, pipe_stdout=True) # In case ffmpeg in not in executable path, add cmd=fullpath like: .run_async(pipe_stdout=True, cmd=r'c:FFmpegbinffmpeg.exe')
)

thread = threading.Thread(target=writer, args=(streamlink_process, ffmpeg_process))
thread.start()

# Read decoded video (frame by frame), and display each frame (using cv2.imshow)
while True:
# Read raw video frame from stdout as bytes array.
in_bytes = ffmpeg_process.stdout.read(width * height * 3)
if not in_bytes:
break
# Transform the byte read into a NumPy array
frame = np.frombuffer(in_bytes, np.uint8).reshape([height, width, 3])
# Display the frame
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
ffmpeg_process.stdout.close()
ffmpeg_process.wait()
#streamlink_process.stdin.close()
streamlink_process.kill()
cv2.destroyAllWindows()

---

指出:

• 代码示例使用了一个到twitch的链接。用电视代替nimo。电视，因为"Nimo破坏了流链接插件"。
• 样本假设宽度和高度事先已知。
• 示例在Windows 10上进行了测试(在安装Streamlink后执行Streamlink .exe:r'c:Program Files (x86)Streamlinkbinstreamlink.exe')。