使用OpenCV和ffpyplayer进行音频和视频同步

Question

我目前正在处理一个"游戏"；这允许用户在观看视频时移动圆圈的位置。视频显示两个人轮流发言。用户的任务是改变出现在活动说话者面前的圆圈的位置。当这种情况发生时，在某个时刻，我计划在用户没有注意到的情况下更改视频；游戏"；圆圈继续显示。

为了达到这个目的，我编写了以下代码。该代码接受用户的输入，并将所有数据发送到TCP服务器，并将信息打印到记录器文件中。但我遇到了一个问题。首先，音频和视频不同步，即使使用最低值的waitkey(1)，音频也比视频快

如有任何关于如何解决此问题的帮助，我们将不胜感激。提前谢谢。

PS-我使用的是Visual Studio代码，我的python版本是3.9.6 64位。

import cv2 as cv #import the OpenCV library
import numpy as np #Import Numpy Library
import socket  # socket creation for Telnet
from datetime import datetime
from telnetlib import Telnet #telnet client
from ffpyplayer.player import MediaPlayer #ffpyplayer for playing audio
current_date_time = datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
HOST = '127.0.0.1'  # The remote host
PORT = 4212  # The same port as used by the server
TCP_PORT = 9999  # port used to connect with the server file
def send_data(message, s, output):
s.sendall(message.encode())
data = s.recv(1024)
output.write('n'+ current_date_time+' '+message+ 'n')
return data
def circle(frame, left):
if left:
cv.circle(frame,(450,250),20,(255,255,255),50)
if not left:
cv.circle(frame,(1400,250),20,(255,255,255),50)
def video():
cap1 = cv.VideoCapture('P1.mp4') # the video that we want
player = MediaPlayer('P1.mp4')
circle_is_left = True
if (cap1.isOpened()== False):
print("Error opening video 1")  
while (cap1.isOpened()):
ret,frame = cap1.read() #capture frame-by-frame video
audio_frame,val=player.get_frame() # capture frame-by-frame audio
if ret== True:
key_pressed = cv.waitKey(1)
if key_pressed == ord(' '): #pressing space bar ends the video
with open('out.txt', 'a') as output:
send_data('video 1 is changed',s,output)
break
elif key_pressed == 2: #left key pressed changes circle to lett
circle_is_left = True
with open('out.txt', 'a') as output:
send_data('left',s,output)
elif key_pressed == 3: # right key pressed changes circle to right
circle_is_left = False
with open('out.txt', 'a') as output:
send_data('Right ',s,output)
circle(frame, circle_is_left) #display the circle at all times
cv.imshow('cap1',frame) #display resulting frame 
if val != 'eof' and audio_frame is not None:
img,t = audio_frame
cap1.release()
cv.destroyAllWindows()

cap2 = cv.VideoCapture('P2.mov') # the video that we want
player2 = MediaPlayer('P2.mov')
circle_is_left = True
if (cap2.isOpened()== False):
print("Error opening video 2")  
while (cap2.isOpened()):
ret,frame = cap2.read() #capture frame-by-frame video
audio_frame,val=player2.get_frame() # capture frame-by-frame audio
if ret== True:
key_pressed = cv.waitKey(1)
if key_pressed == ord(' '): #pressing space bar ends the video
with open('out.txt', 'a') as output:
send_data('video 1 is changed',s,output)
break
elif key_pressed == 2: #left key pressed changes circle to lett
circle_is_left = True
with open('out.txt', 'a') as output:
send_data('left',s,output)
elif key_pressed == 3: # right key pressed changes circle to right
circle_is_left = False
with open('out.txt', 'a') as output:
send_data('Right ',s,output)
circle(frame, circle_is_left) #display the circle at all times
cv.imshow('cap2',frame) #display resulting frame 
if val != 'eof' and audio_frame is not None:
img,t = audio_frame
cap2.release()
cv.destroyAllWindows()
def main():
print("Game1.py is connected to TCP server")
video()

if __name__=='__main__':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, TCP_PORT))
main()

Answer 1

您可以使用cv2.CAP_PROP_POS_MSEC同步视频和音频，如下所示：

import cv2
import time
from ffpyplayer.player import MediaPlayer

def video(file):
cap = cv2.VideoCapture(file)
player = MediaPlayer(file)
start_time = time.time()
while cap.isOpened():
ret, frame = cap.read()
if not ret:
break
_, val = player.get_frame(show=False)
if val == 'eof':
break
cv2.imshow(file, frame)
elapsed = (time.time() - start_time) * 1000  # msec
play_time = int(cap.get(cv2.CAP_PROP_POS_MSEC))
sleep = max(1, int(play_time - elapsed))
if cv2.waitKey(sleep) & 0xFF == ord("q"):
break
player.close_player()
cap.release()
cv2.destroyAllWindows()

参见https://github.com/otamajakusi/opencv_video_with_audio