Body segmentation filter using ml5

Question

我正在尝试制作一个网络摄像头过滤器，它可以进行身体分割，以掩盖一个人身后的背景，并在背景中放置一张照片。类似Zoom或其他平台上的背景选项。我曾想过使用ml5和BodyPix，但我只达到了这一点(https://github.com/ml5js/ml5-library/blob/main/examples/p5js/BodyPix/BodyPix_Webcam/sketch.js)：

let bodypix;
let video;
let segmentation;
const options = {
outputStride: 8, // 8, 16, or 32, default is 16
segmentationThreshold: 0.3, // 0 - 1, defaults to 0.5
};
function preload() {
bodypix = ml5.bodyPix(options);
}
function setup() {
createCanvas(320, 240);
// load up your video
video = createCapture(VIDEO, videoReady);
video.size(width, height);

}
function videoReady() {
bodypix.segment(video, gotResults);
}
function draw() {
background(0);
if (segmentation) {
image(segmentation.backgroundMask, 0, 0, width, height);
}
}
function gotResults(error, result) {
if (error) {
console.log(error);
return;
}
segmentation = result;
bodypix.segment(video, gotResults);
}

有人能为在背景中放置图像找到解决方案吗？也许还有另一种方法来编码这样的东西？谢谢

Answer 1

你非常接近。从segment函数返回的对象为您提供了两个PImage，它们只是用于此目的的彼此的反向掩码。backgroundMask屏蔽了背景，所有未识别的物体都有0 alpha的像素。personMask的作用正好相反，只显示背景。请参阅此处的参考资料：https://github.com/ml5js/ml5-library/blob/main/docs/reference/bodypix.md

let bodypix;
let video;
let segmentation;
let backgroundImage; // start with the variable
const options = {
outputStride: 8, 
segmentationThreshold: 0.3,
};
function preload() {
bodypix = ml5.bodyPix(options);
backgroundImage = loadImage('YOURFILENAME'); // load your file here
}
function setup() {
createCanvas(320, 240);
video = createCapture(VIDEO, videoReady);
video.size(width, height);
}
function videoReady() {
bodypix.segment(video, gotResults);
}
function draw() {
background(0);
image(backgroundImage,0,0,width,height); // this is the background drawn before the subject is drawn
if (segmentation) {
image(segmentation.backgroundMask, 0, 0, width, height); 
}
}
function gotResults(error, result) {
if (error) {
console.log(error);
return;
}
segmentation = result;
bodypix.segment(video, gotResults);
}

以下是一个工作示例：https://editor.p5js.org/[e-mail]/sketches/af83XcLXZ