Phase3:带麦克风输入的着色器(Web)

Question

我想用Phaser 3做一个音频输入vizualizer，我正在尝试将麦克风输入到着色器，但我找不到让它工作的方法。

我对着色器有基本的了解，我可以处理图像纹理，但我真的不知道如何提供声音。我检查了three.js中的一个工作示例：three.jswebaudio-visualizer，我已经设法将麦克风的声音输入作为1024个数字的Uint8Array。

这是我正在使用的着色器：

// simplesound.gsl.js
#ifdef GL_ES
precision highp float;
#endif
precision mediump float;
uniform vec2 resolution;
uniform sampler2D iChannel0;
varying vec2 fragCoord;
void main() {
vec2 uv = fragCoord.xy / resolution.xy;
vec2 mu = texture2D(iChannel0, uv).rg;
float y = uv.y - mu.x;
y = smoothstep(0., 0.02, abs(y - 0.1));
gl_FragColor = vec4(y);
}

这是我的场景代码，试图让它发挥作用：

import Phaser from 'phaser';
// This will provide the array mentioned above with code that will use `navigator.getUserMedia`.
import { setupAudioContext } from '../audiostream';
export default class MainScene2 extends Phaser.Scene {
constructor() {
super({ key: 'MainScene2' });
}
preload() {
this.load.glsl('simplesound', '/static/simplesound.glsl.js');
}
create() {
this.shader = this.add.shader('simplesound', 400, 300, 800, 600);
// When the user presses the 'g' key we will start listening for mic input
const GKey = this.input.keyboard.addKey('G');
GKey.on('down', () => {
setupAudioContext((array) => {
// this array is the array mentioned above, in the three.js example they do something like creating
// a texture from this input and providing that texture to the shader uniform. I tried different things but
// nothing worked :(
//
// I tried using this.shader.setChannel0 and this.shader.setUniform but nothing seems to work as well.
});
});
}
}

我已经尝试了一段时间，但什么都没得到：(

Answer 1

对于没有着色器的可能解决方案，只使用phaser和javascript可能会像这样(实际上没有着色器，但我也对着色器版本的外观非常感兴趣(。

在这个演示中，我使用的是来自音频文件的数据为了使其适用于您的用例，您只需要将麦克风数据插入data变量即可

演示：
(代码中的注释用于突出主要思想(
单击并等待几秒钟顺便说一句：我添加了一些截图，让演示更有活力。

document.body.style = 'margin:0;';

var data = [];
var playing =  -1;
var audioContext = new (window.AudioContext || window.webkitAudioContext)();
var analyser = audioContext.createAnalyser();
var buffer;
var source;
var url = 'https://labs.phaser.io/assets/audio/Rossini - William Tell Overture (8 Bits Version)/left.ogg'
// START Audio part for Demo
function loadAudio() {
var request = new XMLHttpRequest();
request.open('GET', url, true);
request.responseType = 'arraybuffer';
request.onload = function() {
audioContext.decodeAudioData(request.response, function(buf) {
buffer = buf;
playAudio();
});
};
request.send();
}

function playAudio() {
source = audioContext.createBufferSource();
source.buffer = buffer;
source.connect(audioContext.destination);
source.connect(analyser);
source.start(0);
}
// END Audio part for Demo
var config = {
type: Phaser.AUTO,
width: 536,
height: 183,
scene: {
create,
update
},
banner: false
}; 
var game = new Phaser.Game(config);
// this would be the varibale that should be updated from the audio source
var markers;
var createRandomData = true;
function create () {
// Start create Marker texture 
// this could be remove if you want to load an actual image
let g = this.make.graphics({x: 0, y: 0, add: false});

g.lineStyle(10, 0xffffff);
g.beginPath();
g.moveTo(0, 0);
g.lineTo(50, 0);
g.strokePath();
g.generateTexture('marker', 30, 10);
// End create Marker texture 

// Create the markers
// the repeat property sets how many markers you want to display, if you want all 1024 => that would be your value
markers = this.add.group({ key: 'marker', repeat: 50,
setXY: { x: 10, y: 10, stepX: 35 }, setOrigin: { x: 0, y: 0}});

this.add.rectangle(10, 10, 180, 20, 0).setOrigin(0);
let label = this.add.text( 10, 10, 'Click to start music', {color: 'red', fontSize:'20px', fontStyle:'bold'} )

// start and stop the playback of music     
this.input.on('pointerdown', function () {
switch (playing) {
case -1:
loadAudio();
playing = 1;
label.setText('Click to stop music');
break;
case 0:
playAudio();
playing = 1;
label.setText('Click to stop music');
break;
case 1:
source.stop();
playing = 0;
label.setText('Click to start music');
break;
}   
}); 
}
function update(){
if (markers){
// here we update the y-position of the marker in depending on the value of the data. 
// ( min y = 10 and max y ~ 245)
markers.children.iterate(function (child, idx) {
child.y = 10 + (config.height - 20) / 255 * data[idx];

// you could even add some camera shake, for more effect
if(idx < 3 && data[idx] > 253){
this.cameras.main.shake(30);
}
}, this);

// if the analyser is valid and updates the data variable
// this part could some where else, I just wanted to keep the code concise
if(analyser){
var spectrums = new Uint8Array(analyser.frequencyBinCount);
analyser.getByteFrequencyData(spectrums);
// convert data to a plain array and updating the data variable
data = [].slice.call(spectrums);
}
}
}

<script src="https://cdn.jsdelivr.net/npm/phaser@3.55.2/dist/phaser.js"></script>

基本上这个应用程序"仅">根据从音频文件加载的音频数组返回的值，更改每个标记的Y位置
免责声明：这是一个粗略的演示代码，如果应该在生产中使用，可能需要一些清理/改进

Answer 2

这是一个有效的解决方案，使用上述着色器，通过相位器

在对glsl进行了短暂的深入研究之后，在开始理解着色器和周围的概念之后，我发现了着色器是如何与phaser一起工作的(至少对于这个任务来说(，它实际上是非常优雅和直接的，它是如何实现的。

基本上你"只是">需要：

1. 在preload中加载着色器

   this.load.glsl('simplesound', 'simplesound.glsl.js');
   

2. 将着色器添加到scene，在create函数中，例如

   this.shader = this.add.shader('simplesound', x, y, width, height);
   

3. 在create函数
   中创建两个要传递到着色器的纹理(以便可以交换它们((因为选择一个符合两个大小的幂的纹理大小(

   this.g = this.make.graphics({ x: 0, y: 0, add: false });
   this.g.generateTexture('tex0', 64, 64);
   this.g.generateTexture('tex1', 64, 64);
   

4. 在update函数中，创建一个表示音频数据的纹理

   ...
   var spectrums = new Uint8Array(analyser.frequencyBinCount);
   analyser.getByteFrequencyData(spectrums);
   data = [].slice.call(spectrums);
   ...
   let g = this.make.graphics({add:false});
   data.forEach( (value, idx ) => {
   g.fillStyle(value <<16 );
   g.fillRect(idx * 10, 0, 10, value);
   }); 
   

5. 最后更新下一个纹理，并将新创建的纹理的新纹理密钥传递给shader

   if(this.shader.getUniform('iChannel0').textureKey == 'tex0'){
   this.textures.remove('tex1');
   g.generateTexture('tex1', 64, 64);
   this.shader.setChannel0('tex1');
   } else {
   this.textures.remove('tex0');
   g.generateTexture('tex0', 64, 64);
   this.shader.setChannel0('tex0');
   }
   

当然，这段代码可以通过函数、更好的变量和纹理键命名进行优化，但这是留给读者的。

小型工作演示：
(使用音频文件，而不是麦克风，并且基于我之前回答的代码(

document.body.style = 'margin:0;';
const TEXTURE_SIZE = 128;
const glslScript = `#ifdef GL_ES
precision highp float;
#endif
precision mediump float;
uniform vec2 resolution;
uniform sampler2D iChannel0;
varying vec2 fragCoord;
void main() {
vec2 uv = fragCoord.xy / resolution.xy;
vec2 mu = texture2D(iChannel0, uv).rg;
float y = uv.y - mu.x;
y = smoothstep(0., 0.02, abs(y - 0.1));
gl_FragColor = vec4(y);
}`;
var playing = -1;
var audioContext = new (window.AudioContext || window.webkitAudioContext)();
var analyser = audioContext.createAnalyser();
var source;
var url = 'https://labs.phaser.io/assets/audio/Rossini - William Tell Overture (8 Bits Version)/left.ogg'
// START Audio part for Demo
function loadAudio() {
var request = new XMLHttpRequest();
request.open('GET', url, true);
request.responseType = 'arraybuffer';
request.onload = function () {
audioContext.decodeAudioData(request.response, function (buf) {
playAudio(buf);
});
};
request.send();
}
function playAudio(buffer) {
source = audioContext.createBufferSource();
source.buffer = buffer;
source.connect(audioContext.destination);
source.connect(analyser);
source.start(0);
}
// END Audio part for Demo
var config = {
type: Phaser.WEBGL,
width: 536,
height: 183,
scene: {
create,
update
},
banner: false
};
var game = new Phaser.Game(config);
function create() {
this.add.text(10, 10, 'Click to start and stop the music')
.setColor('#000000')
.setOrigin(0)
.setDepth(1000)
.setFontFamily('Arial');

this.add.rectangle(0, 0, config.width, 40, 0xffffff)
.setDepth(999)
.setOrigin(0);
var baseShader = new Phaser.Display.BaseShader('shader', glslScript);
this.shader = this.add.shader(baseShader, 0, 10, config.width, config.height - 10)
.setOrigin(0);
// start and stop the playback of music     
this.input.on('pointerdown', function () {
switch (playing) {
case -1:
loadAudio();
playing = 1;
break;
case 0:
playAudio();
playing = 1;
break;
case 1:
source.stop();
playing = 0;
break;
}
});
}
function update() {
if (analyser) {
var spectrums = new Uint8Array(analyser.frequencyBinCount);
analyser.getByteFrequencyData(spectrums);
let data = [].slice.call(spectrums);
let g = this.make.graphics({ add: false });
data.forEach((value, idx) => {
g.fillStyle(value << 16);
g.fillRect(idx * 10, 0, 10, value);
});

let textureName = this.shader.getUniform('iChannel0').textureKey == 'tex0' ? 'tex1' : 'tex0';

if(this.textures.exists(textureName)){
this.textures.remove(textureName);
}
g.generateTexture(textureName, TEXTURE_SIZE, TEXTURE_SIZE);
this.shader.setChannel0(textureName);

}
}

<script src="https://cdn.jsdelivr.net/npm/phaser@3.55.2/dist/phaser.js"></script>