javascript Phaser3：带麦克风输入的着色器(Web)

cu6pst1q  于 2022-12-28  发布在  Java
关注(0)|答案(2)|浏览(151)

我想做一个音频输入可视化与相位器3，我试图让麦克风输入到一个着色器，但我找不到一种方法来使它的工作。
我对着色器有一个基本的了解，我可以处理图像纹理，但我真的不知道如何提供声音。我检查了一个用three.js制作的工作示例：three.js webaudio - visualizer和我已经设法从麦克风获得声音输入作为1024个数字的Uint8Array。
下面是我使用的着色器：

// simplesound.gsl.js
#ifdef GL_ES
precision highp float;
#endif

precision mediump float;
uniform vec2 resolution;
uniform sampler2D iChannel0;

varying vec2 fragCoord;

void main() {
  vec2 uv = fragCoord.xy / resolution.xy;
  vec2 mu = texture2D(iChannel0, uv).rg;

  float y = uv.y - mu.x;
  y = smoothstep(0., 0.02, abs(y - 0.1));

  gl_FragColor = vec4(y);
}

这是我的场景代码，尝试让它工作：

import Phaser from 'phaser';
// This will provide the array mentioned above with code that will use `navigator.getUserMedia`.
import { setupAudioContext } from '../audiostream';

export default class MainScene2 extends Phaser.Scene {
  constructor() {
    super({ key: 'MainScene2' });
  }

  preload() {
    this.load.glsl('simplesound', '/static/simplesound.glsl.js');
  }

  create() {
    this.shader = this.add.shader('simplesound', 400, 300, 800, 600);

    // When the user presses the 'g' key we will start listening for mic input
    const GKey = this.input.keyboard.addKey('G');

    GKey.on('down', () => {
      setupAudioContext((array) => {
        // this array is the array mentioned above, in the three.js example they do something like creating
        // a texture from this input and providing that texture to the shader uniform. I tried different things but
        // nothing worked :(
        //
        // I tried using this.shader.setChannel0 and this.shader.setUniform but nothing seems to work as well.
      });
    });
  }
}

我已经尝试了一段时间，但无法得到任何东西：（

JavaScript

2条答案

按热度按时间
wyyhbhjk

wyyhbhjk1#

对于一个可能的解决方案没有着色器，通过使用只使用相位器和javascript可以看起来像这样 （真的没有着色器，但我也真的很感兴趣的着色器版本，会看起来像）。
在这个演示中，我使用的是来自音频文件的数据。* 因此，它适用于您的用例，您只需将麦克风数据插入data变量。*

    • 演示：**
  • （代码中的注解用于突出主要思想）*

点击并等待几秒钟。**顺便说一句：**我添加了一些屏幕抖动，给演示更多的果汁。

document.body.style = 'margin:0;';
    
    var data = [];
    var playing =  -1;
    var audioContext = new (window.AudioContext || window.webkitAudioContext)();
    var analyser = audioContext.createAnalyser();
    var buffer;
    var source;
    var url = 'https://labs.phaser.io/assets/audio/Rossini - William Tell Overture (8 Bits Version)/left.ogg'

    // START Audio part for Demo
    function loadAudio() {
        var request = new XMLHttpRequest();
        request.open('GET', url, true);
        request.responseType = 'arraybuffer';
        request.onload = function() {
            audioContext.decodeAudioData(request.response, function(buf) {
                buffer = buf;
                playAudio();
            });
        };
        request.send();
    }
    
    function playAudio() {
        source = audioContext.createBufferSource();
        source.buffer = buffer;
        source.connect(audioContext.destination);
        source.connect(analyser);
        source.start(0);
    }

    // END Audio part for Demo

    var config = {
        type: Phaser.AUTO,
        width: 536,
        height: 183,
        scene: {
            create,
            update
        },
        banner: false
    }; 

    var game = new Phaser.Game(config);

    // this would be the varibale that should be updated from the audio source
    var markers;
    var createRandomData = true;

    function create () {
        // Start create Marker texture 
        // this could be remove if you want to load an actual image
        let g = this.make.graphics({x: 0, y: 0, add: false});
        
        g.lineStyle(10, 0xffffff);
        g.beginPath();
        g.moveTo(0, 0);
        g.lineTo(50, 0);

        g.strokePath();

        g.generateTexture('marker', 30, 10);
        // End create Marker texture 
    
        // Create the markers
        // the repeat property sets how many markers you want to display, if you want all 1024 => that would be your value
        markers = this.add.group({ key: 'marker', repeat: 50,
            setXY: { x: 10, y: 10, stepX: 35 }, setOrigin: { x: 0, y: 0}});
        
        this.add.rectangle(10, 10, 180, 20, 0).setOrigin(0);
        let label = this.add.text( 10, 10, 'Click to start music', {color: 'red', fontSize:'20px', fontStyle:'bold'} )
        
        // start and stop the playback of music     
        this.input.on('pointerdown', function () {
            switch (playing) {
                case -1:
                    loadAudio();
                    playing = 1;
                    label.setText('Click to stop music');
                    break;
                case 0:
                    playAudio();
                    playing = 1;
                    label.setText('Click to stop music');
                    break;
                case 1:
                    source.stop();
                    playing = 0;
                    label.setText('Click to start music');
                    break;
            }   
        }); 

    }

    function update(){
        if (markers){
            // here we update the y-position of the marker in depending on the value of the data. 
            // ( min y = 10 and max y ~ 245)
            markers.children.iterate(function (child, idx) {
                child.y = 10 + (config.height - 20) / 255 * data[idx];
                
                // you could even add some camera shake, for more effect
                if(idx < 3 && data[idx] > 253){
                    this.cameras.main.shake(30);
                }
            }, this);
            
            // if the analyser is valid and updates the data variable
            // this part could some where else, I just wanted to keep the code concise
            if(analyser){
              var spectrums = new Uint8Array(analyser.frequencyBinCount);
              analyser.getByteFrequencyData(spectrums);

              // convert data to a plain array and updating the data variable
              data = [].slice.call(spectrums);
            }
        }
    }
<script src="https://cdn.jsdelivr.net/npm/phaser@3.55.2/dist/phaser.js"></script>

基本上，此应用程序 "仅" 根据从音频数组（从音频文件加载）返回的值更改每个标记的Y位置。

***免责声明：*这是粗略的演示代码，如果要在生产中使用，可能需要进行一些清理/改进。

赞(0）分享  回复(0）举报  2022-12-28
rekjcdws

rekjcdws2#

    • 这是一个工作解决方案，使用上述着色器，通过相位器**

在对***glsl***进行了一次短暂的深入研究之后，在开始理解着色器和周围的概念之后，我发现了着色器是如何与相位器一起工作的（至少在这个任务中），它实际上是非常优雅和简单的，它是如何实现的。

    • 基本上**您 "只需" 需要：

1.在preload中加载着色器

this.load.glsl('simplesound', 'simplesound.glsl.js');

1.将着色器添加到scene，例如在create函数中

this.shader = this.add.shader('simplesound', x, y, width, height);

1.在create函数中创建两个要传递给着色器的纹理（以便可以交换它们）

  • （请注意选择符合2的幂大小的纹理大小）*
this.g = this.make.graphics({ x: 0, y: 0, add: false });
 this.g.generateTexture('tex0', 64, 64);
 this.g.generateTexture('tex1', 64, 64);

1.在update函数中，创建表示***音频数据***的纹理

...
 var spectrums = new Uint8Array(analyser.frequencyBinCount);
 analyser.getByteFrequencyData(spectrums);
 data = [].slice.call(spectrums);
 ...
 let g = this.make.graphics({add:false});
 data.forEach( (value, idx ) => {
     g.fillStyle(value <<16 );
     g.fillRect(idx * 10, 0, 10, value);
 });

1.最后更新 * next * 纹理，并将新创建的纹理的新texture-key传递给shader

if(this.shader.getUniform('iChannel0').textureKey == 'tex0'){
     this.textures.remove('tex1');
     g.generateTexture('tex1', 64, 64);
     this.shader.setChannel0('tex1');
 } else {
     this.textures.remove('tex0');
     g.generateTexture('tex0', 64, 64);
     this.shader.setChannel0('tex0');
 }

当然，这段代码可以用函数、更好的变量和纹理键命名来优化，但这是留给读者的。

    • 小型工作演示：**
  • （使用音频文件，而不是麦克风，是基于我以前的答案的代码）*
document.body.style = 'margin:0;';

const TEXTURE_SIZE = 128;

const glslScript = `#ifdef GL_ES
precision highp float;
#endif

precision mediump float;
uniform vec2 resolution;
uniform sampler2D iChannel0;

varying vec2 fragCoord;

void main() {
  vec2 uv = fragCoord.xy / resolution.xy;
  vec2 mu = texture2D(iChannel0, uv).rg;

  float y = uv.y - mu.x;
  y = smoothstep(0., 0.02, abs(y - 0.1));

  gl_FragColor = vec4(y);
}`;

var playing = -1;
var audioContext = new (window.AudioContext || window.webkitAudioContext)();
var analyser = audioContext.createAnalyser();
var source;
var url = 'https://labs.phaser.io/assets/audio/Rossini - William Tell Overture (8 Bits Version)/left.ogg'

// START Audio part for Demo
function loadAudio() {
    var request = new XMLHttpRequest();
    request.open('GET', url, true);
    request.responseType = 'arraybuffer';
    request.onload = function () {
        audioContext.decodeAudioData(request.response, function (buf) {
            playAudio(buf);
        });
    };
    request.send();
}

function playAudio(buffer) {
    source = audioContext.createBufferSource();
    source.buffer = buffer;
    source.connect(audioContext.destination);
    source.connect(analyser);
    source.start(0);
}

// END Audio part for Demo
var config = {
    type: Phaser.WEBGL,
    width: 536,
    height: 183,
    scene: {
        create,
        update
    },
    banner: false
};

var game = new Phaser.Game(config);

function create() {
    this.add.text(10, 10, 'Click to start and stop the music')
        .setColor('#000000')
        .setOrigin(0)
        .setDepth(1000)
        .setFontFamily('Arial');
        
    this.add.rectangle(0, 0, config.width, 40, 0xffffff)
        .setDepth(999)
        .setOrigin(0);

    var baseShader = new Phaser.Display.BaseShader('shader', glslScript);

    this.shader = this.add.shader(baseShader, 0, 10, config.width, config.height - 10)
        .setOrigin(0);

    // start and stop the playback of music     
    this.input.on('pointerdown', function () {
        switch (playing) {
            case -1:
                loadAudio();
                playing = 1;
                break;
            case 0:
                playAudio();
                playing = 1;
                break;
            case 1:
                source.stop();
                playing = 0;
                break;
        }
    });
}

function update() {

    if (analyser) {
        var spectrums = new Uint8Array(analyser.frequencyBinCount);
        analyser.getByteFrequencyData(spectrums);

        let data = [].slice.call(spectrums);

        let g = this.make.graphics({ add: false });
        data.forEach((value, idx) => {
            g.fillStyle(value << 16);
            g.fillRect(idx * 10, 0, 10, value);
        });
        
        let textureName = this.shader.getUniform('iChannel0').textureKey == 'tex0' ? 'tex1' : 'tex0';
        
        if(this.textures.exists(textureName)){
            this.textures.remove(textureName);
        }
        g.generateTexture(textureName, TEXTURE_SIZE, TEXTURE_SIZE);
        this.shader.setChannel0(textureName);
      
    }
}
<script src="https://cdn.jsdelivr.net/npm/phaser@3.55.2/dist/phaser.js"></script>
赞(0）分享  回复(0）举报  2022-12-28
我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com