Create an effect where the musical frequency of your voice, the left and right of your vocal tract changes with the video capture, with your nose, and with your expression

using spring effect preview work

Put music library after head

11.13 code

let video;
let bodyPose;
let poses = [];
let soundzzs;
let noseXp = 0;
let noseYp = 0;

// 添加默认中心点位置
let centerX;
let centerY;
// 修改音频参数范围，使中心点值为1
const MIN_RATE = 0.5;  // 最低音调为0.5倍
const MAX_RATE = 1.5;  // 最高音调为1.5倍
const MIN_VOLUME = 0;  // 最小音量为0
const MAX_VOLUME = 2.0;  // 最大音量为2倍

function preload() {
  try {
    soundzzs = loadSound('ShuaiLetter.mp3', 
      () => {
        console.log('Sound loaded successfully');
        // 设置音频循环播放并初始化为正常值
        soundzzs.loop();
        soundzzs.rate(1.0);
        soundzzs.setVolume(1.0);
      },
      (err) => console.error('Sound failed to load:', err)
    );
  } catch (error) {
    console.error('Error in preload:', error);
  }
}

async function setup() {
  createCanvas(windowWidth - 50, windowHeight - 50);
  background(1, 1, 1);
  
  // 设置画布中心点
  centerX = width / 2;
  centerY = height / 2;
  
  // 设置视频
  video = createCapture(VIDEO);
  video.size(width, height);
  video.hide();
  
  try {
    bodyPose = await ml5.bodyPose("MoveNet");
    console.log('Model Loaded Successfully');
    bodyPose.detectStart(video, gotPoses);
  } catch (error) {
    console.error('Error loading model:', error);
  }
}

function gotPoses(results) {
  poses = results;
  if (results.length > 0) {
    updateAudioParameters();
  }
}

function updateAudioParameters() {
  if (poses.length > 0 && poses[0].nose) {
    // 计算鼻子位置相对于中心点的偏移比例
    let xOffset = (noseXp - centerX) / (width / 2); // -1 到 1 的范围
    let yOffset = (noseYp - centerY) / (height / 2); // -1 到 1 的范围
    
    // 计算播放速率（影响音调）
    // 在中心点时 rate = 1
    let rate;
    if (xOffset < 0) {
      rate = map(xOffset, -1, 0, MIN_RATE, 1.0);
    } else {
      rate = map(xOffset, 0, 1, 1.0, MAX_RATE);
    }
    rate = constrain(rate, MIN_RATE, MAX_RATE);
    
    // 计算音量
    // 在中心点时 volume = 1
    let volume;
    if (yOffset > 0) {
      volume = map(yOffset, 0, 1, 1.0, MIN_VOLUME);
    } else {
      volume = map(yOffset, -1, 0, MAX_VOLUME, 1.0);
    }
    volume = constrain(volume, MIN_VOLUME, MAX_VOLUME);
    
    // 应用音频参数
    if (soundzzs && soundzzs.isLoaded()) {
      soundzzs.rate(rate);
      soundzzs.setVolume(volume);
    }
  }
}

function draw() {
  // 绘制视频
  image(video, 0, 0, width, height);
  
  // 半透明背景
  background(1, 1, 1, 100);
  
  // 绘制中心点参考线
  stroke(255, 100);
  line(centerX, 0, centerX, height);
  line(0, centerY, width, centerY);
  
  // 只有当有姿势数据时才显示音乐符号
  if (poses.length > 0 && poses[0].nose) {
    musicSignPosition();
    musicSignShow();
    displayAudioParams();
  }
}

function displayAudioParams() {
  // 显示音频参数信息
  fill(255);
  noStroke();
  textSize(16);
  let rate = soundzzs.rate();
  let volume = soundzzs.getVolume();
  text(`Rate: ${rate.toFixed(2)}`, 10, 20);
  text(`Volume: ${volume.toFixed(2)}`, 10, 40);
  
  // 显示中心点位置指示
  if (Math.abs(noseXp - centerX) < 20 && Math.abs(noseYp - centerY) < 20) {
    fill(0, 255, 0);
    text("Center Position!", 10, 60);
  }
}

function musicSignShow() {
  let sizeSign = 80;
  textSize(sizeSign);
  fill(255);
  text("🎵", noseXp - sizeSign/2, noseYp + sizeSign/4);
}

function musicSignPosition() {
  if (poses.length > 0) {
    let pose = poses[0];
    if (pose.nose) {
      fill(236, 1, 90);
      noStroke();
      noseXp = pose.nose.x;
      noseYp = pose.nose.y;
    }
  }
}

function windowResized() {
  resizeCanvas(windowWidth - 50, windowHeight - 50);
  centerX = width / 2;
  centerY = height / 2;
}

with claude help

next week add this effect in the game

音乐的录入抓取数据 by hirgen -p5.js Web Editor

let video;
let bodyPose;
let poses = [];
let soundzzs;
let noseXp = 0;
let noseYp = 0;
let amplitude;
// 中心点位置
let centerX;
let centerY;
// 目标点位置
let targetX;
let targetY;
// 目标点移动速度
let speedX;
let speedY;
const SPEED = 2;
const DIRECTION_CHANGE_INTERVAL = 3000;
let lastDirectionChange = 0;

const MIN_RATE = 0.5;
const MAX_RATE = 1.5;
const MIN_VOLUME = 0;
const MAX_VOLUME = 2.0;
const CATCH_DISTANCE = 50;

function preload() {
  try {
    soundzzs = loadSound('ShuaiLetter.mp3', 
      () => {
        console.log('Sound loaded successfully');
        soundzzs.loop();
        soundzzs.rate(1.0);
        soundzzs.setVolume(1.0);
      },
      (err) => console.error('Sound failed to load:', err)
    );
  } catch (error) {
    console.error('Error in preload:', error);
  }
}

async function setup() {
  createCanvas(windowWidth - 50, windowHeight - 50);
  background(1, 1, 1);
  
  // 初始化目标点位置为画布中心
  targetX = width / 2;
  targetY = height / 2;
  
  updateTargetDirection();
  
  // 设置视频捕捉并镜像
  video = createCapture({
    video: {
      facingMode: "user",
      width: { ideal: width },
      height: { ideal: height }
    }
  });
  video.size(width, height);
  video.hide();
  
  //get music wave
  amplitude=new p5.Amplitude();
  
  try {
    // 初始化 ML5 bodyPose，设置为镜像模式
    bodyPose = await ml5.bodyPose("MoveNet", { flipHorizontal: true });
    console.log('Model Loaded Successfully');
    bodyPose.detectStart(video, gotPoses);
  } catch (error) {
    console.error('Error loading model:', error);
  }
}

function draw() {
  moveTarget();
  
  // 使用 push() 和 pop() 来隔离变换效果
  push();
  // 镜像画面
  translate(width, 0);
  scale(-1, 1);
  // 绘制视频
  image(video, 0, 0, width, height);
  pop();
  
  // 半透明背景
  background(1, 1, 1, 100);
  
  // 绘制目标点
  stroke(0, 255, 0);
  noFill();
  circle(targetX, targetY, CATCH_DISTANCE * 2);
  fill(0, 255, 0);
  noStroke();
  circle(targetX, targetY, 10);
  
  if (poses.length > 0 && poses[0].nose) {
    musicSignPosition();
    musicSignShow();
    displayAudioParams();
  }
}

// 其余函数保持不变...
function updateTargetDirection() {
  let angle = random(TWO_PI);
  speedX = cos(angle) * SPEED;
  speedY = sin(angle) * SPEED;
}

function moveTarget() {
  targetX += speedX;
  targetY += speedY;
  
  if (targetX < CATCH_DISTANCE || targetX > width - CATCH_DISTANCE) {
    speedX *= -1;
  }
  if (targetY < CATCH_DISTANCE || targetY > height - CATCH_DISTANCE) {
    speedY *= -1;
  }
  
  if (millis() - lastDirectionChange > DIRECTION_CHANGE_INTERVAL) {
    updateTargetDirection();
    lastDirectionChange = millis();
  }
}

function gotPoses(results) {
  poses = results;
  if (results.length > 0) {
    updateAudioParameters();
  }
}

function updateAudioParameters() {
  if (poses.length > 0 && poses[0].nose) {
    let distance = dist(noseXp, noseYp, targetX, targetY);
    let maxDistance = dist(0, 0, width/2, height/2);
    
    let distanceRatio = distance / maxDistance;
    
    let rate = map(distanceRatio, 0, 1, 1.0, MIN_RATE);
    let volume = map(distanceRatio, 0, 1, 1.0, MIN_VOLUME);
    
    rate = constrain(rate, MIN_RATE, 1.0);
    volume = constrain(volume, MIN_VOLUME, 1.0);
    
    if (soundzzs && soundzzs.isLoaded()) {
      soundzzs.rate(rate);
      soundzzs.setVolume(volume);
    }
  }
}

function displayAudioParams() {
  fill(255);
  noStroke();
  textSize(16);
  let rate = soundzzs.rate();
  let volume = soundzzs.getVolume();
  text(`Rate: ${rate.toFixed(2)}`, 10, 20);
  text(`Volume: ${volume.toFixed(2)}`, 10, 40);
  
  let distance = dist(noseXp, noseYp, targetX, targetY);
  if (distance < CATCH_DISTANCE) {
    fill(0, 255, 0);
    text("Great! You're on target!", 10, 60);
  }
}

function musicSignShow() {
  let sizeSign = 80;
  textSize(sizeSign);
  fill(255);
  text("🎵", noseXp - sizeSign/2, noseYp + sizeSign/4);
  

  let level=amplitude.getLevel();
  let tt=map(level,0,1,0,320);
  fill(tt,20,200)
  ellipse(200,200,tt,tt)
}

function musicSignPosition() {
  if (poses.length > 0) {
    let pose = poses[0];
    if (pose.nose) {
      fill(236, 1, 90);
      noStroke();
      noseXp = pose.nose.x;
      noseYp = pose.nose.y;
    }
  }
}

function windowResized() {
  resizeCanvas(windowWidth - 50, windowHeight - 50);
}

noseChaseMusicGame by Barry-zong -p5.js Web Editor