2. Sound & Image


Content

1. Manipulating images
2. Using P5's sound library
3. Playing back mp3
4. Synthesize sounds
5. Analyzing mic-input: Amplitude
6. FFT: Analyzing more sound features
7. Using sound features for game design

1. Manipulating images

Reading pixel colors

Link: https://editor.p5js.org/ear___sand/sketches/jV2IKjMcm

Using pixel colors for simple collision detection in an extended version of the previous snake game.

Basic FX programming: manipulating images

var grimesPic;
var step = 15;

function preload() {
    grimesPic = loadImage('img/grimes2.jpg');
  
}

function setup() {
  createCanvas(690, 400);
  //image(grimesPic, 0,0, width, height);
  grimesPic.resize(width,0);
  grimesPic.loadPixels();
  strokeWeight(4);
  noStroke();
  fill(0,60);
  for (let y = 0; y < height; y += 4) {
   for (let x = 0; x < width; x += 12){
     var c = grimesPic.get(x,y);
     fill(red(c),green(c), blue(c), 40);
     stroke(red(c),green(c), blue(c), 40);
     ellipse(x,y, 5,5);
     line(x,y, x+random(-50,150), y+random(-50,150));
   }
  }
}

function draw() {
  
  
}

  

Using the Webcam

var capture;
      var captureOn = false;
      
      function setup() {
        createCanvas(390, 240);
      }
      
      function draw() {
        background(0);
        if(captureOn == true){
          image(capture, 0, 0, 320, 240);
        }
        //filter('INVERT');
      }
      
      function keyPressed(){
        capture = createCapture(VIDEO);
        capture.size(320, 240);
        capture.hide();
        captureOn = !captureOn;
      }

  

As we previously did with a static image, we can now use the information of the pixels to come up with own effects:
https://editor.p5js.org/ear___sand/sketches/Okrv5wMlI

Exercise
Write 2 visual effects for the webcam image.



Using pixel color for collision detection in the snake game



Loading images as artwork for the breakout game

https://editor.p5js.org/ear___sand/sketches/YmEQexaFe
gfx1: https://my.pcloud.com/publink/show?code=XZK9TmkZMOrcNkWewYkzCooh7slUEJT3IykV
gfx2: https://my.pcloud.com/publink/show?code=XZO9TmkZvhBRjJrrHNLylBmAc4CAEzuLPCR7



2. Using the Sound library

First we have to load the sound library in HTML:

< script src="P5/addons/p5.sound.min.js">< /script>
   


3. Playing back mp3

Load and play mp3 - files:

var song;

function setup() {
  song = loadSound('yourFile.mp3');
  song.loop();
  createCanvas(640, 100);
  background(200);
}

function draw(){
  speed = map(mouseY, 0.1, height, 0, 2);
  speed = constrain(speed, 0.01, 4);
  song.rate(speed);
}

function mousePressed() {
  if (song.isPlaying()) {
    song.stop();
  } else {
    song.play();
  }
}
  

Interact with speed and volume:

var song;
var speed;
var amp;

function preload(){
  song = loadSound('material/lutos.mp3');
}

function setup() {
  song.loop();
  createCanvas(640, 200);
  background(100);
  stroke(150);
}

function draw(){
  background(100);
  speed = map(mouseY, 0.1, height, 0, 2);
  speed = constrain(speed, 0.01, 4);
  song.rate(speed);
  amp = map(mouseX, 0.1, width, 0, 1);
  amp = constrain(amp, 0.01, 1);
  song.setVolume(amp);
  line(mouseX, 0, mouseX, height);
  line(0, mouseY, width, mouseY);
}

function mousePressed() {
  if (song.isPlaying()) {
    song.stop();
  } else {
    song.play();
  }
}
  

Exampple: Electroplankton, (2005)


A characteristic feature of Electroplankton is the combination of auditory and visual elements with player interaction to 10 generative music games. Each of the 10 music games has its own rules, own virtual actors and interaction possibilities.


Exercise
Now use sounds in the arcade game. Make it sound, when the ball hits the wall.



4. Synthesize sounds

Noise

function setup() {
  createCanvas(640, 200);
  background(100);
  stroke(150);
  fft = new p5.FFT();
  ns = new p5.Noise()
  ns.amp(0.5);
  ns.start();
}

function draw(){
  background(100);
  var spectrum = fft.analyze();
  noStroke();
  fill(200);

  for (var i = 0; i< spectrum.length; i++){
    var x = map(i, 0, spectrum.length, 0, width*3);
    var h = -height + map(spectrum[i], 0, 255, height, 0);
    rect(x, height, width / spectrum.length, h )
  }
}


Oscillators



Filters



Envelopes



5. Analyzing mic-input: Amplitude

https://editor.p5js.org/ear___sand/sketches/g7HUrzLlH

var mic;
var isOn = false;

function setup() {
  createCanvas(690, 400);
  mic = new p5.AudioIn();
}


function draw() {
  noStroke();
  fill(235, 30);
  rect(0, 0, width, height);
  fill(0);
  text("Press mouse to start tracking mic.", 10, 10)
  var vol = mic.getLevel();
  fill(127);
  stroke(0);

  // Draw an ellipse with height based on volume
  //if(vol > 0.02){
  var r = map(vol, 0, 1, 0, width * 30);
  ellipse(width / 2, height / 2, r + 40, r + 40);
  //}
}

function mousePressed() {

  if (isOn) {
    mic.stop();
    isOn = false;
  } else {
    mic.start();
    isOn = true;
  }
}


6. FFT: Analyzing more sound features

var song;

function preload(){
  song = loadSound('oreo.mp3');
  fft = new p5.FFT();
}

function setup() {
  
  song.rate(1);
  createCanvas(690, 400);
  background(100);
  stroke(150);
}

function draw(){
  background(230);
  fill(0);
  text("Press mouse to start/stop track.",10,10);
  var spectrum = fft.analyze();
  noStroke();
  fill(200,0,0,100);

  for (var i = 0; i< spectrum.length; i++){
    var aktuellerWert = spectrum[i];
    //var x = map(i, 0, spectrum.length, 0, width*3);
    //var h = -height + map(aktuellerWert, 0, 255, height, 0);
    var x = i*4;
    var h = aktuellerWert;
    rect(x, height, 1, -h );
  }
}

function mousePressed() {
  if (song.isPlaying()) {
    song.stop();
  } else {
    song.play();
  }
}


https://editor.p5js.org/ear___sand/sketches/vuHSW0gUC
var song;
    var fft;
    
    function preload() {
      song = loadSound('aHigherOrder-1-21.mp3');
    }
    
    function setup() {
      //createCanvas(windowWidth, windowHeight, WEBGL);
      createCanvas(690, 690);
      fft = new p5.FFT();
      song.rate(1);
      stroke(255);
      strokeWeight(1);
      fill(255,150);
      rectMode(CENTER);
    }
    
    function draw() {
      background(30, 40, 60);
      text("Press any key to start"+Math.round(frameRate()),10,10);
      
      var spectrum = fft.analyze(256);
    
      for (var y = 0; y < 16; y += 1) {
        for (var x = 0; x < 16; x += 1) {
          currentHeight = spectrum[(y * 15) + x];
          var scal = Math.pow(currentHeight,2) / 200 +10;
          var bx = x * 35 + 85;
          var by = y * 35 + 85;
          fill(0,scal/0.5,0,150);
          rect(bx, by, scal, scal);
        }
      }
    
    }
    
    
    function keyPressed() {
      if (song.isPlaying()) {
        song.stop();
      } else {
        song.play();
      }
    }

  


Exercise
Use FFT values with an own track for own visual parameters



7. Using sound features for game design

Exampple: Vib Ribbon (Playstation, 1999). Gameplay oriented on sonic parameters of Music.



Exampple: 140 (2013)