Weekly Assignments Final Project About me Agreement

14. Interface and Application Programming

Assignment

Group assignment

This week we made graphical application using different software libraries. We learned how to use them to make interfaces and more importantly, how to interact with embeded devices with these frameworks.

Link to this week's group assignment

Web browser and JavaScript

For interface I've done some practise of reading and visualizing the sensor data in web browser using WebGL and Web Serial api in week11, I just link to that first.

On week11, I made two web pages, both of them reads the input data from the serial port of the board and does some visualization to the data.

The two pages are linked down below:

On both pages, they have the Get Port button:

<button onclick="getPort()">Get Port</button>

When the button is clicked, it calls the getPort() function, which is shown below:

function getPort() {
  navigator.serial.requestPort().then(async port=> {
    await port.open({baudRate})
    const decoder = new TextDecoder()
    start_drawing()
    // https://developer.mozilla.org/en-US/docs/Web/API/SerialPort#reading_data_from_a_port
    while (port.readable) {
      const reader = port.readable.getReader()
      try {
        while (true) {
          const {value, done} = await reader.read()
          if (done) break
          if (window.on_serial_data) window.on_serial_data(decoder.decode(value))
        }
      }
      catch (err) { console.error(err) }
      finally { reader.releaseLock() }
    }
  })
}

It uses the Web Serial functionality provided by the web browser, the navigator.serial.requestPort() function can open a serial connection if there's a board connected to the USB port.

If the serial port is opened, it first calls the function start_drawing() and continously reads the data from the serial port, and if the function on_serial_data() is defined, it passes the read data to the function to further process the data.

The start_drawing() function and on_serial_data() are defined differently to make different visualizations of the data.

Capacitive Sensing Values With 2D Canvas

As it's shown, the first page shows the readings of the capacitive pads on the board.

const ctx = document.querySelector('canvas').getContext('2d')

The getContext('2d') method on the HTML canvas element gives a 2d rendering context, which allows drawing 2d graphics.

let pad_values = []
let max_pad_value = 0
let last_touched = -1
let processing_time = 0

let remainder = ''
function on_serial_data(data) {
  const lines = (remainder + data).split('\n')
  if (lines.length === 0) return
  remainder = lines.pop()
  for (const line of lines) {
    if (line === '') continue
    if (log) console.log(line)
    const fields = line.split(' ')
    if (fields.length != 12) {
      console.error('invalid line:', line)
      continue
    }
    processing_time = fields.pop()
    last_touched = fields.pop()
    output.textContent = `last touched: ${last_touched}, procesing time: ${processing_time}ms, ` + fields.join('\t')
    pad_values = fields
    max_pad_value = Math.max(max_pad_value, ...fields)
  }
}

The on_serial_data() function processes the serial data line by line, and collects the data values into the pad_values, max_pad_value, last_touched, processing_time variables, and later using them to draw the histogram in start_drawing() function.

function start_drawing() {
  requestAnimationFrame(function draw(time_stamp) {
    const hist_width = 50
    const hist_height = ctx.canvas.height - 30
    const upper_value = 5000
    const col_width = ctx.canvas.width / pad_count

    console.assert(pad_value && pad_values.length === pad_count,
      `pad value count (${pad_values.length}) doesn't match the pad count (${pad_count})`)

    ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height)
    ctx.font = '20px sans-serif'
    ctx.textAlign = 'center'
    ctx.textBaseline = 'bottom'
    const max_val = Math.max(upper_value, max_pad_value)
    for (let i = 0; i < pad_count; ++i) {
      const h = pad_values[i] / max_val * hist_height
      const x = i * col_width + col_width / 2
      ctx.fillStyle = 'black'
      ctx.fillText(pad_values[i], x, ctx.canvas.height)
      if (last_touched == i) ctx.fillStyle = 'red'
      ctx.fillRect(x - hist_width / 2, hist_height - h, hist_width, h)
    }
    ctx.fillStyle = 'black'
    ctx.textAlign = 'start'
    ctx.textBaseline = 'top'
    ctx.fillText('max value: ' +  max_val, 0, 0)

    requestAnimationFrame(draw)
  })
}

The start_drawing() function calls the requestAnimationFrame() function recursively to run a render loop continously. It draws the values as histograms using the ctx.fillRect() and ctx.fillText() methods, etc.

3D Rotation With WebGL

const gl = document.querySelector('canvas').getContext('webgl2')

The getContext('webgl2') method creates a webgl2 rendering context that uses WebGL, which allows rendering 3D graphics using hardware acceleration in the web browser.

const primitives = []
function gl_primitive(type, first_vertex_index, vertex_count) {
  return {type, first_vertex_index, vertex_count}
}

const program = init_gl()

function init_gl() {
  const program = create_shader_program_from_source(gl,
    `#version 300 es

//     uniform float u_scale;
//     uniform vec3 u_offset;
     uniform mat3 u_rotation_mat;
     uniform mat4 u_perspective_mat;

     in vec3 a_position;
     out float alpha;

     void main() {
       vec3 pos = u_rotation_mat * a_position;
       alpha = pos.z + 1.2;
//       pos = u_offset + u_scale * pos;
       pos.z -= 3.0;
       gl_PointSize = 2.0;
       gl_Position = u_perspective_mat * vec4(pos, 1);
     }
    `,
    `#version 300 es

     precision highp float;

     in float alpha;
     out vec4 color;

     void main() {
       color = vec4(0, 0, 0, alpha);
     }
    `
  )

  gl.enable(gl.DEPTH_TEST)
  gl.useProgram(program)

  // generate vertices for a unit sphere
  // https://en.wikipedia.org/wiki/Spherical_coordinate_system#Cartesian_coordinates
  const num_segments = 20
  const vertices = new Float32Array(3 * (num_segments) * (num_segments+1))
  let idx = 0
  for (let i = 0; i < num_segments; ++i) {
    const first = idx
    for (let j = 0; j < num_segments+1; ++j) {
      const theta = Math.PI * j / num_segments
      const phi = Math.PI * 2 * i / num_segments
      /* x */ vertices[idx++] = Math.sin(theta)*Math.cos(phi)
      /* y */ vertices[idx++] = Math.sin(theta)*Math.sin(phi)
      /* z */ vertices[idx++] = Math.cos(theta)
    }
    primitives.push(gl_primitive(gl.LINE_STRIP, first, idx-first))
  }
  console.assert(idx === vertices.length)
  console.assert(primitives.length === num_segments)

  const vertex_buffer = gl.createBuffer()
  gl.bindBuffer(gl.ARRAY_BUFFER, vertex_buffer)
  gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW)
  const pos_attrib_loc = gl.getAttribLocation(program, 'a_position')
  gl.enableVertexAttribArray(pos_attrib_loc)
  gl.vertexAttribPointer(pos_attrib_loc, 3, gl.FLOAT, false, 0, 0)

  return program
}

The init_gl() function creates a shader program, which is a program that runs on the GPU for drawing the 3d scene, and sets the data for all the vertices of a sphere that's going to be drawn.

let ypr = [0, 0, 0] // yaw, pitch, roll Eular angles

let remainder = ''
function on_serial_data(data) {
  const lines = (remainder + data).split('\n')
  if (lines.length == 0) return
  remainder = lines.pop()
  for (const line of lines) {
    if (line === '') continue
    const fields = line.split(' ')
    if (fields.length != 3) {
      console.error('invalid line:', line)
      continue
    }
    ypr = fields.map(degree => degree / 180 * Math.PI)
  }
}

The on_serial_data() function processes the serial data line by line and reads the yaw, pitch, roll Eular angles sent by the board and saves them in the ypr variable.

function start_drawing() {
  const u_perspective_mat_loc = gl.getUniformLocation(program, 'u_perspective_mat')
  const u_rotation_mat_loc = gl.getUniformLocation(program, 'u_rotation_mat')

  const perspective_matrix = perpective_matrix(Math.PI / 4, gl.canvas.width / gl.canvas.height, 0.1, 20)
  requestAnimationFrame(function draw(time_stamp) {
    //gl.clearColor(0, 0, 0, 1)
    //gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT)

    gl.uniformMatrix4fv(u_perspective_mat_loc, false, perspective_matrix.data)
    gl.uniformMatrix3fv(u_rotation_mat_loc, false, ypr_rotation_matrix(...ypr).data)
    for (const p of primitives) {
      gl.drawArrays(p.type, p.first_vertex_index, p.vertex_count)
    }
    requestAnimationFrame(draw)
  })
}

function Mat(row_count, col_count, data) {
  if (row_count * col_count !== data.length) throw new Error(`Data array length ${data.length} does not match matrix dimention of ${row_count}X${col_count}.`)
  return { row_count, col_count, data }
}

function perpective_matrix(fov, aspect, near, far) {
  const f = Math.tan(Math.PI * 0.5 - 0.5 * fov)
  const range_inv = 1 / (near - far)
  return Mat(4, 4, [
    f / aspect, 0, 0, 0,
    0, f, 0, 0,
    0, 0, (near + far) * range_inv, -1,
    0, 0, near * far * range_inv * 2, 0
  ])
}

function ypr_rotation_matrix(yaw, pitch, roll) {
  const sina = Math.sin(yaw)
  const cosa = Math.cos(yaw)
  const sinb = Math.sin(pitch)
  const cosb = Math.cos(pitch)
  const sinc = Math.sin(roll)
  const cosc = Math.cos(roll)
  return Mat(3, 3, [
    // col1
    cosa * cosb,
    sina * cosb,
    -sinb,
    // col2
    cosa * sinb * sinc - sina * cosc,
    sina * sinb * sinc + cosa * cosc,
    cosb * sinc,
    // col3
    cosa * sinb * cosc + sina * sinc,
    sina * sinb * cosc - cosa * sinc,
    cosb * cosc,
  ])
}

And the start_drawing() function runs the rendering loop that sets the matrix for doing the 3D coordinate transformation using the ypr data that is read, showing the sphere rotating as we rotates the accelerometer.

Source files