scoreSink.ts

examples/film/scoreSink.ts · raw · all examples · site home

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
// The AudioWorklet that plays the GPU-rendered score (audioStream.ts):
// runs of chunks of interleaved stereo frames arrive by postMessage (their
// buffers transferred), from a first chunk index on; "start" maps the context's frames to
// the film's. Each output frame plays the film frame it maps to, so the
// sound stays on the film's clock: a chunk that has not arrived plays as
// silence rather than shifting what follows.
export {};

declare const currentFrame: number;
declare function registerProcessor(name: string, processor: unknown): void;
declare class AudioWorkletProcessor {
  readonly port: MessagePort;
}

type Message =
  | { type: "chunks"; index: number; samples: Float32Array }
  | { type: "start"; offset: number }
  | { type: "stop" };

class ScoreSink extends AudioWorkletProcessor {
  private chunks = new Map<number, Float32Array>();
  // Context frame minus film frame; NaN until started
  private offset = NaN;
  // Frames per chunk, and in the film
  private size: number;
  private frames: number;
  // Film frames played as silence because their chunk had not arrived
  private missing = 0;
  private reported = 0;

  constructor(options: { processorOptions: { chunk: number; frames: number } }) {
    super();
    this.size = options.processorOptions.chunk;
    this.frames = options.processorOptions.frames;
    this.port.onmessage = ({ data }: MessageEvent<Message>) => {
      if (data.type === "chunks") {
        const frames = this.size * 2;
        for (let k = 0; k * frames < data.samples.length; k += 1) {
          this.chunks.set(data.index + k, data.samples.subarray(k * frames, (k + 1) * frames));
        }
      } else if (data.type === "start") {
        this.offset = data.offset;
      } else {
        this.offset = NaN;
        this.chunks.clear();
      }
    };
  }

  process(_inputs: Float32Array[][], outputs: Float32Array[][]) {
    const [left, right] = outputs[0];
    let playing = -1;
    for (let i = 0; i < left.length; i += 1) {
      const frame = currentFrame + i - this.offset;
      const index = Math.floor(frame / this.size);
      const samples = frame >= 0 ? this.chunks.get(index) : undefined;
      if (samples) {
        const at = (frame - index * this.size) * 2;
        left[i] = samples[at];
        right[i] = samples[at + 1];
        playing = index;
      } else {
        left[i] = 0;
        right[i] = 0;
        this.missing += frame >= 0 && frame < this.frames ? 1 : 0;
      }
    }
    // Chunks behind the playhead are done
    for (const index of this.chunks.keys()) {
      if (playing >= 0 && index < playing) {
        this.chunks.delete(index);
      }
    }
    if (this.missing !== this.reported) {
      this.reported = this.missing;
      this.port.postMessage({ type: "missing", frames: this.missing });
    }
    return true;
  }
}

registerProcessor("film-score", ScoreSink);