1/*
2audiograph.mjs - show a svg view of the web audio API graph built during a playback
3Copyright (C) 2025 Strudel contributors - see <https://codeberg.org/uzu/strudel/src/branch/main/website/src/repl/audiograph.mjs>
4This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program.  If not, see <https://www.gnu.org/licenses/>.
5*/
6
7// main entry point is `debugAudiograph`
8import { logger } from '@strudel/core';
9import { getAudioContext, getSuperdoughAudioController, webaudioOutput } from '@strudel/webaudio';
10
11let mermaid = null;
12let svgPanZoom = null;
13
14let running = false;
15let hap_count = 0;
16
17let cache = new Map();
18const initCache = JSON.stringify({
19  connect: [],
20  where: [],
21  disconnectAll: 0,
22  disconnectOne: 0,
23  hasStop: false,
24  stopCount: 0,
25  ac: null,
26  creation: null,
27});
28
29let toggleOrig;
30
31function stackTrace() {
32  var err = new Error();
33  const stacktrace = err.stack;
34  const lines = stacktrace.split('\n');
35  let lineIndex = lines.findIndex((line) => line !== 'Error' && !line.includes('audiograph.mjs'));
36  if (lines[lineIndex].includes('gainNode')) lineIndex++;
37  if (lines[lineIndex].includes('getWorklet')) lineIndex++;
38  const line = lines[lineIndex].replace(/\s*at\s/, '').replace('http', '@');
39  let match;
40  match = line.match(/([^@]*)@.*packages(\/[^:]+:\d+:\d+)/);
41  if (match) {
42    return match[1].replace(/[^.:/a-zA-Z0-9]/g, '') + '@' + match[2].replace(/[^.:/a-zA-Z0-9]/g, '');
43  }
44  return '@';
45}
46
47// This captures all AudioNodes lazily
48// when an `.audioid` property is called
49// no solution was found to hook
50// AudioNode's constructor directly
51let audioid = 0;
52const lazyRegister = (o) => {
53  Object.defineProperty(o.prototype, 'audioid', {
54    get: function () {
55      if (!this._audioid) {
56        this._audioid = ++audioid;
57        const s = JSON.parse(initCache);
58        s.type = this.constructor.name === 'AudiographNode' ? this.constructor._parentClassName : this.constructor.name;
59        // special case for subclassed AudioNodes
60        // they are implemented in superdough but hard to get a reference on here
61        // they are not AudioScheduledSourceNodes anyway
62        if (['FeedbackDelayNode', 'VowelNode'].indexOf(s.type) === -1) {
63          s.hasStop = window[s.type].prototype instanceof AudioScheduledSourceNode;
64        }
65        s.ac = this.context?.constructor.name || 'AudioParam';
66        s.creation = s.creation || stackTrace();
67        cache.set(this._audioid, s);
68      }
69      return this._audioid;
70    },
71    enumerable: false,
72    configurable: true,
73  });
74};
75
76// extend a specific AudioNode's constructor
77// necessary when creation is done direclty by
78// calling the constructor
79// eg: new GainNode(...)
80const audioNodeHook = (node) => {
81  const name = node.prototype.constructor.name;
82  const PatchedNode = class AudiographNode extends node {
83    constructor(...args) {
84      super(...args);
85      // trigger the lazy register
86      this._audioid = this.audioid;
87    }
88  };
89  PatchedNode._parentClassName = name;
90  window[name] = PatchedNode;
91};
92
93const drawMessage = async function (message) {
94  const element = document.querySelector('.strudel-mermaid');
95  let gd = '';
96  gd += '---\n';
97  gd += 'config:\n';
98  gd += '  flowchart:\n';
99  gd += '    wrappingWidth: 600\n';
100  gd += '---\n';
101  gd += 'flowchart LR\n';
102  gd += 'id[' + message.replaceAll(' ', '&nbsp;') + ']\n';
103
104  let { svg } = await mermaid.render('strudelSvgId', gd);
105  svg = svg.replace(/max-width:\s[0-9.]*px;/i, 'height: 100%');
106  svg = svg.replaceAll('&amp;nbsp;', ' ');
107  element.innerHTML = svg;
108};
109
110const drawDiagram = async function () {
111  const element = document.querySelector('.strudel-mermaid');
112  let code = window.strudelMirror.code;
113  code = code.replace(/^await debugAudiograph.*\n?/gm, '');
114  code = '// date: ' + new Date().toISOString() + '\n\n' + code;
115  code = '// host: ' + document.location.hostname + '\n' + code;
116  const codeLines = code.split(/(?:\n|\r\n?)/);
117  const maxLineLength = codeLines.reduce((memo, line) => Math.max(memo, line.length), 0);
118
119  // https://mermaid.js.org/syntax/flowchart.html
120  let gd = '';
121  gd += '---\n';
122  gd += 'config:\n';
123  gd += '  flowchart:\n';
124  gd += '    wrappingWidth: ' + 14 * maxLineLength + '\n';
125  gd += '---\n';
126  gd += 'flowchart TB\n';
127  gd += '\tsubgraph AG[STRUDEL AUDIOGRAPH]\n';
128
129  // seed graph builder with all
130  // unconnected nodes
131  let lookup = [];
132  cache.forEach((v, k) => {
133    if (v.connect.length === 0) lookup.push(k);
134  });
135  const relations = [];
136  let curRelations;
137  const zombieCount = 0;
138  const sourceLoc = (stack) => {
139    if (stack === '@') return stack;
140    return stack.replace('@', '\n').replace('/superdough/', '/');
141  };
142  const label = (s) => {
143    const source = s.creation ? '\n' + sourceLoc(s.creation) : '';
144    let lb = '[' + '**' + s.type + '**' + source + ']';
145    if (s.ac === 'OfflineAudioContext') lb = '[' + lb + ']';
146    return lb;
147  };
148  const isConnectLeak = (s) => {
149    return (
150      ['AudioDestinationNode', 'AudioParam'].indexOf(s.type) === -1 &&
151      s.disconnectAll === 0 &&
152      s.connect.length > s.disconnectOne
153    );
154  };
155  const isStopLeak = (s) => {
156    return s.hasStop && s.stopCount === 0;
157  };
158  do {
159    curRelations = relations.length;
160    lookup.slice().forEach((n) => {
161      cache.forEach((v, k) => {
162        if (v.connect.indexOf(n) !== -1) {
163          if (lookup.indexOf(k) === -1) lookup.push(k);
164          gd += v.connect
165            .map((i) => {
166              if (lookup.indexOf(i) === -1) lookup.push(i);
167              if (relations.indexOf(k + '-' + i) === -1) {
168                relations.push(k + '-' + i);
169                return (
170                  '\t\tnode' +
171                  k +
172                  label(v) +
173                  ' -- ' +
174                  sourceLoc(v.where[0]) +
175                  ' --> node' +
176                  i +
177                  label(cache.get(i)) +
178                  '\n'
179                );
180              }
181            })
182            .join('');
183        }
184        if (k === n) {
185          gd += v.connect
186            .map((i) => {
187              if (lookup.indexOf(i) === -1) lookup.push(i);
188              if (relations.indexOf(k + '-' + i) === -1) {
189                relations.push(k + '-' + i);
190                return (
191                  '\t\tnode' +
192                  k +
193                  label(v) +
194                  ' -- ' +
195                  sourceLoc(v.where[0]) +
196                  ' --> node' +
197                  i +
198                  label(cache.get(i)) +
199                  '\n'
200                );
201              }
202            })
203            .join('');
204        }
205      });
206    });
207  } while (relations.length > curRelations /*&& lookup.length < 100*/);
208  // add orphan nodes
209  const inRelation = '-' + relations.join('-') + '-';
210  cache.forEach((v, k) => {
211    if (!inRelation.includes('-' + k + '-')) {
212      gd += '\t\tnode' + k + label(v) + '\n';
213    }
214  });
215
216  const codePlaceholder = 'm'.repeat(maxLineLength);
217  gd += '\tsubgraph LEGEND\n';
218  gd += '\t\tlegend1[in AudioContext]\n';
219  gd += '\t\tlegend2[[in OfflineAudioContext]]\n';
220  gd += '\t\tlegend3[not disconnected]\n';
221  gd += '\t\tlegend4[AudioParam]\n';
222  gd += '\t\tlegend5[AudioDestinationNode]\n';
223  gd += '\t\tlegend6[not stopped]\n';
224  gd += '\tend\n';
225  gd += '\tsubgraph CODE[Strudel Code]\n';
226  // we use a codePlaceholder to
227  // - avoid problems with special chars
228  // - stop mermaid to split lines on space with multiple tspans
229  // - force mermaid to prepare a sufficiently sized zone
230  gd += '\ncode[' + (codePlaceholder + '<br>').repeat(codeLines.length) + ']\n';
231  gd += '\tend\n';
232  gd += '\tend\n';
233  gd += '\tclassDef audioparam fill:#6f6;\n';
234  gd += '\tclassDef destination fill:#99f;\n';
235  gd += '\tclassDef connectleak fill:#f96,stroke:#f00,stroke-width:2px;\n';
236  gd += '\tclassDef stopleak fill:#f55,stroke:#f00,stroke-width:2px;\n';
237  gd += '\tclass legend3 connectleak;\n';
238  gd += '\tclass legend4 audioparam;\n';
239  gd += '\tclass legend5 destination;\n';
240  gd += '\tclass legend6 stopleak;\n';
241  cache.forEach((v, k) => {
242    if (isConnectLeak(v)) {
243      gd += '\tclass node' + k + ' connectleak;\n';
244    } else if (isStopLeak(v)) {
245      gd += '\tclass node' + k + ' stopleak;\n';
246    }
247    if (v.type === 'AudioParam') {
248      gd += '\tclass node' + k + ' audioparam;\n';
249    }
250    if (v.type === 'AudioDestinationNode') {
251      gd += '\tclass node' + k + ' destination;\n';
252    }
253  });
254  let { svg } = await mermaid.render('strudelSvgId', gd);
255
256  // put real code in code zone
257  let idx = 0;
258  const escapeHtml = (unsafe) => {
259    return unsafe
260      .replace(/&/g, '&amp;')
261      .replace(/</g, '&lt;')
262      .replace(/>/g, '&gt;')
263      .replace(/"/g, '&quot;')
264      .replace(/'/g, '&#039;');
265  };
266  svg = svg.replaceAll(codePlaceholder, () => escapeHtml(codeLines[idx++]));
267
268  // improve sizing on web page
269  svg = svg.replace(/max-width:\s[0-9.]*px;/i, 'height: 100%');
270  element.innerHTML = svg;
271
272  // align the code lines
273  let svgText = document.querySelector('[id^=flowchart-code] text');
274  svgText.setAttributeNS(null, 'style', 'text-anchor: start;');
275  const svgElement = document.querySelector('svg');
276  const svgLabel = document.querySelector('svg [id^=flowchart-code] .label');
277  const transformList = svgLabel.transform.baseVal;
278  const svgTransform = svgElement.createSVGTransform();
279  const tspans = Array.from(document.querySelectorAll('[id^=flowchart-code] tspan.text-inner-tspan'));
280  let tspansMaxLength = tspans.reduce((memo, tspan) => Math.max(memo, tspan.getComputedTextLength()), 0);
281  svgTransform.setTranslate(-tspansMaxLength / 2, 0);
282  transformList.appendItem(svgTransform);
283
284  let doPan = false;
285  let eventsHandler;
286  let panZoom;
287  let mousepos;
288
289  eventsHandler = {
290    haltEventListeners: ['mousedown', 'mousemove', 'mouseup'],
291    mouseDownHandler: function (ev) {
292      if (event.target.className == '[object SVGAnimatedString]') {
293        doPan = true;
294        mousepos = {
295          x: ev.clientX,
296          y: ev.clientY,
297        };
298      }
299    },
300    mouseMoveHandler: function (ev) {
301      if (doPan) {
302        panZoom.panBy({
303          x: ev.clientX - mousepos.x,
304          y: ev.clientY - mousepos.y,
305        });
306        mousepos = {
307          x: ev.clientX,
308          y: ev.clientY,
309        };
310        window.getSelection().removeAllRanges();
311      }
312    },
313    mouseUpHandler: function (ev) {
314      doPan = false;
315    },
316    init: function (options) {
317      options.svgElement.addEventListener('mousedown', this.mouseDownHandler, false);
318      options.svgElement.addEventListener('mousemove', this.mouseMoveHandler, false);
319      options.svgElement.addEventListener('mouseup', this.mouseUpHandler, false);
320    },
321    destroy: function (options) {
322      options.svgElement.removeEventListener('mousedown', this.mouseDownHandler, false);
323      options.svgElement.removeEventListener('mousemove', this.mouseMoveHandler, false);
324      options.svgElement.removeEventListener('mouseup', this.mouseUpHandler, false);
325    },
326  };
327  panZoom = svgPanZoom('#strudelSvgId', {
328    zoomEnabled: true,
329    controlIconsEnabled: true,
330    fit: 1,
331    center: 1,
332    zoomScaleSensitivity: 0.4,
333    customEventsHandler: eventsHandler,
334  });
335};
336
337const svgExport = async () => {
338  const a = document.createElement('a');
339  document.body.appendChild(a);
340  a.style = 'display: none';
341
342  const selector = '.strudel-mermaid';
343  const bbox = document.querySelector('svg g').getBBox();
344  let transform, style;
345  // clean pan-zoom viewport
346  const pzViewport = document.querySelector('.svg-pan-zoom_viewport');
347  if (pzViewport) {
348    transform = pzViewport.transform;
349    style = pzViewport.style;
350    pzViewport.setAttribute('transform', '');
351    pzViewport.style = '';
352  }
353
354  const spzMin = await fetch('https://cdn.jsdelivr.net/npm/svg-pan-zoom@3.6.2/dist/svg-pan-zoom.min.js').then((res) =>
355    res.text(),
356  );
357
358  const scriptContent = '<![CDATA[' + spzMin + ';svgPanZoom("svg");]]>';
359  // prepare svg
360  const content = document
361    .querySelector(selector)
362    .innerHTML.replaceAll('<br>', '<br/>')
363    // remove useless tags
364    .replace(/<g id="svg-pan-zoom.*<\/g>/, '<script>' + scriptContent + '</script>')
365    .replace(/<defs>.*<\/defs>/, '')
366    // give inkscape true sizes
367    .replace('width="100%"', 'width="' + bbox.width + '" height="' + bbox.height + '"');
368
369  // restore pan-zoom viewport
370  if (pzViewport) {
371    pzViewport.setAttribute('transform', transform);
372    pzViewport.style = style;
373  }
374
375  // trigger download
376  var blob = new Blob([content], { type: 'image/svg+xml' }),
377    url = window.URL.createObjectURL(blob);
378  a.href = url;
379  a.download = 'audiograph.svg';
380  a.click();
381  window.URL.revokeObjectURL(url);
382};
383
384const resetAudioOutput = function (audioid) {
385  // calling reset on SuperdoughAudioController
386  // will discard output nodes AND recreate them
387  // so we keep the same `cache` to handle the
388  // `disconnects` knowing that new nodes will be
389  // stricly after the current audioid.
390  // then we purge the old nodes from the `cache`
391  // to have a clean state
392
393  // make sure destination will be recreated in the
394  // cache
395  const destination = getAudioContext().destination;
396  if (destination._audioid) delete destination._audioid;
397
398  const sac = getSuperdoughAudioController();
399  sac.reset();
400  Array.from(cache.keys()).map((k) => {
401    if (k <= audioid) cache.delete(k);
402  });
403};
404
405const postProcessing = async function () {
406  hap_count = 0;
407  await drawDiagram();
408
409  resetAudioOutput(audioid);
410};
411
412const defaultOptions = {
413  StopAfterHapCount: 10,
414  hapsBatch: 0,
415  maxEdges: 10000,
416  maxTextSize: 200000,
417  audioAPIBreathingRoomSec: 5,
418};
419
420// `StopAfterHapCount` :
421//         The player will auto-stop after hap count have
422//         been played. when StopAfterHapCount = 0, it will
423//         continue playing until 'stop' is clicked
424// `audioAPIBreathingRoomSec` :
425//         how much time should we wait after 'stop' to let
426//         the audioAPI finish its tail of ondended calls
427// `hapsBatch` :
428//         the AudioGraph will be displayed every hapsBatch haps
429//         when hapsBatch = 0, AudioGraph will only be displayed
430//         after and auto-stop or after 'stop' is clicked
431//         In hapsBatch mode you will probably see a trailing of
432//         non disconnected notes on the graph because the audio
433//         API may have some lag disconnecting them
434//         cf also audioAPIBreathingRoomSec
435// `maxEdges`
436//         This is a mermaid.js config that forces a hard limit
437//         on the maximum number of Edges of a graph
438//         needs a reload to be taken into account
439// `maxTextSize`
440//         This is a mermaid.js config that forces a hard limit
441//         on the maximum text size of a graph definition
442//         needs a reload to be taken into account
443
444export const debugAudiograph = async (argOptions = {}) => {
445  const options = Object.assign({}, defaultOptions, argOptions);
446  const { StopAfterHapCount, hapsBatch, maxEdges, maxTextSize, audioAPIBreathingRoomSec } = options;
447  const sm = window.strudelMirror;
448  const code = sm.code;
449  if (!code.match(/await\s+debugAudiograph/)) {
450    throw new Error('you need to call `await debugAudiograph()` for audiograph to work');
451  }
452  const emptyOptions = /await\s+debugAudiograph\(\)/.exec(code);
453  if (emptyOptions) {
454    const cutCode = emptyOptions.index + emptyOptions[0].length - 1;
455    const codeOptions = JSON.stringify({ StopAfterHapCount: StopAfterHapCount }).replaceAll('"', '');
456    sm.setCode(code.slice(0, cutCode) + codeOptions + code.slice(cutCode));
457  }
458
459  if (window.audiograph === undefined) {
460    const ag = (window.audiograph = {});
461
462    toggleOrig = sm.toggle;
463
464    ////////////////////////////////////////
465    // step 1: web audio api instrumentation
466    ////////////////////////////////////////
467
468    // path AudioNode & AudioParam
469    // to give them lazy ids
470    // this captures both `ac.createGain`
471    // and `new GainNode(..)` patterns
472    lazyRegister(AudioNode);
473    lazyRegister(AudioParam);
474    lazyRegister(PeriodicWave);
475
476    const audioNodes = [
477      AudioBufferSourceNode,
478      AudioWorkletNode,
479      AnalyserNode,
480      BiquadFilterNode,
481      ChannelMergerNode,
482      ChannelSplitterNode,
483      ConstantSourceNode,
484      ConvolverNode,
485      DelayNode,
486      DynamicsCompressorNode,
487      GainNode,
488      IIRFilterNode,
489      OscillatorNode,
490      PannerNode,
491      StereoPannerNode,
492      WaveShaperNode,
493    ];
494    audioNodes.map((n) => {
495      if (n.prototype instanceof AudioScheduledSourceNode) {
496        const stopOrig = n.prototype.stop;
497        n.prototype.stop = function (...args) {
498          // stop called
499          const result = stopOrig.call(this, ...args);
500          const s = cache.get(this.audioid);
501          s.stopCount++;
502          return result;
503        };
504      }
505      audioNodeHook(n);
506    });
507
508    // patch BaseAudioContext factory methods
509    // to capture the source reference
510    Object.getOwnPropertyNames(BaseAudioContext.prototype)
511      .filter((n) => n.startsWith('create') && ['createBuffer'].indexOf(n) === -1)
512      .map((name) => {
513        const orig = BaseAudioContext.prototype[name];
514        BaseAudioContext.prototype[name] = function (...args) {
515          const result = orig.call(this, ...args);
516          const s = cache.get(result.audioid);
517          s.creation = stackTrace();
518          return result;
519        };
520      });
521
522    const connectOrig = AudioNode.prototype.connect;
523    AudioNode.prototype.connect = function (destination, ...args) {
524      const result = connectOrig.call(this, destination, ...args);
525      const s = cache.get(this.audioid);
526      s.connect.push(destination.audioid);
527      s.where.push(stackTrace());
528      return result;
529    };
530
531    const disconnectOrig = AudioNode.prototype.disconnect;
532    AudioNode.prototype.disconnect = function (destination, ...args) {
533      const result = disconnectOrig.call(this, destination, ...args);
534      const s = cache.get(this.audioid);
535      if (s.connect.length) {
536        if (destination) {
537          s.disconnectOne++;
538        } else {
539          s.disconnectAll++;
540        }
541      } else {
542        logger('WEIRD: node ' + this.audioid + 'called disconnect before any call to connect !');
543        //logger(new Error().stack);
544        console.log(cache);
545      }
546      return result;
547    };
548
549    // call reset 2 times to handle reload + 'play'
550    // the first reset's disconnect adds audioid tags on previous outputs
551    // that were not tagged (wrong cutoff)
552    resetAudioOutput(audioid);
553    // the second reset has the correct audioid cutoff
554    resetAudioOutput(audioid);
555
556    ////////////////////////////////////////
557    // step 2: Load external modules
558    ////////////////////////////////////////
559
560    const { default: mermaidModule } = await import(
561      'https://cdn.jsdelivr.net/npm/mermaid@11.12.1/dist/mermaid.esm.mjs'
562    );
563    mermaid = mermaidModule;
564
565    mermaid.initialize({
566      startOnLoad: false,
567      themeCSS: '.flowchart { height: 100%; }',
568      maxEdges: maxEdges,
569      maxTextSize: maxTextSize,
570      htmlLabels: false,
571      flowchart: {
572        htmlLabels: false,
573      },
574    });
575
576    const { default: svgPanZoomModule } = await import('https://esm.sh/svg-pan-zoom');
577    svgPanZoom = svgPanZoomModule;
578
579    //////////////////////////////////////////
580    // step 3: UI modifications
581    //////////////////////////////////////////
582
583    // add audiograph panel
584    if (!document.querySelector('.strudel-mermaid')) {
585      const mermaidDiv = document.createElement('div');
586      mermaidDiv.className = 'strudel-mermaid';
587      mermaidDiv.style = 'min-height: 600px; width: 60%';
588      const referenceNode = document.querySelector('#code');
589      referenceNode.parentNode.insertBefore(mermaidDiv, referenceNode.nextSibling);
590    }
591
592    // add svg export button
593    if (!document.querySelector('button[title=svg]')) {
594      const exportButton = document.createElement('button');
595      exportButton.innerHTML = '<span>ExportDiagram</span>';
596      exportButton.title = 'svg';
597      exportButton.onclick = svgExport;
598      const updateButton = document.querySelector('button[title=update]');
599      updateButton.parentNode.insertBefore(exportButton, updateButton);
600    }
601  }
602
603  if (!running) {
604    running = true;
605  }
606
607  if (hapsBatch === 0 || hap_count < hapsBatch) {
608    let msg = '';
609    msg += 'Recording activity...';
610    msg += '\npress stop to build diagram';
611    if (StopAfterHapCount) {
612      msg += '\nwill stop automatically in ' + Math.max(StopAfterHapCount - hap_count, 0) + ' haps';
613    }
614    await drawMessage(msg);
615  }
616
617  sm.toggle = async () => {
618    running = false;
619    sm.toggle = toggleOrig;
620    // schedule `toggle` on the js main loop
621    // to avoid interfering with any on-flight onTick
622    // not doing this can lead to a phase > 0 which will
623    // break the next start
624    setTimeout(sm.toggle.bind(sm), 0);
625    await drawMessage('please wait ' + audioAPIBreathingRoomSec + ' seconds\n' + 'the audio API is finishing its work');
626    setTimeout(postProcessing, audioAPIBreathingRoomSec * 1000);
627  };
628
629  /*global all*/
630  all((pat) =>
631    pat.onTrigger(async (hap, duration, cps, t) => {
632      hap_count++;
633      const key = Object.entries(hap.value)
634        .map((param) => param.join('/'))
635        .join('/');
636
637      // if we reached StopAfterHapCount, click 'stop'
638      if (StopAfterHapCount && hap_count > StopAfterHapCount) {
639        if (running) {
640          await sm.toggle();
641        }
642        // stop sending haps to superdough(...)
643        return;
644      }
645
646      await webaudioOutput(hap, t, hap.duration / cps, cps, t);
647
648      if (hapsBatch && hap_count % hapsBatch === 0) drawDiagram();
649    }),
650  );
651};