how to use Audio API in MV plugin?

Yitzi Litt · 2018-11-05T05:21:02+0900

How would I access the Web Audio API in an RPGMMV plugin? Specifically, I want to have a given Boolean variable turn on whenever the audio level in a BGM track reaches a given point, allowing me to then have onscreen actions "react" to the music. This should be possible, but I haven't seen anyone call the Web Audio API in a plugin before. As I'm not all that great at coding, I could use a pre-made template/example to work from. Thanks!

You can Actually Do this , infact RPGmaker MV does this to a certain extent,

Basically in most of RPGmakers standard asset, inside the .ogg Metafiles there two parameters called loop length and loop start. these parameters are used to try and make the song loop from a given point.

Unfortunately getting this information is quite cryptic!!!
it does it differently for .ogg and the crappy .m4a formats to make it worse. heres the source code for MV does this

Source code

Code:

WebAudio.prototype._readLoopComments = function(array) {
    this._readOgg(array);
    this._readMp4(array);
};

/**
 * @method _readOgg
 * @param {Uint8Array} array
 * @private
 */
WebAudio.prototype._readOgg = function(array) {
    var index = 0;
    while (index < array.length) {
        if (this._readFourCharacters(array, index) === 'OggS') {
            index += 26;
            var vorbisHeaderFound = false;
            var numSegments = array[index++];
            var segments = [];
            for (var i = 0; i < numSegments; i++) {
                segments.push(array[index++]);
            }
            for (i = 0; i < numSegments; i++) {
                if (this._readFourCharacters(array, index + 1) === 'vorb') {
                    var headerType = array[index];
                    if (headerType === 1) {
                        this._sampleRate = this._readLittleEndian(array, index + 12);
                    } else if (headerType === 3) {
                        this._readMetaData(array, index, segments[i]);
                    }
                    vorbisHeaderFound = true;
                }
                index += segments[i];
            }
            if (!vorbisHeaderFound) {
                break;
            }
        } else {
            break;
        }
    }
};

/**
 * @method _readMp4
 * @param {Uint8Array} array
 * @private
 */
WebAudio.prototype._readMp4 = function(array) {
    if (this._readFourCharacters(array, 4) === 'ftyp') {
        var index = 0;
        while (index < array.length) {
            var size = this._readBigEndian(array, index);
            var name = this._readFourCharacters(array, index + 4);
            if (name === 'moov') {
                index += 8;
            } else {
                if (name === 'mvhd') {
                    this._sampleRate = this._readBigEndian(array, index + 20);
                }
                if (name === 'udta' || name === 'meta') {
                    this._readMetaData(array, index, size);
                }
                index += size;
                if (size <= 1) {
                    break;
                }
            }
        }
    }
};

/**
 * @method _readMetaData
 * @param {Uint8Array} array
 * @param {Number} index
 * @param {Number} size
 * @private
 */
WebAudio.prototype._readMetaData = function(array, index, size) {
    for (var i = index; i < index + size - 10; i++) {
        if (this._readFourCharacters(array, i) === 'LOOP') {
            var text = '';
            while (array[i] > 0) {
                text += String.fromCharCode(array[i++]);
            }
            if (text.match(/LOOPSTART=([0-9]+)/)) {
                this._loopStart = parseInt(RegExp.$1);
            }
            if (text.match(/LOOPLENGTH=([0-9]+)/)) {
                this._loopLength = parseInt(RegExp.$1);
            }
            if (text == 'LOOPSTART' || text == 'LOOPLENGTH') {
                var text2 = '';
                i += 16;
                while (array[i] > 0) {
                    text2 += String.fromCharCode(array[i++]);
                }
                if (text == 'LOOPSTART') {
                    this._loopStart = parseInt(text2);
                } else {
                    this._loopLength = parseInt(text2);
                }
            }
        }
    }
};

/**
 * @method _readLittleEndian
 * @param {Uint8Array} array
 * @param {Number} index
 * @private
 */
WebAudio.prototype._readLittleEndian = function(array, index) {
    return (array[index + 3] * 0x1000000 + array[index + 2] * 0x10000 +
            array[index + 1] * 0x100 + array[index + 0]);
};

/**
 * @method _readBigEndian
 * @param {Uint8Array} array
 * @param {Number} index
 * @private
 */
WebAudio.prototype._readBigEndian = function(array, index) {
    return (array[index + 0] * 0x1000000 + array[index + 1] * 0x10000 +
            array[index + 2] * 0x100 + array[index + 3]);
};

/**
 * @method _readFourCharacters
 * @param {Uint8Array} array
 * @param {Number} index
 * @private
 */
WebAudio.prototype._readFourCharacters = function(array, index) {
    var string = '';
    for (var i = 0; i < 4; i++) {
        string += String.fromCharCode(array[index + i]);
    }
    return string;
};

As you can see its kinda crazy and somewhat complex, but i should be able to help with this. As im still working on a Audio Plugin myself (DSEAudioEngine).
Basically it parse some meta (via readLoopComments() ) data right at the moment it loads before the ultra slow decoding begins.

now don't do what @Poryyg said (usually he's spot on BTW), since WebAudio._conext.currentTime does not represent the song at All!!! that starts counting the moment WebAudio is initialized and continues to count upwards indefinably until MV closes or WebAudio is reinitializes. this means setting an event condition on it directly WILL produce a buggy mess.

the MV devs Knew this and already used a work around for this, unfortunately that work around is embedded withe the API itself.

basically you will need some AudioEditing software such as Audacity, open the song who want that event in. then go to edit > metadata and you should see a list with the Author, track number etc. if you open a MV song you should also see a LOOPSTART and a LOOPLENGTH. that my good freind is where you add your Event time in Milliseconds and name it whatever you want.