matcha_tts_common_voice_01_en_001

Matcha-TTS CommonVoice EN001

you can test variation models | Github Demo

Source Audio

https://commonvoice.mozilla.org/en/datasets Common Voice Corpus 1

I called audios 42da7f26(head-audio-id)_290(files) EN001 (No plan to include audios in this repo)

Any Good point?

LJSpeech is much better quality,but it's female voice.This one is men.

VCTK 109 voices are similar quality,but that is ODC-By License.

This audio is just under MIT more easy to continue training or something.

however I recommend you use VCTK,ODC-By License is not so problem.I'm going to create new voices with this in future.

How to Train

Train with IPA text(this folk) https://github.com/akjava/Matcha-TTS-Japanese

check this repo's config files. however there are no audio copy tools.TODO later

Files Info

checkpoints

Matcha-TTS checkpoint - epoch seems big but train with only 290 audios

Sadly I lost between 3599 - 4499 checkpoints.I'm sorry.

As I see Training metrics. 6399 seems overfitting,however my english listening skill is poor and I cant evaluate it.

ONNX

github codes - see sourcecode github Page - Test Onnx Example

onnx simplified loading speed is now 1.5 times faster.

from onnxsim import simplify
import onnx

model = onnx.load("en001_6399_T2.onnx")
model_simp, check = simplify(model)

onnx.save(model_simp, "en001_6399_T2_simplify.onnx")

timesteps is default(5) ,small time steps ;The infer speed is somewhat faster, but the quality is lower.

If you need original onnx do like official way

python -m matcha.onnx.export checkpoint_epoch=5699.ckpt en001_5699t2.onnx  --vocoder-name hifigan_T2_v1 --n-timesteps 5 --vocoder-checkpoint generator_v1
python -m matcha.onnx.export checkpoint_epoch=5699.ckpt en001_5699.onnx  --vocoder-name hifigan_univ_v1 --n-timesteps 5 --vocoder-checkpoint g_02500000

• T2 means Vocoder is hifigan_T2_v1
• Unif means Voder is hifigan_univ_v1

you can quantize this onnx,but 3 times smaller, but 4-5 times slower,that why I did't include that.

from onnxruntime.quantization import quantize_dynamic, QuantType
quantized_model = quantize_dynamic(src_model_path, dst_model_path, weight_type=QuantType.QUInt8)

To use onnx need something,below is old sample

const _pad = "_";
const _punctuation = ";:,.!?¡¿—…\"«»“” ";
const _letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const _letters_ipa = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ";

// below code called Spread syntax
const Symbols = [_pad, ..._punctuation, ..._letters, ..._letters_ipa];

const SpaceId = Symbols.indexOf(' ');

const symbolToId = {};
const idToSymbol = {};

// initialize symbolToId and  idToSymbol
for (let i = 0; i < Symbols.length; i++) {
symbolToId[Symbols[i]] = i;
idToSymbol[i] = Symbols[i];
}

class MatchaOnnx {
    constructor() {
    }
    async load_model(model_path,options={}){
        this.session = await ort.InferenceSession.create(model_path,options);
    }

        get_output_names_html(){
        if (typeof this.session=='undefined'){
            return null
        }
        let outputNamesString = '[outputs]<br>';
        const outputNames = this.session.outputNames;
        for (let outputName of outputNames) {
            console.log(outputName)
            outputNamesString+=outputName+"<br>"
        }
        return outputNamesString.trim()
    }

    get_input_names_html(){
        if (typeof this.session=='undefined'){
            return null
        }
        
        let inputNamesString = '[Inputs]<br>';
        const inputNames = this.session.inputNames;

        for (let inputName of inputNames) {
            console.log(inputName)
            inputNamesString+=inputName+"<br>"
        }
        return inputNamesString.trim()
    }


    processText(text) {
    const x = this.intersperse(this.textToSequence(text));
    const x_phones = this.sequenceToText(x);
    const textList = [];
    for (let i = 1; i < x_phones.length; i += 2) {
    textList.push(x_phones[i]);
    }

    return {
    x: x,
    x_length: x.length,
    x_phones: x_phones,
    x_phones_label: textList.join(""),
    };
}


    basicCleaners2(text, lowercase = false) {
    if (lowercase) {
    text = text.toLowerCase();
    }
    text = text.replace(/\s+/g, " ");
    return text;
}

    textToSequence(text) {
    const sequenceList = [];
    const clean_text = this.basicCleaners2(text);
    for (let i = 0; i < clean_text.length; i++) {
    const symbol = clean_text[i];
    sequenceList.push(symbolToId[symbol]);
    }
    return sequenceList;
}

    intersperse(sequence, item = 0) {
    const sequenceList = [item];
    for (let i = 0; i < sequence.length; i++) {
    sequenceList.push(sequence[i]);
    sequenceList.push(item);
    }
    return sequenceList;
    }

    sequenceToText(sequence) {
    const textList = [];
    for (let i = 0; i < sequence.length; i++) {
    const symbol = idToSymbol[sequence[i]];
    textList.push(symbol);
    }
    return textList.join("");
}

async infer(text, temperature, speed) {
    console.log(this.session)
    const dic = this.processText(text);
console.log(`x:${dic.x.join(", ")}`);
console.log(`x_length:${dic.x_length}`);
console.log(`x_phones_label:${dic.x_phones_label}`);
    
// Prepare input tensors (assuming your ONNX Runtime library uses similar syntax)
//const x_tensor = new this.session.Tensor('long', dic.x, [1, dic.x.length]);
//const x_length_tensor = new this.session.Tensor('long', [dic.x.length], [1]);
//const scales_tensor = new this.session.Tensor('float', [temperature, speed], [2]);

const dataX = new BigInt64Array(dic.x.length)
for (let i = 0; i < dic.x.length; i++) {
    //console.log(dic.x[i])
    dataX[i] = BigInt(dic.x[i]); // Convert each number to a BigInt
    }
const data_x_length = new BigInt64Array(1)
data_x_length[0] = BigInt(dic.x_length)

//const dataX = Int32Array.from([dic.x_length])
const tensorX = new ort.Tensor('int64', dataX, [1, dic.x.length]);
// const data_x_length = Int32Array.from([dic.x_length])
const tensor_x_length = new ort.Tensor('int64', data_x_length, [1]);
const data_scale = Float32Array.from( [temperature, speed])
const tensor_scale = new ort.Tensor('float32', data_scale, [2]);


// Run inference
const output = await this.session.run({
x: tensorX,
x_lengths: tensor_x_length,
scales: tensor_scale,
});
console.log(output)
// Extract output (assuming your ONNX Runtime library uses similar syntax)
const wav_array = output.wav.data;
console.log(wav_array[0]);
console.log(wav_array.length);

const x_lengths_array = output.wav_lengths.data;
console.log(x_lengths_array.join(", "));

return wav_array;
}


}

convert to wav

function webWavPlay(f32array){
    blob = float32ArrayToWav(f32array)
    url = createObjectUrlFromBlob(blob)
    console.log(url)
    playAudioFromUrl(url)
}

function createObjectUrlFromBlob(blob) {
    const url = URL.createObjectURL(blob);
    return url;
    }

function playAudioFromUrl(url) {
    const audio = new Audio(url);
    audio.play().catch(error => console.error('Failed to play audio:', error));
    }

    
//I copied
//https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-de/blob/main/app-tts.js
        // this function is copied/modified from
// https://gist.github.com/meziantou/edb7217fddfbb70e899e
function float32ArrayToWav(floatSamples, sampleRate=22050) {
        let samples = new Int16Array(floatSamples.length);
        for (let i = 0; i < samples.length; ++i) {
          let s = floatSamples[i];
          if (s >= 1)
            s = 1;
          else if (s <= -1)
            s = -1;
      
          samples[i] = s * 32767;
        }
      
        let buf = new ArrayBuffer(44 + samples.length * 2);
        var view = new DataView(buf);
      
        // http://soundfile.sapp.org/doc/WaveFormat/
        //                   F F I R
        view.setUint32(0, 0x46464952, true);               // chunkID
        view.setUint32(4, 36 + samples.length * 2, true);  // chunkSize
        //                   E V A W
        view.setUint32(8, 0x45564157, true);  // format
                                              //
        //                      t m f
        view.setUint32(12, 0x20746d66, true);          // subchunk1ID
        view.setUint32(16, 16, true);                  // subchunk1Size, 16 for PCM
        view.setUint32(20, 1, true);                   // audioFormat, 1 for PCM
        view.setUint16(22, 1, true);                   // numChannels: 1 channel
        view.setUint32(24, sampleRate, true);          // sampleRate
        view.setUint32(28, sampleRate * 2, true);      // byteRate
        view.setUint16(32, 2, true);                   // blockAlign
        view.setUint16(34, 16, true);                  // bitsPerSample
        view.setUint32(36, 0x61746164, true);          // Subchunk2ID
        view.setUint32(40, samples.length * 2, true);  // subchunk2Size
      
        let offset = 44;
        for (let i = 0; i < samples.length; ++i) {
          view.setInt16(offset, samples[i], true);
          offset += 2;
        }
      
        return new Blob([view], {type: 'audio/wav'});
      }

Audio

I cut with VAD tools and denoise with resemble-enhance