Automate Lip-sync in Blender using Python!
Learn how to use Python and cmusphinx/pocketsphinx to create a speech to animation system in Blender!
Feel free to ask me any questions about the project! I'd be happy to help with any issues!
YouTube wouldn't let me include the code while there was a angled bracket. If you copy the code below make sure you replace the unicode bracket I used with an actual bracket for this line - (if seg.end_frame＞frame_end:).
Also make sure to update the directories to reflect your install locations.
MAIN PYTHON CODE BEGINS BELOW
import pyaudio
from pocketsphinx import Decoder, Config
import os
Model = r"C:/Users/USER/Documents/cmusphinx/cmusphinx-en-us-5.2"
#Lookup table for phonemes to visemes
phoneme_to_viseme = {
"AA" : "AEI" , "AE" : "AEI" , "AO" : "AEI", "AW" : "AEI", "AY" : "AEI", "EH" : "AEI", "ER" : "AEI", "EY" : "AEI", "IH" : "AEI", "IY" : "AEI",
"AH" : "UHHAH" , "HH" : "UHHAH",
"B" : "BMP" , "M" : "BMP", "P" : "BMP", "SIL" : "BMP",
"CH" : "CHJSH", "JH" : "CHJSH", "SH" : "CHJSH",
"D" : "DGKN", "G" : "DGKN", "K" : "DGKN", "N" : "DGKN", "NG" : "DGKN", "S" : "DGKN", "T" : "DGKN", "Y" : "DGKN", "Z" : "DGKN", "ZH" : "DGKN",
"F" : "FV", "V" : "FV",
"L" : "L",
"OW" : "O", "OY" : "O",
"R" : "R",
"TH" : "TH",
"UH" : "UHHAH",
"UW" : "OOUUW", "W" : "OOUUW"
}
#Set up the model
config = Config()
config.set_string("-lm", None)
config.set_string("-dict", None)
config.set_string("-jsgf", None)
config.set_string("-fsg", None)
config.set_string("-keyphrase", None)
config.set_string("-kws", None)
config.set_string("-lmctl", None)
config.set_string("-hmm", os.path.join(Model, "cmusphinx-en-us-5.2"))
config.set_string("-allphone", "en-us-phone.lm.dmp")
#Adjust cmusphinx
config.set_float("-lw", 2.0)
config.set_float("-beam", 1e-20)
config.set_float("-pbeam", 1e-20)
#Set Decoder to config specs
decoder = Decoder(config)
#Open incoming mic, start the stream for decoding
audiostream = pyaudio.PyAudio()
stream = audiostream.open(format = pyaudio.paInt16, channels = 1, rate = 16000, input=True, output = True, frames_per_buffer = 1024)
stream.start_stream()
decoder.start_utt()
print("debug_begin")
frame_end = 0
#Main loop
while True:
buf = stream.read(1024, exception_on_overflow=False)
if buf:
decoder.process_raw(buf, False, False)
#Detect phoneme segment
segments = decoder.seg()
if segments is not None:
for seg in segments:
#Stop from repeating segment output
if seg.end_frame＞ frame_end:
#Lookup corresponding viseme
phoneme = seg.word
viseme = phoneme_to_viseme.get(phoneme, "BMP")
print(viseme)
#Write current viseme to text
with open("current_viseme.txt", "w") as f:
f.write(viseme)
frame_end = seg.end_frame
else:
break
decoder.end_utt()
END OF MAIN PYTHON SCRIPT
BLENDER PYTHON SCRIPT BEGINS BELOW
import bpy
import os
obj = bpy.data.objects["Mouth"]
keys = obj.data.shape_keys.key_blocks
def update_viseme(scene):
try:
with open(r"C:\Users\USER\Documents\current_viseme.txt") as f:
viseme = f.read().strip()
print(viseme)
for key in keys:
key.value = 0.0
key.keyframe_insert(data_path='value', frame=bpy.context.scene.frame_current)
if viseme in keys:
keys[viseme].value = 1.0
keys[viseme].keyframe_insert(data_path='value', frame=bpy.context.scene.frame_current)
else:
print("viseme missing", viseme)
except:
pass
bpy.app.handlers.frame_change_pre.clear()
bpy.app.handlers.frame_change_pre.append(update_viseme)