MakeitTalkで、1枚の顔画像を音声から動かす

cedro

4年前

1.はじめに

　以前、1枚の顔画像を動画の様に動かす、First-Order-Motion-Model という技術をご紹介しましたが、今回は1枚の顔画像を音声から動かす、MakeitTalkという技術をご紹介します。

2.MakeitTalkとは？

　以下に、MakeitTalkのパイプラインを示します。

　音声情報と1枚の顔画像から得た顔のランドマーク情報を、唇付近を制御するブロック(Speech Content Animation)と、顔全体を制御するブロック(Speaker -Aware Animation)に入力します。

　それぞれのブロックでLSTMなどを使ってランドマークの動きを予測し、その結果を合成して音声に同期した顔のランドマークの予測を行います。

　最後に、顔のランドマークの予測を元に、Face Warp あるいは Image2Image Translation で、顔画像を生成します。

3.コード

　コードはGoogle Colabで動かす形にしてGithubに上げてありますので、それに沿って説明して行きます。自分で動かしてみたい方は、この「リンク」をクリックし表示されたノートブックの先頭にある「Colab on Web」ボタンをクリックすると動かせます。

　まず、セットアップを行います。

!git clone https://github.com/cedro3/MakeItTalk.git
%cd MakeItTalk/
!export PYTHONPATH=/content/MakeItTalk:$PYTHONPATH
!pip install -r requirements.txt
!pip install tensorboardX

!git clone https://github.com/cedro3/MakeItTalk.git

%cd MakeItTalk/

!export PYTHONPATH=/content/MakeItTalk:$PYTHONPATH

!pip install -r requirements.txt

!pip install tensorboardX

　次に、学習済みの重みをダウンロードします。

!mkdir examples/dump
!mkdir examples/ckpt
!pip install gdown
!gdown -O examples/ckpt/ckpt_autovc.pth https://drive.google.com/uc?id=1ZiwPp_h62LtjU0DwpelLUoodKPR85K7x
!gdown -O examples/ckpt/ckpt_content_branch.pth https://drive.google.com/uc?id=1r3bfEvTVl6pCNw5xwUhEglwDHjWtAqQp
!gdown -O examples/ckpt/ckpt_speaker_branch.pth https://drive.google.com/uc?id=1rV0jkyDqPW-aDJcj7xSO6Zt1zSXqn1mu
!gdown -O examples/ckpt/ckpt_116_i2i_comb.pth https://drive.google.com/uc?id=1i2LJXKp-yWKIEEgJ7C6cE3_2NirfY_0a
!gdown -O examples/dump/emb.pickle https://drive.google.com/uc?id=18-0CYl5E6ungS3H4rRSHjfYvvm-WwjTI

!mkdir examples/dump

!mkdir examples/ckpt

!pip install gdown

!gdown -O examples/ckpt/ckpt_autovc.pth https://drive.google.com/uc?id=1ZiwPp_h62LtjU0DwpelLUoodKPR85K7x

!gdown -O examples/ckpt/ckpt_content_branch.pth https://drive.google.com/uc?id=1r3bfEvTVl6pCNw5xwUhEglwDHjWtAqQp

!gdown -O examples/ckpt/ckpt_speaker_branch.pth https://drive.google.com/uc?id=1rV0jkyDqPW-aDJcj7xSO6Zt1zSXqn1mu

!gdown -O examples/ckpt/ckpt_116_i2i_comb.pth https://drive.google.com/uc?id=1i2LJXKp-yWKIEEgJ7C6cE3_2NirfY_0a

!gdown -O examples/dump/emb.pickle https://drive.google.com/uc?id=18-0CYl5E6ungS3H4rRSHjfYvvm-WwjTI

　処理する静止画の指定とパラメータの設定を行います。静止画は examples フォルダーの中にある jpg の中から拡張子なしのファイル名で指定します。

　パラメータは、瞬きをするか、動画の初期状態で口を開けるか、そして口の縦横の動きや頭の動きをどの程度にするかを設定します。

# ------ 設定 ------
default_head_name = '05'           # the image name (with no .jpg) to animate
ADD_NAIVE_EYE = True                 # whether add naive eye blink
CLOSE_INPUT_FACE_MOUTH = False       # if your image has an opened mouth, put this as True, else False
AMP_LIP_SHAPE_X = 2.                 # amplify the lip motion in horizontal direction
AMP_LIP_SHAPE_Y = 2.                 # amplify the lip motion in vertical direction
AMP_HEAD_POSE_MOTION = 0.7           # amplify the head pose motion (usually smaller than 1.0, put it to 0. for a static head pose)

# ------ 設定 ------

default_head_name = '05' # the image name (with no .jpg) to animate

ADD_NAIVE_EYE = True # whether add naive eye blink

CLOSE_INPUT_FACE_MOUTH = False # if your image has an opened mouth, put this as True, else False

AMP_LIP_SHAPE_X = 2. # amplify the lip motion in horizontal direction

AMP_LIP_SHAPE_Y = 2. # amplify the lip motion in vertical direction

AMP_HEAD_POSE_MOTION = 0.7 # amplify the head pose motion (usually smaller than 1.0, put it to 0. for a static head pose)

　それでは、音声に連動して静止画が動くmp4動画を作成します。このとき音声は、examples フォルダーの中にある＊＊＊.wav を使います。

# ------ 音声に連動して静止画が動くmp4動画を作成 ------

import sys
sys.path.append("thirdparty/AdaptiveWingLoss")
import os, glob
import numpy as np
import cv2
import argparse
from src.approaches.train_image_translation import Image_translation_block
import torch
import pickle
import face_alignment
from src.autovc.AutoVC_mel_Convertor_retrain_version import AutoVC_mel_Convertor
import shutil
import time
import util.utils as util
from scipy.signal import savgol_filter
from src.approaches.train_audio2landmark import Audio2landmark_model


parser = argparse.ArgumentParser()
parser.add_argument('--jpg', type=str, default='{}.jpg'.format(default_head_name))
parser.add_argument('--close_input_face_mouth', default=CLOSE_INPUT_FACE_MOUTH, action='store_true')

parser.add_argument('--load_AUTOVC_name', type=str, default='examples/ckpt/ckpt_autovc.pth')
parser.add_argument('--load_a2l_G_name', type=str, default='examples/ckpt/ckpt_speaker_branch.pth')
parser.add_argument('--load_a2l_C_name', type=str, default='examples/ckpt/ckpt_content_branch.pth') #ckpt_audio2landmark_c.pth')
parser.add_argument('--load_G_name', type=str, default='examples/ckpt/ckpt_116_i2i_comb.pth') #ckpt_image2image.pth') #ckpt_i2i_finetune_150.pth') #c

parser.add_argument('--amp_lip_x', type=float, default=AMP_LIP_SHAPE_X)
parser.add_argument('--amp_lip_y', type=float, default=AMP_LIP_SHAPE_Y)
parser.add_argument('--amp_pos', type=float, default=AMP_HEAD_POSE_MOTION)
parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) #  ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',
parser.add_argument('--add_audio_in', default=False, action='store_true')
parser.add_argument('--comb_fan_awing', default=False, action='store_true')
parser.add_argument('--output_folder', type=str, default='examples')

parser.add_argument('--test_end2end', default=True, action='store_true')
parser.add_argument('--dump_dir', type=str, default='', help='')
parser.add_argument('--pos_dim', default=7, type=int)
parser.add_argument('--use_prior_net', default=True, action='store_true')
parser.add_argument('--transformer_d_model', default=32, type=int)
parser.add_argument('--transformer_N', default=2, type=int)
parser.add_argument('--transformer_heads', default=2, type=int)
parser.add_argument('--spk_emb_enc_size', default=16, type=int)
parser.add_argument('--init_content_encoder', type=str, default='')
parser.add_argument('--lr', type=float, default=1e-3, help='learning rate')
parser.add_argument('--reg_lr', type=float, default=1e-6, help='weight decay')
parser.add_argument('--write', default=False, action='store_true')
parser.add_argument('--segment_batch_size', type=int, default=1, help='batch size')
parser.add_argument('--emb_coef', default=3.0, type=float)
parser.add_argument('--lambda_laplacian_smooth_loss', default=1.0, type=float)
parser.add_argument('--use_11spk_only', default=False, action='store_true')
parser.add_argument('-f')

opt_parser = parser.parse_args()

# load the image and detect its landmark
img =cv2.imread('examples/' + opt_parser.jpg)
predictor = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device='cpu', flip_input=True)
shapes = predictor.get_landmarks(img)
if (not shapes or len(shapes) != 1):
    print('Cannot detect face landmarks. Exit.')
    exit(-1)
shape_3d = shapes[0]

if(opt_parser.close_input_face_mouth):
    util.close_input_face_mouth(shape_3d)

# Simple manual adjustment to landmarks in case FAN is not accurate, e.g.
shape_3d[48:, 0] = (shape_3d[48:, 0] - np.mean(shape_3d[48:, 0])) * 1.05 + np.mean(shape_3d[48:, 0]) # wider lips
shape_3d[49:54, 1] += 0.           # thinner upper lip
shape_3d[55:60, 1] -= 1.           # thinner lower lip
shape_3d[[37,38,43,44], 1] -=2.    # larger eyes
shape_3d[[40,41,46,47], 1] +=2.    # larger eyes

shape_3d, scale, shift = util.norm_input_face(shape_3d)


# Generate input data for inference based on uploaded audio MakeItTalk/examples/*.wav
au_data = []
au_emb = []
ains = glob.glob1('examples', '*.wav')
ains = [item for item in ains if item is not 'tmp.wav']
ains.sort()
for ain in ains:
    os.system('ffmpeg -y -loglevel error -i examples/{} -ar 16000 examples/tmp.wav'.format(ain))
    shutil.copyfile('examples/tmp.wav', 'examples/{}'.format(ain))

    # au embedding
    from thirdparty.resemblyer_util.speaker_emb import get_spk_emb
    me, ae = get_spk_emb('examples/{}'.format(ain))
    au_emb.append(me.reshape(-1))

    print('Processing audio file', ain)
    c = AutoVC_mel_Convertor('examples')

    au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),
           autovc_model_path=opt_parser.load_AUTOVC_name)
    au_data += au_data_i
if(os.path.isfile('examples/tmp.wav')):
    os.remove('examples/tmp.wav')

# landmark fake placeholder
fl_data = []
rot_tran, rot_quat, anchor_t_shape = [], [], []
for au, info in au_data:
    au_length = au.shape[0]
    fl = np.zeros(shape=(au_length, 68 * 3))
    fl_data.append((fl, info))
    rot_tran.append(np.zeros(shape=(au_length, 3, 4)))
    rot_quat.append(np.zeros(shape=(au_length, 4)))
    anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))

if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):
    os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):
    os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):
    os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))
if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):
    os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))

with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:
    pickle.dump(fl_data, fp)
with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:
    pickle.dump(au_data, fp)
with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:
    gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}
    pickle.dump(gaze, fp)


# Audio-to-Landmarks prediction    
!pwd
model = Audio2landmark_model(opt_parser, jpg_shape=shape_3d)
if(len(opt_parser.reuse_train_emb_list) == 0):
    model.test(au_emb=au_emb)
else:
    model.test(au_emb=None)


# Natural face animation via Image-to-image translation    
fls = glob.glob1('examples', 'pred_fls_*.txt')
fls.sort()

for i in range(0,len(fls)):
    fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3))
    fl[:, :, 0:2] = -fl[:, :, 0:2]
    fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift

    if (ADD_NAIVE_EYE):
        fl = util.add_naive_eye(fl)

    # additional smooth
    fl = fl.reshape((-1, 204))
    fl[:, :48 * 3] = savgol_filter(fl[:, :48 * 3], 15, 3, axis=0)
    fl[:, 48*3:] = savgol_filter(fl[:, 48*3:], 5, 3, axis=0)
    fl = fl.reshape((-1, 68, 3))

    ''' Imag2image translation '''
    model = Image_translation_block(opt_parser, single_test=True)
    with torch.no_grad():
        model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])
        print('finish image2image gen')
    os.remove(os.path.join('examples', fls[i]))

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

# ------ 音声に連動して静止画が動くmp4動画を作成 ------

import sys

sys.path.append("thirdparty/AdaptiveWingLoss")

import os, glob

import numpy as np

import cv2

import argparse

from src.approaches.train_image_translation import Image_translation_block

import torch

import pickle

import face_alignment

from src.autovc.AutoVC_mel_Convertor_retrain_version import AutoVC_mel_Convertor

import shutil

import time

import util.utils as util

from scipy.signal import savgol_filter

from src.approaches.train_audio2landmark import Audio2landmark_model

parser = argparse.ArgumentParser()

parser.add_argument('--jpg', type=str, default='{}.jpg'.format(default_head_name))

parser.add_argument('--close_input_face_mouth', default=CLOSE_INPUT_FACE_MOUTH, action='store_true')

parser.add_argument('--load_AUTOVC_name', type=str, default='examples/ckpt/ckpt_autovc.pth')

parser.add_argument('--load_a2l_G_name', type=str, default='examples/ckpt/ckpt_speaker_branch.pth')

parser.add_argument('--load_a2l_C_name', type=str, default='examples/ckpt/ckpt_content_branch.pth') #ckpt_audio2landmark_c.pth')

parser.add_argument('--load_G_name', type=str, default='examples/ckpt/ckpt_116_i2i_comb.pth') #ckpt_image2image.pth') #ckpt_i2i_finetune_150.pth') #c

parser.add_argument('--amp_lip_x', type=float, default=AMP_LIP_SHAPE_X)

parser.add_argument('--amp_lip_y', type=float, default=AMP_LIP_SHAPE_Y)

parser.add_argument('--amp_pos', type=float, default=AMP_HEAD_POSE_MOTION)

parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',

parser.add_argument('--add_audio_in', default=False, action='store_true')

parser.add_argument('--comb_fan_awing', default=False, action='store_true')

parser.add_argument('--output_folder', type=str, default='examples')

parser.add_argument('--test_end2end', default=True, action='store_true')

parser.add_argument('--dump_dir', type=str, default='', help='')

parser.add_argument('--pos_dim', default=7, type=int)

parser.add_argument('--use_prior_net', default=True, action='store_true')

parser.add_argument('--transformer_d_model', default=32, type=int)

parser.add_argument('--transformer_N', default=2, type=int)

parser.add_argument('--transformer_heads', default=2, type=int)

parser.add_argument('--spk_emb_enc_size', default=16, type=int)

parser.add_argument('--init_content_encoder', type=str, default='')

parser.add_argument('--lr', type=float, default=1e-3, help='learning rate')

parser.add_argument('--reg_lr', type=float, default=1e-6, help='weight decay')

parser.add_argument('--write', default=False, action='store_true')

parser.add_argument('--segment_batch_size', type=int, default=1, help='batch size')

parser.add_argument('--emb_coef', default=3.0, type=float)

parser.add_argument('--lambda_laplacian_smooth_loss', default=1.0, type=float)

parser.add_argument('--use_11spk_only', default=False, action='store_true')

parser.add_argument('-f')

opt_parser = parser.parse_args()

# load the image and detect its landmark

img =cv2.imread('examples/' + opt_parser.jpg)

predictor = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device='cpu', flip_input=True)

shapes = predictor.get_landmarks(img)

if (not shapes or len(shapes) != 1):

print('Cannot detect face landmarks. Exit.')

exit(-1)

shape_3d = shapes[0]

if(opt_parser.close_input_face_mouth):

util.close_input_face_mouth(shape_3d)

# Simple manual adjustment to landmarks in case FAN is not accurate, e.g.

shape_3d[48:, 0] = (shape_3d[48:, 0] - np.mean(shape_3d[48:, 0])) * 1.05 + np.mean(shape_3d[48:, 0]) # wider lips

shape_3d[49:54, 1] += 0. # thinner upper lip

shape_3d[55:60, 1] -= 1. # thinner lower lip

shape_3d[[37,38,43,44], 1] -=2. # larger eyes

shape_3d[[40,41,46,47], 1] +=2. # larger eyes

shape_3d, scale, shift = util.norm_input_face(shape_3d)

# Generate input data for inference based on uploaded audio MakeItTalk/examples/*.wav

au_data = []

au_emb = []

ains = glob.glob1('examples', '*.wav')

ains = [item for item in ains if item is not 'tmp.wav']

ains.sort()

for ain in ains:

os.system('ffmpeg -y -loglevel error -i examples/{} -ar 16000 examples/tmp.wav'.format(ain))

shutil.copyfile('examples/tmp.wav', 'examples/{}'.format(ain))

# au embedding

from thirdparty.resemblyer_util.speaker_emb import get_spk_emb

me, ae = get_spk_emb('examples/{}'.format(ain))

au_emb.append(me.reshape(-1))

print('Processing audio file', ain)

c = AutoVC_mel_Convertor('examples')

au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),

autovc_model_path=opt_parser.load_AUTOVC_name)

au_data += au_data_i

if(os.path.isfile('examples/tmp.wav')):

os.remove('examples/tmp.wav')

# landmark fake placeholder

fl_data = []

rot_tran, rot_quat, anchor_t_shape = [], [], []

for au, info in au_data:

au_length = au.shape[0]

fl = np.zeros(shape=(au_length, 68 * 3))

fl_data.append((fl, info))

rot_tran.append(np.zeros(shape=(au_length, 3, 4)))

rot_quat.append(np.zeros(shape=(au_length, 4)))

anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))

if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):

os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))

if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):

os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))

if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):

os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))

if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):

os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))

with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:

pickle.dump(fl_data, fp)

with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:

pickle.dump(au_data, fp)

with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:

gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}

pickle.dump(gaze, fp)

# Audio-to-Landmarks prediction

!pwd

model = Audio2landmark_model(opt_parser, jpg_shape=shape_3d)

if(len(opt_parser.reuse_train_emb_list) == 0):

model.test(au_emb=au_emb)

else:

model.test(au_emb=None)

# Natural face animation via Image-to-image translation

fls = glob.glob1('examples', 'pred_fls_*.txt')

fls.sort()

for i in range(0,len(fls)):

fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3))

fl[:, :, 0:2] = -fl[:, :, 0:2]

fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift

if (ADD_NAIVE_EYE):

fl = util.add_naive_eye(fl)

# additional smooth

fl = fl.reshape((-1, 204))

fl[:, :48 * 3] = savgol_filter(fl[:, :48 * 3], 15, 3, axis=0)

fl[:, 48*3:] = savgol_filter(fl[:, 48*3:], 5, 3, axis=0)

fl = fl.reshape((-1, 68, 3))

''' Imag2image translation '''

model = Image_translation_block(opt_parser, single_test=True)

with torch.no_grad():

model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])

print('finish image2image gen')

os.remove(os.path.join('examples', fls[i]))

　作成したmp4動画を再生します。

# ------ mp4の再生 ------

from IPython.display import HTML
from base64 import b64encode

for ain in ains:
  OUTPUT_MP4_NAME = '{}_pred_fls_{}_audio_embed.mp4'.format(
    opt_parser.jpg.split('.')[0],
    ain.split('.')[0]
    )
  mp4 = open('examples/{}'.format(OUTPUT_MP4_NAME),'rb').read()
  data_url = "data:video/mp4;base64," + b64encode(mp4).decode()

  print('Display animation: examples/{}'.format(OUTPUT_MP4_NAME))
  display(HTML("""
  <video width=600 controls>
        <source src="%s" type="video/mp4">
  </video>
  """ % data_url))