backend.py

# -*- coding: utf-8 -*-

"""
# File name:    backend.py
# Time :        2022/2/20 17:06
# Author:       xyguoo@163.com
# Description:  This is the backend interface for editing. When you want to customize custom editing and
                modification functions (such as replacing mask), it is recommended to call these interfaces directly.
                In the finally main section of this file, there is an example of using Backend
"""

import sys
sys.path.append('.')

import copy
import os
from datetime import datetime
import cv2
import torch

from global_value_utils import HAIR_IDX, TEMP_FOLDER
from my_torchlib.train_utils import generate_noise
from shape_branch.shape_util import mask_label_to_one_hot, mask_one_hot_to_label, split_hair_face
from util.mask_color_util import mask_to_rgb
from wrap_codes.mask_adaptor import wrap_by_imgs

from hair_editor import HairEditor
from util.color_from_hsv_to_gaussian import DistTranslation


class LatentRepresentation:
    def __init__(self):
        self.color = None
        self.curliness = None
        self.shape = None
        self.texture = None
        self.face = None


class Backend(HairEditor):
    """
    This is the main interface set, please call these interfaces directly to customize custom editing
    """

    def __init__(self, maximum_value_fe, blending=True, temp_path=os.path.join(TEMP_FOLDER, 'demo_output')):
        """
        :param maximum_value_fe: The max value in sliding bar of the frontend ui. If you just use the backend,
            but not use the frontend, this value is
        :param blending: Whether use poisson blending between hair region and others region
        :param temp_path: the dir that save temp content, which can be removed after execution
        """
        super().__init__(True, True)
        self.target_img = None
        self.input_img = None
        self.target_mask = None
        self.input_mask = None
        self.cur_latent = None
        self.target_latent = None
        self.cur_mask = None
        self.input_sean_code = None
        self.target_size = 256
        self.maximum_value_fe = maximum_value_fe
        self.temp_path = temp_path
        self.blending = blending
        self.dist_translation = DistTranslation()

    def parse_img(self, img_rgb, target_img=False):
        """
        mask the input image
        :param img_rgb: with channel order rgb
        :param target_img: is the target image?
        """
        img_ts = cv2.resize(img_rgb, (self.target_size, self.target_size))
        mask = self.get_mask(img_rgb)
        lr = LatentRepresentation()

        # infer shape
        if target_img:
            out_mask = None
        else:
            mask_batch = self.preprocess_mask(mask)
            mask_tensor = torch.tensor(mask_batch, dtype=torch.uint8, device=self.device)
            mask_one_hot = mask_label_to_one_hot(mask_tensor)
            hair, face = split_hair_face(mask_one_hot)
            hair_code = self.mask_generator.forward_hair_encoder(hair, testing=True)
            face_code = self.mask_generator.forward_face_encoder(face)
            lr.shape = hair_code
            lr.face = face_code
            out_mask = self.mask_generator.forward_decode_by_code(hair_code, face_code)
            out_mask = mask_one_hot_to_label(out_mask).cpu().numpy()[0]

        # infer feature
        input_code = self.get_code(self.preprocess_img(img_rgb), mask_batch)
        hair_feature = input_code[:, HAIR_IDX]

        out_color = self.feature_rgb_predictor({'code': hair_feature})

        c = out_color['rgb_mean'].detach().cpu().numpy()
        c_hsv = cv2.cvtColor(c[None, ...].astype('uint8'), cv2.COLOR_RGB2HSV)
        c_hsv = torch.tensor(c_hsv).to(self.device)[0]
        lr.color = {'hsv': c_hsv, 'pca_std': out_color['pca_std']}

        out_enc = self.feature_encoder({'code': hair_feature})
        lr.curliness = out_enc['noise_curliness']
        lr.texture = out_enc['noise']
        return img_ts, out_mask, lr, mask, input_code, hair_feature

    def tensor_hsv_to_rgb(self, hsv):
        """
        input tensor with hsv and convert to rgb
        """
        c = hsv.detach().cpu().numpy()
        c_rgb = cv2.cvtColor(c[None, ...].astype('uint8'), cv2.COLOR_HSV2RGB)
        c_rgb = torch.tensor(c_rgb).to(self.device)[0]
        return c_rgb

    def tensor_rgb_to_hsv(self, rgb):
        """
        input tensor with rgb and convert to hsv
        """

        c = rgb.detach().cpu().numpy()
        c_hsv = cv2.cvtColor(c[None, ...].astype('uint8'), cv2.COLOR_RGB2HSV)
        c_hsv = torch.tensor(c_hsv).to(self.device)[0]
        return c_hsv

    def set_input_img(self, img_rgb):
        """
        set and parse the input image
        :param img_rgb:
        """
        self.input_img, self.cur_mask, self.cur_latent, \
        self.input_mask, self.input_sean_code, self.input_hair_feature = self.parse_img(img_rgb)
        input_mask_show = mask_to_rgb(self.cur_mask, draw_type=1)
        return self.input_img, input_mask_show

    def set_target_img(self, img_rgb):
        """
        set and parse the target image
        :param img_rgb:
        """
        self.target_img, _, self.target_latent, \
        self.target_mask, _, self.target_hair_feature = self.parse_img(img_rgb)
        input_maks_show = mask_to_rgb(self.target_mask, draw_type=1)
        return self.target_img, input_maks_show

    def output(self, target_latent=None, feature=None):
        """
        generate an color_texture image
        :param target_latent: if the latent representation of target is not provided,
                `self.cur_latent` and `self.cur_mask` will be used
        :param feature: if edited hair feature X is not provided, it will be generated with color and texture branch
        :return: color_texture image
        """
        if target_latent is None:
            target_latent = self.cur_latent
            target_mask = self.cur_mask
        else:
            target_mask = self.refresh_cur_mask(target_latent)[0]

        if 'rgb_mean' in target_latent.color:
            target_color_rgb = self.target_latent.color['rgb_mean']
        else:
            target_color_rgb = self.tensor_hsv_to_rgb(target_latent.color['hsv'])

        if feature is None:
            data = {'noise': target_latent.texture, 'noise_curliness': target_latent.curliness,
                    'rgb_mean': target_color_rgb, 'pca_std': target_latent.color['pca_std']}
            feature = self.feature_generator(data)['code']
        self.input_sean_code[:, HAIR_IDX] = feature
        edit_img = self.gen_img(self.input_sean_code, target_mask[None, None, ...])
        output_img, _ = self.postprocess_blending(self.input_img, edit_img, self.input_mask, target_mask,
                                                  blending=self.blending)

        return output_img

    def change_curliness(self, val):
        """
        change the latent representation of curliness of texture
        """
        self.cur_latent.curliness[0] = val

    # def change_color(self, val, idx):
    #     val = (val + self.maximum_value_fe) / 2 / self.maximum_value_fe
    #     if idx == 3:
    #         self.cur_latent.color['pca_std'][0] = val * 100 + 20
    #     else:
    #         if idx == 0:
    #             val *= 179
    #         if idx == 1:
    #             val *= 255
    #         if idx == 2:
    #             val *= 255
    #         self.cur_latent.color['hsv'][0][idx] = val

    def change_color(self, val, idx):
        """
        change the latent representation of color
        idx=0 is hue, idx=1 is saturation, idx=2 is brightness, idx=3 is variance

        please note that when idx=3, the variance is not obey gaussian,
        the value range is in [-self.maximum_value_fe, self.maximum_value_fe]
        """
        if idx == 3:
            val = (val + self.maximum_value_fe) / 2 / self.maximum_value_fe
            self.cur_latent.color['pca_std'][0] = val * 100 + 20
        else:
            val = self.dist_translation.gaussian_to_val(idx, val)
            self.cur_latent.color['hsv'][0][idx] = val

    def change_shape(self, val, idx):
        """
        change the latent representation of shape
        :param val: latent value
        :param idx: for current checkpoint, idx=0 is length, idx=1 is volumn, idx=2 is bangs_direction, idx=3 is bangs
        """
        self.continue_change_with_direction('shape', self.shape_dirs[idx], val)
        self.refresh_cur_mask()

    def change_texture(self, val, idx):
        """
        change the latent representation of texture
        :param val: latent value
        :param idx: for current checkpoint, idx=0 is smoothness, idx=1 is thickness
        """
        self.continue_change_with_direction('texture', self.texture_dirs[idx], val)

    def get_curliness_be2fe(self):
        """
        convert the latent representation to the value of sliding bar in the frontend of Pyqt ui, and return it
        """
        return self.cur_latent.curliness[0]

    def get_color_be2fe(self):
        """
        convert the latent representation to the value of sliding bar in the frontend of Pyqt ui, and return it
        """
        c_hsv = self.cur_latent.color['hsv'].detach().cpu().numpy()[0]
        # color0 = c_hsv[0] / 179 * 2 * self.maximum_value_fe - self.maximum_value_fe
        # color1 = c_hsv[1] / 255 * 2 * self.maximum_value_fe - self.maximum_value_fe
        # color2 = c_hsv[2] / 255 * 2 * self.maximum_value_fe - self.maximum_value_fe
        color0 = self.dist_translation.val_to_gaussian(0, c_hsv[0])
        color1 = self.dist_translation.val_to_gaussian(1, c_hsv[1])
        color2 = self.dist_translation.val_to_gaussian(2, c_hsv[2])
        var_fe = (self.cur_latent.color['pca_std'][0] - 20) / 100 * 2 * self.maximum_value_fe - self.maximum_value_fe
        return color0, color1, color2, var_fe

    def get_shape_be2fe(self):
        """
        convert the latent representation to the value of sliding bar in the frontend of Pyqt ui, and return it
        """
        res = []
        for idx in range(4):
            res.append(torch.dot(self.cur_latent.shape[0], self.shape_dirs[idx]))
        return res

    def get_texture_be2fe(self):
        """
        convert the latent representation to the value of sliding bar in the frontend of Pyqt ui, and return it
        """
        res = []
        for idx in range(2):
            res.append(torch.dot(self.cur_latent.texture[0], self.texture_dirs[idx]))
        return res

    def transfer_latent_representation(self, flag, refresh=True):
        """
        transfer the latent representation of target image to input image.
        i.e., transfer from self.target_latent to self.cur_latent
        :param flag: 'color', 'texture' or 'shape'
        :param refresh: whether refresh mask
        :return:
        """
        if flag == 'shape':
            wt, _ = wrap_by_imgs(self.target_img, self.input_img, wrap_temp_folder=self.temp_path,
                                 need_crop=False)
            wt = self.preprocess_mask(wt)
            self.warp_target = wt[0, 0]
            mask_tensor = torch.tensor(wt, dtype=torch.uint8, device=self.device)
            mask_one_hot = mask_label_to_one_hot(mask_tensor)
            hair, face = split_hair_face(mask_one_hot)
            hair_code = self.mask_generator.forward_hair_encoder(hair, testing=True)
            face_code = self.mask_generator.forward_face_encoder(face)
            self.target_latent.shape = hair_code
            self.target_latent.face = face_code

            self.refresh_cur_mask()

        target_att = self.target_latent.__getattribute__(flag)
        if isinstance(target_att, torch.Tensor):
            self.cur_latent.__setattr__(flag, target_att.clone())
        else:
            cp_dict = copy.copy(target_att)
            for ke in cp_dict:
                cp_dict[ke] = cp_dict[ke].clone()
            self.cur_latent.__setattr__(flag, cp_dict)

        if flag == 'shape' and refresh:
            self.refresh_cur_mask()

        if flag == 'texture':
            self.transfer_latent_representation('curliness')

    def refresh_cur_mask(self, target_latent=None):
        """
        refresh and generate current mask
        :param target_latent: if no target latent is given, self.cur_mask will be used
        :return:
        """
        if target_latent is None:
            target_latent = self.cur_latent
        out_mask = self.mask_generator.forward_decode_by_code(target_latent.shape, target_latent.face)
        out_mask = mask_one_hot_to_label(out_mask).cpu().numpy()[0]
        self.cur_mask = out_mask
        return out_mask, mask_to_rgb(out_mask, draw_type=1)

    def get_cur_mask(self):
        """
        get the mask, which can be visited
        """
        return mask_to_rgb(self.cur_mask, draw_type=1)

    def interpolate_hsv(self, hsv1, hsv2, alpha):
        """
        final hsv = hsv1 * (1 - alpha) + hsv2 * alpha
        return: final hsv
        """
        rgb1 = self.tensor_hsv_to_rgb(hsv1)
        rgb2 = self.tensor_hsv_to_rgb(hsv2)
        rgb = rgb1 * (1 - alpha) + rgb2 * alpha
        inter_hsv = self.tensor_rgb_to_hsv(rgb)
        return inter_hsv

    def interpolate_triple(self, latent1, latent2, latent3, alpha1, alpha2, alpha3):
        """
        (latent1 * alpha1 + latent2 * alpha2 + latent3 * alpha3) / (alpha1 + alpha2 + alpha3)
        :return: final latent
        """
        latent12 = self.interpolate(latent1, latent2, alpha2 / (alpha1 + alpha2))
        latent_res = self.interpolate(latent12, latent3, alpha3)
        return latent_res

    def interpolate(self, latent1, latent2, alpha):
        """
        final latent = latent1 * (1 - alpha) + latent2 * alpha
        :return: final latent
        """
        result_latent = LatentRepresentation()
        for att in ['curliness', 'shape', 'texture']:
            result_latent.__setattr__(att, latent1.__getattribute__(att) * (1 - alpha) +
                                      latent2.__getattribute__(att) * alpha)
        color_dic = {}
        color_dic['pca_std'] = latent1.color['pca_std'] * (1 - alpha) + latent2.color['pca_std'] * alpha
        color_dic['hsv'] = self.interpolate_hsv(latent1.color['hsv'], latent2.color['hsv'], alpha)

        result_latent.color = color_dic
        result_latent.face = self.cur_latent.face
        return result_latent

    def interpolate_each_att(self, latent1, latent2, alpha, att_name):
        """
        interpolate a certain latent
        :param att_name: curliness, shape, texture, color
        :return: final full latent
        """
        result_latent = LatentRepresentation()
        for att in ['curliness', 'shape', 'texture']:
            result_latent.__setattr__(att, self.cur_latent.__getattribute__(att).clone())

        if att_name == 'shape':
            # keep color
            color_dic = {}
            for semantic in ['hsv', 'pca_std']:
                color_dic[semantic] = self.cur_latent.color[semantic].clone()
            result_latent.__setattr__(att_name, latent1.__getattribute__(att_name) * (1 - alpha) +
                                      latent2.__getattribute__(att_name) * alpha)
        elif att_name in ['curliness', 'texture']:
            # keep color
            color_dic = {}
            for semantic in ['hsv', 'pca_std']:
                color_dic[semantic] = self.cur_latent.color[semantic].clone()
            result_latent.__setattr__('curliness', latent1.__getattribute__('curliness') * (1 - alpha) +
                                      latent2.__getattribute__('curliness') * alpha)
            result_latent.__setattr__('texture', latent1.__getattribute__('texture') * (1 - alpha) +
                                      latent2.__getattribute__('texture') * alpha)
        else:
            color_dic = {}
            # for semantic in ['hsv', 'pca_std']:
            color_dic['pca_std'] = latent1.color['pca_std'] * (1 - alpha) + latent2.color['pca_std'] * alpha
            color_dic['hsv'] = self.interpolate_hsv(latent1.color['hsv'], latent2.color['hsv'], alpha)

        result_latent.color = color_dic
        result_latent.face = self.cur_latent.face

        return result_latent

    @staticmethod
    def show_hair_region(mask, non_hair_value=0):
        """
        Get hair region, and full none hair  region as `non_hair_value`
        :param mask:
        :param non_hair_value:
        :return:
        """
        mask_rgb = mask_to_rgb(mask, draw_type=1)
        mask_rgb[mask != HAIR_IDX] = non_hair_value
        return mask_rgb

    def directly_change_hair_mask(self, hair_mask):
        """
        Directly replace hair region with a hair_mask. This is a recommend method to imply mask transfer.
        :param hair_mask:
        :return:
        """
        hair_mask = hair_mask == HAIR_IDX
        face_logit = self.mask_generator.forward_face_decoder(self.cur_latent.face)
        hair_logit = torch.tensor(hair_mask)[None, None, ...].type_as(face_logit).to(self.device)
        hair_logit = hair_logit * (face_logit.max() - face_logit.min() + 2) + face_logit.min() - 1
        mask = self.mask_generator.forward_decoder(hair_logit, face_logit)
        self.cur_mask = mask_one_hot_to_label(mask).cpu().numpy()[0]

    def get_random_texture(self):
        """
        sample a texture latent code randomly
        :return:
        """
        random_latent = generate_noise(1, 8)
        random_latent = random_latent.to(self.device)
        self.cur_latent.texture = random_latent

    def get_random_shape(self):
        """
        sample a shape latent code randomly
        :return:
        """
        random_latent = generate_noise(1, 16)
        random_latent = random_latent.to(self.device)
        self.cur_latent.shape = random_latent
        self.refresh_cur_mask()

    def get_random_curliness(self):
        """
        sample a curliness latent code randomly
        :return:
        """
        random_latent = generate_noise(1, 1)
        random_latent = random_latent.to(self.device)
        self.cur_latent.curliness = random_latent

    def continue_change_with_direction(self, att_name, direction, val):
        """
        change the latent code value on a projection direction
        :param att_name: shape or texture
        :param direction: projection direction
        :param val: coordinate on this projection direction
        :return:
        """
        att = self.cur_latent.__getattribute__(att_name)
        att = att + (val - torch.dot(att[0], direction)) * direction
        self.cur_latent.__setattr__(att_name, att)
        if att_name == 'shape':
            self.refresh_cur_mask()


"""
This is a example of using Backend for costume editing
"""
if __name__ == '__main__':
    be = Backend(2.5)
    from util.imutil import read_rgb, write_rgb

    input_image = read_rgb('imgs/input_img.png')
    target_image = read_rgb('imgs/out_img.png')

    """
    If the image need crop
    """
    # input_image = be.crop_face(input_image)
    # target_image = be.crop_face(target_image)

    start_time = datetime.now()

    input_image = cv2.resize(input_image, (256, 256))

    be.set_input_img(input_image)
    be.set_target_img(target_image)

    # transfer all latent code from target image to input image
    be.transfer_latent_representation('color')
    be.transfer_latent_representation('texture')
    be.transfer_latent_representation('shape')

    # change the variance manually
    #be.change_color(1.0, 2)

    #out_mask = be.get_mask(input_image)
    output_img = be.output()
    write_rgb('temp.png', output_img)
    print(f'Image process takes {datetime.now() - start_time}')

    # above is the output image

    #im2 = read_rgb('imgs/00037.png')
    #im2 = cv2.resize(im2, (256, 256))
    #be.set_target_img(im2)
    #be.transfer_latent_representation('shape')
    #output_img2 = be.output()
    # above is the output image 2