import os import cv2 import glob from pathlib import Path import utils import argparse import numpy as np import torch from utils import convert_state_dict from models import restormer_arch from data.preprocess.crop_merge_image import stride_integral os.sys.path.append('./data/MBD/') from data.MBD.infer import net1_net2_infer_single_im # Единая константа для всех задач MAX_SIZE = 1800 def _resize_if_needed(img, max_size=MAX_SIZE): """Ресайз с сохранением пропорций. Возвращает (img, was_resized).""" h, w = img.shape[:2] scale = min(max_size / max(h, w), 1.0) if scale < 1.0: new_w, new_h = int(w * scale), int(h * scale) img = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_AREA) print(f'[DocRes] Resized to {new_w}x{new_h}') return img, True return img, False def dewarp_prompt(img): mask = net1_net2_infer_single_im(img,'data/MBD/checkpoint/mbd.pkl') base_coord = utils.getBasecoord(256,256)/256 img[mask==0]=0 mask = cv2.resize(mask,(256,256))/255 return img,np.concatenate((base_coord,np.expand_dims(mask,-1)),-1) def deshadow_prompt(img): h,w = img.shape[:2] img = cv2.resize(img,(1024,1024)) rgb_planes = cv2.split(img) result_planes = [] result_norm_planes = [] bg_imgs = [] for plane in rgb_planes: dilated_img = cv2.dilate(plane, np.ones((7,7), np.uint8)) bg_img = cv2.medianBlur(dilated_img, 21) bg_imgs.append(bg_img) diff_img = 255 - cv2.absdiff(plane, bg_img) norm_img = cv2.normalize(diff_img,None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1) result_planes.append(diff_img) result_norm_planes.append(norm_img) bg_imgs = cv2.merge(bg_imgs) bg_imgs = cv2.resize(bg_imgs,(w,h)) result_norm = cv2.merge(result_norm_planes) result_norm[result_norm==0]=1 shadow_map = np.clip(img.astype(float)/result_norm.astype(float)*255,0,255).astype(np.uint8) shadow_map = cv2.resize(shadow_map,(w,h)) shadow_map = cv2.cvtColor(shadow_map,cv2.COLOR_BGR2GRAY) shadow_map = cv2.cvtColor(shadow_map,cv2.COLOR_GRAY2BGR) return bg_imgs def deblur_prompt(img): x = cv2.Sobel(img,cv2.CV_16S,1,0) y = cv2.Sobel(img,cv2.CV_16S,0,1) absX = cv2.convertScaleAbs(x) absY = cv2.convertScaleAbs(y) high_frequency = cv2.addWeighted(absX,0.5,absY,0.5,0) high_frequency = cv2.cvtColor(high_frequency,cv2.COLOR_BGR2GRAY) high_frequency = cv2.cvtColor(high_frequency,cv2.COLOR_GRAY2BGR) return high_frequency def appearance_prompt(img): h,w = img.shape[:2] img = cv2.resize(img,(1024,1024)) rgb_planes = cv2.split(img) result_planes = [] result_norm_planes = [] for plane in rgb_planes: dilated_img = cv2.dilate(plane, np.ones((7,7), np.uint8)) bg_img = cv2.medianBlur(dilated_img, 21) diff_img = 255 - cv2.absdiff(plane, bg_img) norm_img = cv2.normalize(diff_img,None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1) result_planes.append(diff_img) result_norm_planes.append(norm_img) result_norm = cv2.merge(result_norm_planes) result_norm = cv2.resize(result_norm,(w,h)) return result_norm def binarization_promptv2(img): result,thresh = utils.SauvolaModBinarization(img) thresh = thresh.astype(np.uint8) result[result>155]=255 result[result<=155]=0 x = cv2.Sobel(img,cv2.CV_16S,1,0) y = cv2.Sobel(img,cv2.CV_16S,0,1) absX = cv2.convertScaleAbs(x) absY = cv2.convertScaleAbs(y) high_frequency = cv2.addWeighted(absX,0.5,absY,0.5,0) high_frequency = cv2.cvtColor(high_frequency,cv2.COLOR_BGR2GRAY) return np.concatenate((np.expand_dims(thresh,-1),np.expand_dims(high_frequency,-1),np.expand_dims(result,-1)),-1) def dewarping(model,im_path): INPUT_SIZE=256 im_org = cv2.imread(im_path) im_masked, prompt_org = dewarp_prompt(im_org.copy()) h,w = im_masked.shape[:2] im_masked = im_masked.copy() im_masked = cv2.resize(im_masked,(INPUT_SIZE,INPUT_SIZE)) im_masked = im_masked / 255.0 im_masked = torch.from_numpy(im_masked.transpose(2,0,1)).unsqueeze(0) im_masked = im_masked.float().to(DEVICE) prompt = torch.from_numpy(prompt_org.transpose(2,0,1)).unsqueeze(0) prompt = prompt.float().to(DEVICE) in_im = torch.cat((im_masked,prompt),dim=1) base_coord = utils.getBasecoord(INPUT_SIZE,INPUT_SIZE)/INPUT_SIZE model = model.float() with torch.no_grad(): pred = model(in_im) pred = pred[0][:2].permute(1,2,0).cpu().numpy() pred = pred+base_coord for i in range(15): pred = cv2.blur(pred,(3,3),borderType=cv2.BORDER_REPLICATE) pred = cv2.resize(pred,(w,h))*(w,h) pred = pred.astype(np.float32) out_im = cv2.remap(im_org,pred[:,:,0],pred[:,:,1],cv2.INTER_LINEAR) prompt_org = (prompt_org*255).astype(np.uint8) prompt_org = cv2.resize(prompt_org,im_org.shape[:2][::-1]) return prompt_org[:,:,0],prompt_org[:,:,1],prompt_org[:,:,2],out_im def appearance(model, im_path): im_org = cv2.imread(im_path) im_org, _ = _resize_if_needed(im_org) h, w = im_org.shape[:2] prompt = appearance_prompt(im_org) in_im = np.concatenate((im_org, prompt), -1) in_im, padding_h, padding_w = stride_integral(in_im, 8) in_im = in_im / 255.0 in_im = torch.from_numpy(in_im.transpose(2, 0, 1)).unsqueeze(0) in_im = in_im.half().to(DEVICE) model = model.half() with torch.no_grad(): pred = model(in_im) pred = torch.clamp(pred, 0, 1) pred = pred[0].permute(1, 2, 0).cpu().numpy() pred = (pred * 255).astype(np.uint8) out_im = pred[padding_h:, padding_w:] return prompt[:, :, 0], prompt[:, :, 1], prompt[:, :, 2], out_im def deshadowing(model, im_path): im_org = cv2.imread(im_path) im_org, _ = _resize_if_needed(im_org) h, w = im_org.shape[:2] prompt = deshadow_prompt(im_org) in_im = np.concatenate((im_org, prompt), -1) in_im, padding_h, padding_w = stride_integral(in_im, 8) in_im = in_im / 255.0 in_im = torch.from_numpy(in_im.transpose(2, 0, 1)).unsqueeze(0) in_im = in_im.half().to(DEVICE) model = model.half() with torch.no_grad(): pred = model(in_im) pred = torch.clamp(pred, 0, 1) pred = pred[0].permute(1, 2, 0).cpu().numpy() pred = (pred * 255).astype(np.uint8) out_im = pred[padding_h:, padding_w:] return prompt[:, :, 0], prompt[:, :, 1], prompt[:, :, 2], out_im def deblurring(model, im_path): im_org = cv2.imread(im_path) im_org, _ = _resize_if_needed(im_org) in_im, padding_h, padding_w = stride_integral(im_org, 8) prompt = deblur_prompt(in_im) in_im = np.concatenate((in_im, prompt), -1) in_im = in_im / 255.0 in_im = torch.from_numpy(in_im.transpose(2, 0, 1)).unsqueeze(0) in_im = in_im.half().to(DEVICE) model = model.half() with torch.no_grad(): pred = model(in_im) pred = torch.clamp(pred, 0, 1) pred = pred[0].permute(1, 2, 0).cpu().numpy() pred = (pred * 255).astype(np.uint8) out_im = pred[padding_h:, padding_w:] return prompt[:, :, 0], prompt[:, :, 1], prompt[:, :, 2], out_im def binarization(model, im_path): im_org = cv2.imread(im_path) im_org, _ = _resize_if_needed(im_org) im, padding_h, padding_w = stride_integral(im_org, 8) prompt = binarization_promptv2(im) h, w = im.shape[:2] in_im = np.concatenate((im, prompt), -1) in_im = in_im / 255.0 in_im = torch.from_numpy(in_im.transpose(2, 0, 1)).unsqueeze(0) in_im = in_im.half().to(DEVICE) model = model.half() with torch.no_grad(): pred = model(in_im) pred = pred[:, :2, :, :] pred = torch.max(torch.softmax(pred, 1), 1)[1] pred = pred[0].cpu().numpy() pred = (pred * 255).astype(np.uint8) pred = cv2.resize(pred, (w, h)) out_im = pred[padding_h:, padding_w:] return prompt[:, :, 0], prompt[:, :, 1], prompt[:, :, 2], out_im def get_args(): parser = argparse.ArgumentParser(description='Params') parser.add_argument('--model_path', nargs='?', type=str, default='./checkpoints/docres.pkl',help='Path of the saved checkpoint') parser.add_argument('--im_path', nargs='?', type=str, default='./distorted/', help='Path of input document image') parser.add_argument('--out_folder', nargs='?', type=str, default='./restorted/', help='Folder of the output images') parser.add_argument('--task', nargs='?', type=str, default='dewarping', help='task that need to be executed') parser.add_argument('--save_dtsprompt', nargs='?', type=int, default=0, help='Width of the input image') args = parser.parse_args() possible_tasks = ['dewarping','deshadowing','appearance','deblurring','binarization','end2end'] assert args.task in possible_tasks, 'Unsupported task, task must be one of '+', '.join(possible_tasks) return args def model_init(args): model = restormer_arch.Restormer( inp_channels=6, out_channels=3, dim = 48, num_blocks = [2,3,3,4], num_refinement_blocks = 4, heads = [1,2,4,8], ffn_expansion_factor = 2.66, bias = False, LayerNorm_type = 'WithBias', dual_pixel_task = True ) if DEVICE.type == 'cpu': state = convert_state_dict(torch.load(args.model_path, map_location='cpu')['model_state']) else: state = convert_state_dict(torch.load(args.model_path, map_location='cuda:0')['model_state']) model.load_state_dict(state) model.eval() model = model.to(DEVICE) return model def inference_one_im(model,im_path,task): if task=='dewarping': prompt1,prompt2,prompt3,restorted = dewarping(model,im_path) elif task=='deshadowing': prompt1,prompt2,prompt3,restorted = deshadowing(model,im_path) elif task=='appearance': prompt1,prompt2,prompt3,restorted = appearance(model,im_path) elif task=='deblurring': prompt1,prompt2,prompt3,restorted = deblurring(model,im_path) elif task=='binarization': prompt1,prompt2,prompt3,restorted = binarization(model,im_path) elif task=='end2end': prompt1,prompt2,prompt3,restorted = dewarping(model,im_path) cv2.imwrite('restorted/step1.jpg',restorted) prompt1,prompt2,prompt3,restorted = deshadowing(model,'restorted/step1.jpg') cv2.imwrite('restorted/step2.jpg',restorted) prompt1,prompt2,prompt3,restorted = appearance(model,'restorted/step2.jpg') return prompt1,prompt2,prompt3,restorted def save_results( img_path: str, out_folder: str, task: str, save_dtsprompt: bool, ): im_name = os.path.split(img_path)[-1] im_format = '.'+im_name.split('.')[-1] save_path = os.path.join(out_folder, im_name.replace(im_format, '_' + task + im_format)) cv2.imwrite(save_path, restorted) if save_dtsprompt: cv2.imwrite(save_path.replace(im_format, '_prompt1' + im_format), prompt1) cv2.imwrite(save_path.replace(im_format, '_prompt2' + im_format), prompt2) cv2.imwrite(save_path.replace(im_format, '_prompt3' + im_format), prompt3) if __name__ == '__main__': DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') args = get_args() model = model_init(args) img_source = args.im_path if Path(img_source).is_dir(): img_paths = glob.glob(os.path.join(img_source, '*')) for img_path in img_paths: prompt1,prompt2,prompt3,restorted = inference_one_im(model,img_path,args.task) save_results( img_path=img_path, out_folder=args.out_folder, task=args.task, save_dtsprompt=args.save_dtsprompt, ) else: prompt1,prompt2,prompt3,restorted = inference_one_im(model,img_source,args.task) save_results( img_path=img_source, out_folder=args.out_folder, task=args.task, save_dtsprompt=args.save_dtsprompt, )