From dcf64d43e0b3a205ebce565d1cf7e71955916824 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 18 Mar 2021 20:54:53 +0530 Subject: [PATCH] [movefiles] Fix bugs and make more robust --- yt_dlp/YoutubeDL.py | 37 +++++++++++-------- yt_dlp/postprocessor/embedthumbnail.py | 7 +++- yt_dlp/postprocessor/ffmpeg.py | 17 ++++++--- .../postprocessor/movefilesafterdownload.py | 9 +---- 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 0979252c9..5cfd830e2 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -2075,6 +2075,7 @@ class YoutubeDL(object): info_dict = self.pre_process(info_dict) + # info_dict['_filename'] needs to be set for backward compatibility info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True) temp_filename = self.prepare_filename(info_dict, 'temp') files_to_move = {} @@ -2159,6 +2160,7 @@ class YoutubeDL(object): sub_filename_final = subtitles_filename(sub_fn, sub_lang, sub_format, info_dict.get('ext')) if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)): self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format)) + sub_info['filepath'] = sub_filename files_to_move[sub_filename] = sub_filename_final else: self.to_screen('[info] Writing video subtitles to: ' + sub_filename) @@ -2168,13 +2170,15 @@ class YoutubeDL(object): # See https://github.com/ytdl-org/youtube-dl/issues/10268 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile: subfile.write(sub_info['data']) + sub_info['filepath'] = sub_filename files_to_move[sub_filename] = sub_filename_final except (OSError, IOError): self.report_error('Cannot write subtitles file ' + sub_filename) return else: try: - dl(sub_filename, sub_info, subtitle=True) + dl(sub_filename, sub_info.copy(), subtitle=True) + sub_info['filepath'] = sub_filename files_to_move[sub_filename] = sub_filename_final except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning('Unable to download subtitle for "%s": %s' % @@ -2223,7 +2227,7 @@ class YoutubeDL(object): for thumb_ext in self._write_thumbnails(info_dict, thumb_fn_temp): thumb_filename_temp = replace_extension(thumb_fn_temp, thumb_ext, info_dict.get('ext')) thumb_filename = replace_extension(thumbfn, thumb_ext, info_dict.get('ext')) - files_to_move[thumb_filename_temp] = info_dict['__thumbnail_filename'] = thumb_filename + files_to_move[thumb_filename_temp] = thumb_filename # Write internet shortcut files url_link = webloc_link = desktop_link = False @@ -2529,15 +2533,17 @@ class YoutubeDL(object): (k, v) for k, v in info_dict.items() if (k[0] != '_' or k == '_type') and k not in fields_to_remove) - def run_pp(self, pp, infodict, files_to_move={}): + def run_pp(self, pp, infodict): files_to_delete = [] + if '__files_to_move' not in infodict: + infodict['__files_to_move'] = {} files_to_delete, infodict = pp.run(infodict) if not files_to_delete: - return files_to_move, infodict + return infodict if self.params.get('keepvideo', False): for f in files_to_delete: - files_to_move.setdefault(f, '') + infodict['__files_to_move'].setdefault(f, '') else: for old_filename in set(files_to_delete): self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename) @@ -2545,9 +2551,9 @@ class YoutubeDL(object): os.remove(encodeFilename(old_filename)) except (IOError, OSError): self.report_warning('Unable to remove downloaded original file') - if old_filename in files_to_move: - del files_to_move[old_filename] - return files_to_move, infodict + if old_filename in infodict['__files_to_move']: + del infodict['__files_to_move'][old_filename] + return infodict @staticmethod def post_extract(info_dict): @@ -2570,20 +2576,21 @@ class YoutubeDL(object): def pre_process(self, ie_info): info = dict(ie_info) for pp in self._pps['beforedl']: - info = self.run_pp(pp, info)[1] + info = self.run_pp(pp, info) return info - def post_process(self, filename, ie_info, files_to_move={}): + def post_process(self, filename, ie_info, files_to_move=None): """Run all the postprocessors on the given file.""" info = dict(ie_info) info['filepath'] = filename - info['__files_to_move'] = {} + info['__files_to_move'] = files_to_move or {} for pp in ie_info.get('__postprocessors', []) + self._pps['normal']: - files_to_move, info = self.run_pp(pp, info, files_to_move) - info = self.run_pp(MoveFilesAfterDownloadPP(self, files_to_move), info)[1] + info = self.run_pp(pp, info) + info = self.run_pp(MoveFilesAfterDownloadPP(self), info) + del info['__files_to_move'] for pp in self._pps['aftermove']: - info = self.run_pp(pp, info, {})[1] + info = self.run_pp(pp, info) def _make_archive_id(self, info_dict): video_id = info_dict.get('id') @@ -2951,7 +2958,7 @@ class YoutubeDL(object): thumb_ext = determine_ext(t['url'], 'jpg') suffix = '%s.' % t['id'] if multiple else '' thumb_display_id = '%s ' % t['id'] if multiple else '' - t['filename'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext')) + t['filepath'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext')) if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)): ret.append(suffix + thumb_ext) diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py index 82e7e2004..7f759cc30 100644 --- a/yt_dlp/postprocessor/embedthumbnail.py +++ b/yt_dlp/postprocessor/embedthumbnail.py @@ -47,7 +47,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor): self.to_screen('There aren\'t any thumbnails to embed') return [], info - original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filename'] + initial_thumbnail = original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filepath'] if not os.path.exists(encodeFilename(thumbnail_filename)): self.report_warning('Skipping embedding the thumbnail because the file is missing.') @@ -65,6 +65,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor): if thumbnail_ext != 'webp' and is_webp(thumbnail_filename): self.to_screen('Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename) thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp') + if os.path.exists(thumbnail_webp_filename): + os.remove(thumbnail_webp_filename) os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename)) original_thumbnail = thumbnail_filename = thumbnail_webp_filename thumbnail_ext = 'webp' @@ -194,7 +196,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor): files_to_delete = [thumbnail_filename] if self._already_have_thumbnail: info['__files_to_move'][original_thumbnail] = replace_extension( - info['__thumbnail_filename'], os.path.splitext(original_thumbnail)[1][1:]) + info['__files_to_move'][initial_thumbnail], + os.path.splitext(original_thumbnail)[1][1:]) if original_thumbnail == thumbnail_filename: files_to_delete = [] elif original_thumbnail != thumbnail_filename: diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 7d0452dbc..dd07e7c18 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -486,7 +486,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): self.report_warning('JSON subtitles cannot be embedded') elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': sub_langs.append(lang) - sub_filenames.append(subtitles_filename(filename, lang, sub_ext, ext)) + sub_filenames.append(sub_info['filepath']) else: if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt': webm_vtt_warn = True @@ -732,9 +732,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): 'You have requested to convert json subtitles into another format, ' 'which is currently not possible') continue - old_file = subtitles_filename(filename, lang, ext, info.get('ext')) + old_file = sub['filepath'] sub_filenames.append(old_file) - new_file = subtitles_filename(filename, lang, new_ext, info.get('ext')) + new_file = replace_extension(old_file, new_ext) if ext in ('dfxp', 'ttml', 'tt'): self.report_warning( @@ -742,7 +742,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): 'which results in style information loss') dfxp_file = old_file - srt_file = subtitles_filename(filename, lang, 'srt', info.get('ext')) + srt_file = replace_extension(old_file, 'srt') with open(dfxp_file, 'rb') as f: srt_data = dfxp2srt(f.read()) @@ -753,7 +753,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): subs[lang] = { 'ext': 'srt', - 'data': srt_data + 'data': srt_data, + 'filepath': srt_file, } if new_ext == 'srt': @@ -767,8 +768,12 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): subs[lang] = { 'ext': new_ext, 'data': f.read(), + 'filepath': new_file, } + info['__files_to_move'][new_file] = replace_extension( + info['__files_to_move'][old_file], new_ext) + return sub_filenames, info @@ -789,7 +794,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): if not self._downloader._ensure_dir_exists(encodeFilename(destination)): return - chapter['_filename'] = destination + chapter['filepath'] = destination self.to_screen('Chapter %03d; Destination: %s' % (number, destination)) return ( destination, diff --git a/yt_dlp/postprocessor/movefilesafterdownload.py b/yt_dlp/postprocessor/movefilesafterdownload.py index fa61317ed..0ab7744ca 100644 --- a/yt_dlp/postprocessor/movefilesafterdownload.py +++ b/yt_dlp/postprocessor/movefilesafterdownload.py @@ -13,10 +13,6 @@ from ..utils import ( class MoveFilesAfterDownloadPP(PostProcessor): - def __init__(self, downloader, files_to_move): - PostProcessor.__init__(self, downloader) - self.files_to_move = files_to_move - @classmethod def pp_key(cls): return 'MoveFiles' @@ -25,11 +21,10 @@ class MoveFilesAfterDownloadPP(PostProcessor): dl_path, dl_name = os.path.split(encodeFilename(info['filepath'])) finaldir = info.get('__finaldir', dl_path) finalpath = os.path.join(finaldir, dl_name) - self.files_to_move.update(info['__files_to_move']) - self.files_to_move[info['filepath']] = decodeFilename(finalpath) + info['__files_to_move'][info['filepath']] = decodeFilename(finalpath) make_newfilename = lambda old: decodeFilename(os.path.join(finaldir, os.path.basename(encodeFilename(old)))) - for oldfile, newfile in self.files_to_move.items(): + for oldfile, newfile in info['__files_to_move'].items(): if not newfile: newfile = make_newfilename(oldfile) if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)):