From 0202b52a0c0a15da6073a122aae7ed6693e18f01 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 23 Jan 2021 17:48:12 +0530 Subject: [PATCH] #29 New option `-P`/`--paths` to give different paths for different types of files Syntax: `-P "type:path" -P "type:path"` Types: home, temp, description, annotation, subtitle, infojson, thumbnail --- README.md | 22 +- youtube_dlc/YoutubeDL.py | 235 ++++++++++++------ youtube_dlc/__init__.py | 6 +- youtube_dlc/options.py | 108 +++++--- youtube_dlc/postprocessor/__init__.py | 2 + .../postprocessor/movefilesafterdownload.py | 52 ++++ youtube_dlc/utils.py | 12 + 7 files changed, 321 insertions(+), 116 deletions(-) create mode 100644 youtube_dlc/postprocessor/movefilesafterdownload.py diff --git a/README.md b/README.md index 71fc41684..a2ddc3db5 100644 --- a/README.md +++ b/README.md @@ -150,9 +150,9 @@ Then simply type this compatibility) if this option is found inside the system configuration file, the user configuration is not loaded - --config-location PATH Location of the configuration file; either - the path to the config or its containing - directory + --config-location PATH Location of the main configuration file; + either the path to the config or its + containing directory --flat-playlist Do not extract the videos of a playlist, only list them --flat-videos Do not resolve the video urls @@ -316,6 +316,17 @@ Then simply type this stdin), one URL per line. Lines starting with '#', ';' or ']' are considered as comments and ignored + -P, --paths TYPE:PATH The paths where the files should be + downloaded. Specify the type of file and + the path separated by a colon ":" + (supported: description|annotation|subtitle + |infojson|thumbnail). Additionally, you can + also provide "home" and "temp" paths. All + intermediary files are first downloaded to + the temp path and then the final files are + moved over to the home path after download + is finished. Note that this option is + ignored if --output is an absolute path -o, --output TEMPLATE Output filename template, see "OUTPUT TEMPLATE" for details --autonumber-start NUMBER Specify the start value for %(autonumber)s @@ -651,8 +662,9 @@ Then simply type this You can configure youtube-dlc by placing any supported command line option to a configuration file. The configuration is loaded from the following locations: -1. The file given by `--config-location` +1. **Main Configuration**: The file given by `--config-location` 1. **Portable Configuration**: `yt-dlp.conf` or `youtube-dlc.conf` in the same directory as the bundled binary. If you are running from source-code (`/youtube_dlc/__main__.py`), the root directory is used instead. +1. **Home Configuration**: `yt-dlp.conf` or `youtube-dlc.conf` in the home path given by `-P "home:"`, or in the current directory if no such path is given 1. **User Configuration**: * `%XDG_CONFIG_HOME%/yt-dlp/config` (recommended on Linux/macOS) * `%XDG_CONFIG_HOME%/yt-dlp.conf` @@ -710,7 +722,7 @@ set HOME=%USERPROFILE% # OUTPUT TEMPLATE -The `-o` option allows users to indicate a template for the output file names. +The `-o` option is used to indicate a template for the output file names while `-P` option is used to specify the path each type of file should be saved to. **tl;dr:** [navigate me to examples](#output-template-examples). diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index 208cae17e..58f50a556 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -69,6 +69,7 @@ from .utils import ( iri_to_uri, ISO3166Utils, locked_file, + make_dir, make_HTTPS_handler, MaxDownloadsReached, orderedSet, @@ -114,8 +115,9 @@ from .postprocessor import ( FFmpegFixupStretchedPP, FFmpegMergerPP, FFmpegPostProcessor, - FFmpegSubtitlesConvertorPP, + # FFmpegSubtitlesConvertorPP, get_postprocessor, + MoveFilesAfterDownloadPP, ) from .version import __version__ @@ -257,6 +259,8 @@ class YoutubeDL(object): postprocessors: A list of dictionaries, each with an entry * key: The name of the postprocessor. See youtube_dlc/postprocessor/__init__.py for a list. + * _after_move: Optional. If True, run this post_processor + after 'MoveFilesAfterDownload' as well as any further keyword arguments for the postprocessor. post_hooks: A list of functions that get called as the final step @@ -369,6 +373,8 @@ class YoutubeDL(object): params = None _ies = [] _pps = [] + _pps_end = [] + __prepare_filename_warned = False _download_retcode = None _num_downloads = None _playlist_level = 0 @@ -382,6 +388,8 @@ class YoutubeDL(object): self._ies = [] self._ies_instances = {} self._pps = [] + self._pps_end = [] + self.__prepare_filename_warned = False self._post_hooks = [] self._progress_hooks = [] self._download_retcode = 0 @@ -483,8 +491,11 @@ class YoutubeDL(object): pp_class = get_postprocessor(pp_def_raw['key']) pp_def = dict(pp_def_raw) del pp_def['key'] + after_move = pp_def.get('_after_move', False) + if '_after_move' in pp_def: + del pp_def['_after_move'] pp = pp_class(self, **compat_kwargs(pp_def)) - self.add_post_processor(pp) + self.add_post_processor(pp, after_move=after_move) for ph in self.params.get('post_hooks', []): self.add_post_hook(ph) @@ -536,9 +547,12 @@ class YoutubeDL(object): for ie in gen_extractor_classes(): self.add_info_extractor(ie) - def add_post_processor(self, pp): + def add_post_processor(self, pp, after_move=False): """Add a PostProcessor object to the end of the chain.""" - self._pps.append(pp) + if after_move: + self._pps_end.append(pp) + else: + self._pps.append(pp) pp.set_downloader(self) def add_post_hook(self, ph): @@ -702,7 +716,7 @@ class YoutubeDL(object): except UnicodeEncodeError: self.to_screen('Deleting already existent file') - def prepare_filename(self, info_dict): + def prepare_filename(self, info_dict, warn=False): """Generate the output filename.""" try: template_dict = dict(info_dict) @@ -796,11 +810,33 @@ class YoutubeDL(object): # to workaround encoding issues with subprocess on python2 @ Windows if sys.version_info < (3, 0) and sys.platform == 'win32': filename = encodeFilename(filename, True).decode(preferredencoding()) - return sanitize_path(filename) + filename = sanitize_path(filename) + + if warn and not self.__prepare_filename_warned: + if not self.params.get('paths'): + pass + elif filename == '-': + self.report_warning('--paths is ignored when an outputting to stdout') + elif os.path.isabs(filename): + self.report_warning('--paths is ignored since an absolute path is given in output template') + self.__prepare_filename_warned = True + + return filename except ValueError as err: self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')') return None + def prepare_filepath(self, filename, dir_type=''): + if filename == '-': + return filename + paths = self.params.get('paths', {}) + assert isinstance(paths, dict) + homepath = expand_path(paths.get('home', '').strip()) + assert isinstance(homepath, compat_str) + subdir = expand_path(paths.get(dir_type, '').strip()) if dir_type else '' + assert isinstance(subdir, compat_str) + return sanitize_path(os.path.join(homepath, subdir, filename)) + def _match_entry(self, info_dict, incomplete): """ Returns None if the file should be downloaded """ @@ -972,7 +1008,8 @@ class YoutubeDL(object): if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or extract_flat is True): self.__forced_printings( - ie_result, self.prepare_filename(ie_result), + ie_result, + self.prepare_filepath(self.prepare_filename(ie_result)), incomplete=True) return ie_result @@ -1890,6 +1927,8 @@ class YoutubeDL(object): assert info_dict.get('_type', 'video') == 'video' + info_dict.setdefault('__postprocessors', []) + max_downloads = self.params.get('max_downloads') if max_downloads is not None: if self._num_downloads >= int(max_downloads): @@ -1906,10 +1945,13 @@ class YoutubeDL(object): self._num_downloads += 1 - info_dict['_filename'] = filename = self.prepare_filename(info_dict) + filename = self.prepare_filename(info_dict, warn=True) + info_dict['_filename'] = full_filename = self.prepare_filepath(filename) + temp_filename = self.prepare_filepath(filename, 'temp') + files_to_move = {} # Forced printings - self.__forced_printings(info_dict, filename, incomplete=False) + self.__forced_printings(info_dict, full_filename, incomplete=False) if self.params.get('simulate', False): if self.params.get('force_write_download_archive', False): @@ -1922,20 +1964,19 @@ class YoutubeDL(object): return def ensure_dir_exists(path): - try: - dn = os.path.dirname(path) - if dn and not os.path.exists(dn): - os.makedirs(dn) - return True - except (OSError, IOError) as err: - self.report_error('unable to create directory ' + error_to_compat_str(err)) - return False + return make_dir(path, self.report_error) - if not ensure_dir_exists(sanitize_path(encodeFilename(filename))): + if not ensure_dir_exists(encodeFilename(full_filename)): + return + if not ensure_dir_exists(encodeFilename(temp_filename)): return if self.params.get('writedescription', False): - descfn = replace_extension(filename, 'description', info_dict.get('ext')) + descfn = replace_extension( + self.prepare_filepath(filename, 'description'), + 'description', info_dict.get('ext')) + if not ensure_dir_exists(encodeFilename(descfn)): + return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)): self.to_screen('[info] Video description is already present') elif info_dict.get('description') is None: @@ -1950,7 +1991,11 @@ class YoutubeDL(object): return if self.params.get('writeannotations', False): - annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext')) + annofn = replace_extension( + self.prepare_filepath(filename, 'annotation'), + 'annotations.xml', info_dict.get('ext')) + if not ensure_dir_exists(encodeFilename(annofn)): + return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)): self.to_screen('[info] Video annotations are already present') elif not info_dict.get('annotations'): @@ -1984,9 +2029,13 @@ class YoutubeDL(object): # ie = self.get_info_extractor(info_dict['extractor_key']) for sub_lang, sub_info in subtitles.items(): sub_format = sub_info['ext'] - sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext')) + sub_filename = subtitles_filename(temp_filename, sub_lang, sub_format, info_dict.get('ext')) + sub_filename_final = subtitles_filename( + self.prepare_filepath(filename, 'subtitle'), + sub_lang, sub_format, info_dict.get('ext')) if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)): self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format)) + files_to_move[sub_filename] = sub_filename_final else: self.to_screen('[info] Writing video subtitles to: ' + sub_filename) if sub_info.get('data') is not None: @@ -1995,6 +2044,7 @@ class YoutubeDL(object): # See https://github.com/ytdl-org/youtube-dl/issues/10268 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile: subfile.write(sub_info['data']) + files_to_move[sub_filename] = sub_filename_final except (OSError, IOError): self.report_error('Cannot write subtitles file ' + sub_filename) return @@ -2010,6 +2060,7 @@ class YoutubeDL(object): with io.open(encodeFilename(sub_filename), 'wb') as subfile: subfile.write(sub_data) ''' + files_to_move[sub_filename] = sub_filename_final except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning('Unable to download subtitle for "%s": %s' % (sub_lang, error_to_compat_str(err))) @@ -2017,29 +2068,32 @@ class YoutubeDL(object): if self.params.get('skip_download', False): if self.params.get('convertsubtitles', False): - subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles')) + # subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles')) filename_real_ext = os.path.splitext(filename)[1][1:] filename_wo_ext = ( - os.path.splitext(filename)[0] + os.path.splitext(full_filename)[0] if filename_real_ext == info_dict['ext'] - else filename) + else full_filename) afilename = '%s.%s' % (filename_wo_ext, self.params.get('convertsubtitles')) - if subconv.available: - info_dict.setdefault('__postprocessors', []) - # info_dict['__postprocessors'].append(subconv) + # if subconv.available: + # info_dict['__postprocessors'].append(subconv) if os.path.exists(encodeFilename(afilename)): self.to_screen( '[download] %s has already been downloaded and ' 'converted' % afilename) else: try: - self.post_process(filename, info_dict) + self.post_process(full_filename, info_dict, files_to_move) except (PostProcessingError) as err: self.report_error('postprocessing: %s' % str(err)) return if self.params.get('writeinfojson', False): - infofn = replace_extension(filename, 'info.json', info_dict.get('ext')) + infofn = replace_extension( + self.prepare_filepath(filename, 'infojson'), + 'info.json', info_dict.get('ext')) + if not ensure_dir_exists(encodeFilename(infofn)): + return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)): self.to_screen('[info] Video description metadata is already present') else: @@ -2050,7 +2104,9 @@ class YoutubeDL(object): self.report_error('Cannot write metadata to JSON file ' + infofn) return - self._write_thumbnails(info_dict, filename) + thumbdir = os.path.dirname(self.prepare_filepath(filename, 'thumbnail')) + for thumbfn in self._write_thumbnails(info_dict, temp_filename): + files_to_move[thumbfn] = os.path.join(thumbdir, os.path.basename(thumbfn)) # Write internet shortcut files url_link = webloc_link = desktop_link = False @@ -2075,7 +2131,7 @@ class YoutubeDL(object): ascii_url = iri_to_uri(info_dict['webpage_url']) def _write_link_file(extension, template, newline, embed_filename): - linkfn = replace_extension(filename, extension, info_dict.get('ext')) + linkfn = replace_extension(full_filename, extension, info_dict.get('ext')) if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(linkfn)): self.to_screen('[info] Internet shortcut is already present') else: @@ -2105,9 +2161,27 @@ class YoutubeDL(object): must_record_download_archive = False if not self.params.get('skip_download', False): try: + + def existing_file(filename, temp_filename): + file_exists = os.path.exists(encodeFilename(filename)) + tempfile_exists = ( + False if temp_filename == filename + else os.path.exists(encodeFilename(temp_filename))) + if not self.params.get('overwrites', False) and (file_exists or tempfile_exists): + existing_filename = temp_filename if tempfile_exists else filename + self.to_screen('[download] %s has already been downloaded and merged' % existing_filename) + return existing_filename + if tempfile_exists: + self.report_file_delete(temp_filename) + os.remove(encodeFilename(temp_filename)) + if file_exists: + self.report_file_delete(filename) + os.remove(encodeFilename(filename)) + return None + + success = True if info_dict.get('requested_formats') is not None: downloaded = [] - success = True merger = FFmpegMergerPP(self) if not merger.available: postprocessors = [] @@ -2136,32 +2210,31 @@ class YoutubeDL(object): # TODO: Check acodec/vcodec return False - filename_real_ext = os.path.splitext(filename)[1][1:] - filename_wo_ext = ( - os.path.splitext(filename)[0] - if filename_real_ext == info_dict['ext'] - else filename) requested_formats = info_dict['requested_formats'] + old_ext = info_dict['ext'] if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats): info_dict['ext'] = 'mkv' self.report_warning( 'Requested formats are incompatible for merge and will be merged into mkv.') + + def correct_ext(filename): + filename_real_ext = os.path.splitext(filename)[1][1:] + filename_wo_ext = ( + os.path.splitext(filename)[0] + if filename_real_ext == old_ext + else filename) + return '%s.%s' % (filename_wo_ext, info_dict['ext']) + # Ensure filename always has a correct extension for successful merge - filename = '%s.%s' % (filename_wo_ext, info_dict['ext']) - file_exists = os.path.exists(encodeFilename(filename)) - if not self.params.get('overwrites', False) and file_exists: - self.to_screen( - '[download] %s has already been downloaded and ' - 'merged' % filename) - else: - if file_exists: - self.report_file_delete(filename) - os.remove(encodeFilename(filename)) + full_filename = correct_ext(full_filename) + temp_filename = correct_ext(temp_filename) + dl_filename = existing_file(full_filename, temp_filename) + if dl_filename is None: for f in requested_formats: new_info = dict(info_dict) new_info.update(f) fname = prepend_extension( - self.prepare_filename(new_info), + self.prepare_filepath(self.prepare_filename(new_info), 'temp'), 'f%s' % f['format_id'], new_info['ext']) if not ensure_dir_exists(fname): return @@ -2173,14 +2246,17 @@ class YoutubeDL(object): # Even if there were no downloads, it is being merged only now info_dict['__real_download'] = True else: - # Delete existing file with --yes-overwrites - if self.params.get('overwrites', False): - if os.path.exists(encodeFilename(filename)): - self.report_file_delete(filename) - os.remove(encodeFilename(filename)) # Just a single file - success, real_download = dl(filename, info_dict) - info_dict['__real_download'] = real_download + dl_filename = existing_file(full_filename, temp_filename) + if dl_filename is None: + success, real_download = dl(temp_filename, info_dict) + info_dict['__real_download'] = real_download + + # info_dict['__temp_filename'] = temp_filename + dl_filename = dl_filename or temp_filename + info_dict['__dl_filename'] = dl_filename + info_dict['__final_filename'] = full_filename + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_error('unable to download video data: %s' % error_to_compat_str(err)) return @@ -2206,7 +2282,6 @@ class YoutubeDL(object): elif fixup_policy == 'detect_or_warn': stretched_pp = FFmpegFixupStretchedPP(self) if stretched_pp.available: - info_dict.setdefault('__postprocessors', []) info_dict['__postprocessors'].append(stretched_pp) else: self.report_warning( @@ -2225,7 +2300,6 @@ class YoutubeDL(object): elif fixup_policy == 'detect_or_warn': fixup_pp = FFmpegFixupM4aPP(self) if fixup_pp.available: - info_dict.setdefault('__postprocessors', []) info_dict['__postprocessors'].append(fixup_pp) else: self.report_warning( @@ -2244,7 +2318,6 @@ class YoutubeDL(object): elif fixup_policy == 'detect_or_warn': fixup_pp = FFmpegFixupM3u8PP(self) if fixup_pp.available: - info_dict.setdefault('__postprocessors', []) info_dict['__postprocessors'].append(fixup_pp) else: self.report_warning( @@ -2254,13 +2327,13 @@ class YoutubeDL(object): assert fixup_policy in ('ignore', 'never') try: - self.post_process(filename, info_dict) + self.post_process(dl_filename, info_dict, files_to_move) except (PostProcessingError) as err: self.report_error('postprocessing: %s' % str(err)) return try: for ph in self._post_hooks: - ph(filename) + ph(full_filename) except Exception as err: self.report_error('post hooks: %s' % str(err)) return @@ -2326,27 +2399,41 @@ class YoutubeDL(object): (k, v) for k, v in info_dict.items() if k not in ['requested_formats', 'requested_subtitles']) - def post_process(self, filename, ie_info): + def post_process(self, filename, ie_info, files_to_move={}): """Run all the postprocessors on the given file.""" info = dict(ie_info) info['filepath'] = filename - pps_chain = [] - if ie_info.get('__postprocessors') is not None: - pps_chain.extend(ie_info['__postprocessors']) - pps_chain.extend(self._pps) - for pp in pps_chain: + + def run_pp(pp): files_to_delete = [] + infodict = info try: - files_to_delete, info = pp.run(info) + files_to_delete, infodict = pp.run(infodict) except PostProcessingError as e: self.report_error(e.msg) - if files_to_delete and not self.params.get('keepvideo', False): + if not files_to_delete: + return infodict + + if self.params.get('keepvideo', False): + for f in files_to_delete: + files_to_move.setdefault(f, '') + else: for old_filename in set(files_to_delete): self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename) try: os.remove(encodeFilename(old_filename)) except (IOError, OSError): self.report_warning('Unable to remove downloaded original file') + if old_filename in files_to_move: + del files_to_move[old_filename] + return infodict + + for pp in ie_info.get('__postprocessors', []) + self._pps: + info = run_pp(pp) + info = run_pp(MoveFilesAfterDownloadPP(self, files_to_move)) + files_to_move = {} + for pp in self._pps_end: + info = run_pp(pp) def _make_archive_id(self, info_dict): video_id = info_dict.get('id') @@ -2700,14 +2787,11 @@ class YoutubeDL(object): if thumbnails: thumbnails = [thumbnails[-1]] elif self.params.get('write_all_thumbnails', False): - thumbnails = info_dict.get('thumbnails') + thumbnails = info_dict.get('thumbnails') or [] else: - return - - if not thumbnails: - # No thumbnails present, so return immediately - return + thumbnails = [] + ret = [] for t in thumbnails: thumb_ext = determine_ext(t['url'], 'jpg') suffix = '_%s' % t['id'] if len(thumbnails) > 1 else '' @@ -2715,6 +2799,7 @@ class YoutubeDL(object): t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext')) if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)): + ret.append(thumb_filename) self.to_screen('[%s] %s: Thumbnail %sis already present' % (info_dict['extractor'], info_dict['id'], thumb_display_id)) else: @@ -2724,8 +2809,10 @@ class YoutubeDL(object): uf = self.urlopen(t['url']) with open(encodeFilename(thumb_filename), 'wb') as thumbf: shutil.copyfileobj(uf, thumbf) + ret.append(thumb_filename) self.to_screen('[%s] %s: Writing thumbnail %sto: %s' % (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename)) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning('Unable to download thumbnail "%s": %s' % (t['url'], error_to_compat_str(err))) + return ret diff --git a/youtube_dlc/__init__.py b/youtube_dlc/__init__.py index 5bf54b556..ee6120395 100644 --- a/youtube_dlc/__init__.py +++ b/youtube_dlc/__init__.py @@ -244,6 +244,7 @@ def _real_main(argv=None): parser.error('Cannot download a video and extract audio into the same' ' file! Use "{0}.%(ext)s" instead of "{0}" as the output' ' template'.format(outtmpl)) + for f in opts.format_sort: if re.match(InfoExtractor.FormatSort.regex, f) is None: parser.error('invalid format sort string "%s" specified' % f) @@ -318,12 +319,12 @@ def _real_main(argv=None): 'force': opts.sponskrub_force, 'ignoreerror': opts.sponskrub is None, }) - # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way. - # So if the user is able to remove the file before your postprocessor runs it might cause a few problems. + # ExecAfterDownload must be the last PP if opts.exec_cmd: postprocessors.append({ 'key': 'ExecAfterDownload', 'exec_cmd': opts.exec_cmd, + '_after_move': True }) _args_compat_warning = 'WARNING: %s given without specifying name. The arguments will be given to all %s\n' @@ -372,6 +373,7 @@ def _real_main(argv=None): 'listformats': opts.listformats, 'listformats_table': opts.listformats_table, 'outtmpl': outtmpl, + 'paths': opts.paths, 'autonumber_size': opts.autonumber_size, 'autonumber_start': opts.autonumber_start, 'restrictfilenames': opts.restrictfilenames, diff --git a/youtube_dlc/options.py b/youtube_dlc/options.py index 7a30882f1..7a18f0f84 100644 --- a/youtube_dlc/options.py +++ b/youtube_dlc/options.py @@ -14,6 +14,7 @@ from .compat import ( compat_shlex_split, ) from .utils import ( + expand_path, preferredencoding, write_string, ) @@ -62,7 +63,7 @@ def parseOpts(overrideArguments=None): userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name) userConf = _readOptions(userConfFile, default=None) if userConf is not None: - return userConf + return userConf, userConfFile # appdata appdata_dir = compat_getenv('appdata') @@ -70,19 +71,21 @@ def parseOpts(overrideArguments=None): userConfFile = os.path.join(appdata_dir, package_name, 'config') userConf = _readOptions(userConfFile, default=None) if userConf is None: - userConf = _readOptions('%s.txt' % userConfFile, default=None) + userConfFile += '.txt' + userConf = _readOptions(userConfFile, default=None) if userConf is not None: - return userConf + return userConf, userConfFile # home userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name) userConf = _readOptions(userConfFile, default=None) if userConf is None: - userConf = _readOptions('%s.txt' % userConfFile, default=None) + userConfFile += '.txt' + userConf = _readOptions(userConfFile, default=None) if userConf is not None: - return userConf + return userConf, userConfFile - return default + return default, None def _format_option_string(option): ''' ('-o', '--option') -> -o, --format METAVAR''' @@ -187,7 +190,7 @@ def parseOpts(overrideArguments=None): general.add_option( '--config-location', dest='config_location', metavar='PATH', - help='Location of the configuration file; either the path to the config or its containing directory') + help='Location of the main configuration file; either the path to the config or its containing directory') general.add_option( '--flat-playlist', action='store_const', dest='extract_flat', const='in_playlist', default=False, @@ -641,7 +644,7 @@ def parseOpts(overrideArguments=None): metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str', action='callback', callback=_dict_from_multiple_values_options_callback, callback_kwargs={ - 'allowed_keys': '|'.join(list_external_downloaders()), + 'allowed_keys': '|'.join(list_external_downloaders()), 'default_key': 'default', 'process': compat_shlex_split}, help=( 'Give these arguments to the external downloader. ' @@ -819,6 +822,21 @@ def parseOpts(overrideArguments=None): filesystem.add_option( '--id', default=False, action='store_true', dest='useid', help=optparse.SUPPRESS_HELP) + filesystem.add_option( + '-P', '--paths', + metavar='TYPE:PATH', dest='paths', default={}, type='str', + action='callback', callback=_dict_from_multiple_values_options_callback, + callback_kwargs={ + 'allowed_keys': 'home|temp|config|description|annotation|subtitle|infojson|thumbnail', + 'process': lambda x: x.strip()}, + help=( + 'The paths where the files should be downloaded. ' + 'Specify the type of file and the path separated by a colon ":" ' + '(supported: description|annotation|subtitle|infojson|thumbnail). ' + 'Additionally, you can also provide "home" and "temp" paths. ' + 'All intermediary files are first downloaded to the temp path and ' + 'then the final files are moved over to the home path after download is finished. ' + 'Note that this option is ignored if --output is an absolute path')) filesystem.add_option( '-o', '--output', dest='outtmpl', metavar='TEMPLATE', @@ -1171,59 +1189,79 @@ def parseOpts(overrideArguments=None): return conf configs = { - 'command_line': compat_conf(sys.argv[1:]), - 'custom': [], 'portable': [], 'user': [], 'system': []} - opts, args = parser.parse_args(configs['command_line']) + 'command-line': compat_conf(sys.argv[1:]), + 'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []} + paths = {'command-line': False} + opts, args = parser.parse_args(configs['command-line']) def get_configs(): - if '--config-location' in configs['command_line']: + if '--config-location' in configs['command-line']: location = compat_expanduser(opts.config_location) if os.path.isdir(location): location = os.path.join(location, 'youtube-dlc.conf') if not os.path.exists(location): parser.error('config-location %s does not exist.' % location) - configs['custom'] = _readOptions(location) - - if '--ignore-config' in configs['command_line']: + configs['custom'] = _readOptions(location, default=None) + if configs['custom'] is None: + configs['custom'] = [] + else: + paths['custom'] = location + if '--ignore-config' in configs['command-line']: return if '--ignore-config' in configs['custom']: return + def read_options(path, user=False): + func = _readUserConf if user else _readOptions + current_path = os.path.join(path, 'yt-dlp.conf') + config = func(current_path, default=None) + if user: + config, current_path = config + if config is None: + current_path = os.path.join(path, 'youtube-dlc.conf') + config = func(current_path, default=None) + if user: + config, current_path = config + if config is None: + return [], None + return config, current_path + def get_portable_path(): path = os.path.dirname(sys.argv[0]) if os.path.abspath(sys.argv[0]) != os.path.abspath(sys.executable): # Not packaged path = os.path.join(path, '..') return os.path.abspath(path) - run_path = get_portable_path() - configs['portable'] = _readOptions(os.path.join(run_path, 'yt-dlp.conf'), default=None) - if configs['portable'] is None: - configs['portable'] = _readOptions(os.path.join(run_path, 'youtube-dlc.conf')) - + configs['portable'], paths['portable'] = read_options(get_portable_path()) if '--ignore-config' in configs['portable']: return - configs['system'] = _readOptions('/etc/yt-dlp.conf', default=None) - if configs['system'] is None: - configs['system'] = _readOptions('/etc/youtube-dlc.conf') + def get_home_path(): + opts = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line'])[0] + return expand_path(opts.paths.get('home', '')).strip() + + configs['home'], paths['home'] = read_options(get_home_path()) + if '--ignore-config' in configs['home']: + return + + configs['system'], paths['system'] = read_options('/etc') if '--ignore-config' in configs['system']: return - configs['user'] = _readUserConf('yt-dlp', default=None) - if configs['user'] is None: - configs['user'] = _readUserConf('youtube-dlc') + + configs['user'], paths['user'] = read_options('', True) if '--ignore-config' in configs['user']: - configs['system'] = [] + configs['system'], paths['system'] = [], None get_configs() - argv = configs['system'] + configs['user'] + configs['portable'] + configs['custom'] + configs['command_line'] + argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line'] opts, args = parser.parse_args(argv) if opts.verbose: - for conf_label, conf in ( - ('System config', configs['system']), - ('User config', configs['user']), - ('Portable config', configs['portable']), - ('Custom config', configs['custom']), - ('Command-line args', configs['command_line'])): - write_string('[debug] %s: %s\n' % (conf_label, repr(_hide_login_info(conf)))) + for label in ('System', 'User', 'Portable', 'Home', 'Custom', 'Command-line'): + key = label.lower() + if paths.get(key) is None: + continue + if paths[key]: + write_string('[debug] %s config file: %s\n' % (label, paths[key])) + write_string('[debug] %s config: %s\n' % (label, repr(_hide_login_info(configs[key])))) return parser, opts, args diff --git a/youtube_dlc/postprocessor/__init__.py b/youtube_dlc/postprocessor/__init__.py index e160909a7..840a83b0e 100644 --- a/youtube_dlc/postprocessor/__init__.py +++ b/youtube_dlc/postprocessor/__init__.py @@ -17,6 +17,7 @@ from .ffmpeg import ( from .xattrpp import XAttrMetadataPP from .execafterdownload import ExecAfterDownloadPP from .metadatafromtitle import MetadataFromTitlePP +from .movefilesafterdownload import MoveFilesAfterDownloadPP from .sponskrub import SponSkrubPP @@ -39,6 +40,7 @@ __all__ = [ 'FFmpegVideoConvertorPP', 'FFmpegVideoRemuxerPP', 'MetadataFromTitlePP', + 'MoveFilesAfterDownloadPP', 'SponSkrubPP', 'XAttrMetadataPP', ] diff --git a/youtube_dlc/postprocessor/movefilesafterdownload.py b/youtube_dlc/postprocessor/movefilesafterdownload.py new file mode 100644 index 000000000..3f7f529a9 --- /dev/null +++ b/youtube_dlc/postprocessor/movefilesafterdownload.py @@ -0,0 +1,52 @@ +from __future__ import unicode_literals +import os +import shutil + +from .common import PostProcessor +from ..utils import ( + encodeFilename, + make_dir, + PostProcessingError, +) +from ..compat import compat_str + + +class MoveFilesAfterDownloadPP(PostProcessor): + + def __init__(self, downloader, files_to_move): + PostProcessor.__init__(self, downloader) + self.files_to_move = files_to_move + + @classmethod + def pp_key(cls): + return 'MoveFiles' + + def run(self, info): + if info.get('__dl_filename') is None: + return [], info + self.files_to_move.setdefault(info['__dl_filename'], '') + outdir = os.path.dirname(os.path.abspath(encodeFilename(info['__final_filename']))) + + for oldfile, newfile in self.files_to_move.items(): + if not os.path.exists(encodeFilename(oldfile)): + self.report_warning('File "%s" cannot be found' % oldfile) + continue + if not newfile: + newfile = compat_str(os.path.join(outdir, os.path.basename(encodeFilename(oldfile)))) + if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)): + continue + if os.path.exists(encodeFilename(newfile)): + if self.get_param('overwrites', True): + self.report_warning('Replacing existing file "%s"' % newfile) + os.path.remove(encodeFilename(newfile)) + else: + self.report_warning( + 'Cannot move file "%s" out of temporary directory since "%s" already exists. ' + % (oldfile, newfile)) + continue + make_dir(newfile, PostProcessingError) + self.to_screen('Moving file "%s" to "%s"' % (oldfile, newfile)) + shutil.move(oldfile, newfile) # os.rename cannot move between volumes + + info['filepath'] = info['__final_filename'] + return [], info diff --git a/youtube_dlc/utils.py b/youtube_dlc/utils.py index 1ec30bafd..6740f0cdb 100644 --- a/youtube_dlc/utils.py +++ b/youtube_dlc/utils.py @@ -5893,3 +5893,15 @@ _HEX_TABLE = '0123456789abcdef' def random_uuidv4(): return re.sub(r'[xy]', lambda x: _HEX_TABLE[random.randint(0, 15)], 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx') + + +def make_dir(path, to_screen=None): + try: + dn = os.path.dirname(path) + if dn and not os.path.exists(dn): + os.makedirs(dn) + return True + except (OSError, IOError) as err: + if callable(to_screen) is not None: + to_screen('unable to create directory ' + error_to_compat_str(err)) + return False