#29 New option -P/--paths to give different paths for different types of files

Syntax: `-P "type:path" -P "type:path"`
Types: home, temp, description, annotation, subtitle, infojson, thumbnail

README.md

@@ -150,9 +150,9 @@ Then simply type this
                                      compatibility) if this option is found
                                      inside the system configuration file, the
                                      user configuration is not loaded
-    --config-location PATH           Location of the configuration file; either
-                                     the path to the config or its containing
-                                     directory
+    --config-location PATH           Location of the main configuration file;
+                                     either the path to the config or its
+                                     containing directory
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them
     --flat-videos                    Do not resolve the video urls
@@ -316,6 +316,17 @@ Then simply type this
                                      stdin), one URL per line. Lines starting
                                      with '#', ';' or ']' are considered as
                                      comments and ignored
+    -P, --paths TYPE:PATH            The paths where the files should be
+                                     downloaded. Specify the type of file and
+                                     the path separated by a colon ":"
+                                     (supported: description|annotation|subtitle
+                                     |infojson|thumbnail). Additionally, you can
+                                     also provide "home" and "temp" paths. All
+                                     intermediary files are first downloaded to
+                                     the temp path and then the final files are
+                                     moved over to the home path after download
+                                     is finished. Note that this option is
+                                     ignored if --output is an absolute path
     -o, --output TEMPLATE            Output filename template, see "OUTPUT
                                      TEMPLATE" for details
     --autonumber-start NUMBER        Specify the start value for %(autonumber)s
@@ -651,8 +662,9 @@ Then simply type this
 
 You can configure youtube-dlc by placing any supported command line option to a configuration file. The configuration is loaded from the following locations:
 
-1. The file given by `--config-location`
+1. **Main Configuration**: The file given by `--config-location`
 1. **Portable Configuration**: `yt-dlp.conf` or `youtube-dlc.conf` in the same directory as the bundled binary. If you are running from source-code (`<root dir>/youtube_dlc/__main__.py`), the root directory is used instead.
+1. **Home Configuration**: `yt-dlp.conf` or `youtube-dlc.conf` in the home path given by `-P "home:<path>"`, or in the current directory if no such path is given
 1. **User Configuration**:
     * `%XDG_CONFIG_HOME%/yt-dlp/config` (recommended on Linux/macOS)
     * `%XDG_CONFIG_HOME%/yt-dlp.conf`
@@ -710,7 +722,7 @@ set HOME=%USERPROFILE%
 
 # OUTPUT TEMPLATE
 
-The `-o` option allows users to indicate a template for the output file names.
+The `-o` option is used to indicate a template for the output file names while `-P` option is used to specify the path each type of file should be saved to.
 
 **tl;dr:** [navigate me to examples](#output-template-examples).
 

youtube_dlc/YoutubeDL.py

@@ -69,6 +69,7 @@ from .utils import (
     iri_to_uri,
     ISO3166Utils,
     locked_file,
+    make_dir,
     make_HTTPS_handler,
     MaxDownloadsReached,
     orderedSet,
@@ -114,8 +115,9 @@ from .postprocessor import (
     FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
-    FFmpegSubtitlesConvertorPP,
+    # FFmpegSubtitlesConvertorPP,
     get_postprocessor,
+    MoveFilesAfterDownloadPP,
 )
 from .version import __version__
 
@@ -257,6 +259,8 @@ class YoutubeDL(object):
     postprocessors:    A list of dictionaries, each with an entry
                        * key:  The name of the postprocessor. See
                                youtube_dlc/postprocessor/__init__.py for a list.
+                       * _after_move: Optional. If True, run this post_processor
+                               after 'MoveFilesAfterDownload'
                        as well as any further keyword arguments for the
                        postprocessor.
     post_hooks:        A list of functions that get called as the final step
@@ -369,6 +373,8 @@ class YoutubeDL(object):
     params = None
     _ies = []
     _pps = []
+    _pps_end = []
+    __prepare_filename_warned = False
     _download_retcode = None
     _num_downloads = None
     _playlist_level = 0
@@ -382,6 +388,8 @@ class YoutubeDL(object):
         self._ies = []
         self._ies_instances = {}
         self._pps = []
+        self._pps_end = []
+        self.__prepare_filename_warned = False
         self._post_hooks = []
         self._progress_hooks = []
         self._download_retcode = 0
@@ -483,8 +491,11 @@ class YoutubeDL(object):
             pp_class = get_postprocessor(pp_def_raw['key'])
             pp_def = dict(pp_def_raw)
             del pp_def['key']
+            after_move = pp_def.get('_after_move', False)
+            if '_after_move' in pp_def:
+                del pp_def['_after_move']
             pp = pp_class(self, **compat_kwargs(pp_def))
-            self.add_post_processor(pp)
+            self.add_post_processor(pp, after_move=after_move)
 
         for ph in self.params.get('post_hooks', []):
             self.add_post_hook(ph)
@@ -536,9 +547,12 @@ class YoutubeDL(object):
         for ie in gen_extractor_classes():
             self.add_info_extractor(ie)
 
-    def add_post_processor(self, pp):
+    def add_post_processor(self, pp, after_move=False):
         """Add a PostProcessor object to the end of the chain."""
-        self._pps.append(pp)
+        if after_move:
+            self._pps_end.append(pp)
+        else:
+            self._pps.append(pp)
         pp.set_downloader(self)
 
     def add_post_hook(self, ph):
@@ -702,7 +716,7 @@ class YoutubeDL(object):
         except UnicodeEncodeError:
             self.to_screen('Deleting already existent file')
 
-    def prepare_filename(self, info_dict):
+    def prepare_filename(self, info_dict, warn=False):
         """Generate the output filename."""
         try:
             template_dict = dict(info_dict)
@@ -796,11 +810,33 @@ class YoutubeDL(object):
             # to workaround encoding issues with subprocess on python2 @ Windows
             if sys.version_info < (3, 0) and sys.platform == 'win32':
                 filename = encodeFilename(filename, True).decode(preferredencoding())
-            return sanitize_path(filename)
+            filename = sanitize_path(filename)
+
+            if warn and not self.__prepare_filename_warned:
+                if not self.params.get('paths'):
+                    pass
+                elif filename == '-':
+                    self.report_warning('--paths is ignored when an outputting to stdout')
+                elif os.path.isabs(filename):
+                    self.report_warning('--paths is ignored since an absolute path is given in output template')
+                self.__prepare_filename_warned = True
+
+            return filename
         except ValueError as err:
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None
 
+    def prepare_filepath(self, filename, dir_type=''):
+        if filename == '-':
+            return filename
+        paths = self.params.get('paths', {})
+        assert isinstance(paths, dict)
+        homepath = expand_path(paths.get('home', '').strip())
+        assert isinstance(homepath, compat_str)
+        subdir = expand_path(paths.get(dir_type, '').strip()) if dir_type else ''
+        assert isinstance(subdir, compat_str)
+        return sanitize_path(os.path.join(homepath, subdir, filename))
+
     def _match_entry(self, info_dict, incomplete):
         """ Returns None if the file should be downloaded """
 
@@ -972,7 +1008,8 @@ class YoutubeDL(object):
             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info)
                     or extract_flat is True):
                 self.__forced_printings(
-                    ie_result, self.prepare_filename(ie_result),
+                    ie_result,
+                    self.prepare_filepath(self.prepare_filename(ie_result)),
                     incomplete=True)
                 return ie_result
 
@@ -1890,6 +1927,8 @@ class YoutubeDL(object):
 
         assert info_dict.get('_type', 'video') == 'video'
 
+        info_dict.setdefault('__postprocessors', [])
+
         max_downloads = self.params.get('max_downloads')
         if max_downloads is not None:
             if self._num_downloads >= int(max_downloads):
@@ -1906,10 +1945,13 @@ class YoutubeDL(object):
 
         self._num_downloads += 1
 
-        info_dict['_filename'] = filename = self.prepare_filename(info_dict)
+        filename = self.prepare_filename(info_dict, warn=True)
+        info_dict['_filename'] = full_filename = self.prepare_filepath(filename)
+        temp_filename = self.prepare_filepath(filename, 'temp')
+        files_to_move = {}
 
         # Forced printings
-        self.__forced_printings(info_dict, filename, incomplete=False)
+        self.__forced_printings(info_dict, full_filename, incomplete=False)
 
         if self.params.get('simulate', False):
             if self.params.get('force_write_download_archive', False):
@@ -1922,20 +1964,19 @@ class YoutubeDL(object):
             return
 
         def ensure_dir_exists(path):
-            try:
-                dn = os.path.dirname(path)
-                if dn and not os.path.exists(dn):
-                    os.makedirs(dn)
-                return True
-            except (OSError, IOError) as err:
-                self.report_error('unable to create directory ' + error_to_compat_str(err))
-                return False
+            return make_dir(path, self.report_error)
 
-        if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
+        if not ensure_dir_exists(encodeFilename(full_filename)):
+            return
+        if not ensure_dir_exists(encodeFilename(temp_filename)):
             return
 
         if self.params.get('writedescription', False):
-            descfn = replace_extension(filename, 'description', info_dict.get('ext'))
+            descfn = replace_extension(
+                self.prepare_filepath(filename, 'description'),
+                'description', info_dict.get('ext'))
+            if not ensure_dir_exists(encodeFilename(descfn)):
+                return
             if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)):
                 self.to_screen('[info] Video description is already present')
             elif info_dict.get('description') is None:
@@ -1950,7 +1991,11 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeannotations', False):
-            annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
+            annofn = replace_extension(
+                self.prepare_filepath(filename, 'annotation'),
+                'annotations.xml', info_dict.get('ext'))
+            if not ensure_dir_exists(encodeFilename(annofn)):
+                return
             if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)):
                 self.to_screen('[info] Video annotations are already present')
             elif not info_dict.get('annotations'):
@@ -1984,9 +2029,13 @@ class YoutubeDL(object):
             # ie = self.get_info_extractor(info_dict['extractor_key'])
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
-                sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext'))
+                sub_filename = subtitles_filename(temp_filename, sub_lang, sub_format, info_dict.get('ext'))
+                sub_filename_final = subtitles_filename(
+                    self.prepare_filepath(filename, 'subtitle'),
+                    sub_lang, sub_format, info_dict.get('ext'))
                 if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
                     self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
+                    files_to_move[sub_filename] = sub_filename_final
                 else:
                     self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
                     if sub_info.get('data') is not None:
@@ -1995,6 +2044,7 @@ class YoutubeDL(object):
                             # See https://github.com/ytdl-org/youtube-dl/issues/10268
                             with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                                 subfile.write(sub_info['data'])
+                            files_to_move[sub_filename] = sub_filename_final
                         except (OSError, IOError):
                             self.report_error('Cannot write subtitles file ' + sub_filename)
                             return
@@ -2010,6 +2060,7 @@ class YoutubeDL(object):
                                 with io.open(encodeFilename(sub_filename), 'wb') as subfile:
                                     subfile.write(sub_data)
                             '''
+                            files_to_move[sub_filename] = sub_filename_final
                         except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                             self.report_warning('Unable to download subtitle for "%s": %s' %
                                                 (sub_lang, error_to_compat_str(err)))
@@ -2017,29 +2068,32 @@ class YoutubeDL(object):
 
         if self.params.get('skip_download', False):
             if self.params.get('convertsubtitles', False):
-                subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles'))
+                # subconv = FFmpegSubtitlesConvertorPP(self, format=self.params.get('convertsubtitles'))
                 filename_real_ext = os.path.splitext(filename)[1][1:]
                 filename_wo_ext = (
-                    os.path.splitext(filename)[0]
+                    os.path.splitext(full_filename)[0]
                     if filename_real_ext == info_dict['ext']
-                    else filename)
+                    else full_filename)
                 afilename = '%s.%s' % (filename_wo_ext, self.params.get('convertsubtitles'))
-                if subconv.available:
-                    info_dict.setdefault('__postprocessors', [])
-                    # info_dict['__postprocessors'].append(subconv)
+                # if subconv.available:
+                #     info_dict['__postprocessors'].append(subconv)
                 if os.path.exists(encodeFilename(afilename)):
                     self.to_screen(
                         '[download] %s has already been downloaded and '
                         'converted' % afilename)
                 else:
                     try:
-                        self.post_process(filename, info_dict)
+                        self.post_process(full_filename, info_dict, files_to_move)
                     except (PostProcessingError) as err:
                         self.report_error('postprocessing: %s' % str(err))
                         return
 
         if self.params.get('writeinfojson', False):
-            infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
+            infofn = replace_extension(
+                self.prepare_filepath(filename, 'infojson'),
+                'info.json', info_dict.get('ext'))
+            if not ensure_dir_exists(encodeFilename(infofn)):
+                return
             if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)):
                 self.to_screen('[info] Video description metadata is already present')
             else:
@@ -2050,7 +2104,9 @@ class YoutubeDL(object):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
 
-        self._write_thumbnails(info_dict, filename)
+        thumbdir = os.path.dirname(self.prepare_filepath(filename, 'thumbnail'))
+        for thumbfn in self._write_thumbnails(info_dict, temp_filename):
+            files_to_move[thumbfn] = os.path.join(thumbdir, os.path.basename(thumbfn))
 
         # Write internet shortcut files
         url_link = webloc_link = desktop_link = False
@@ -2075,7 +2131,7 @@ class YoutubeDL(object):
             ascii_url = iri_to_uri(info_dict['webpage_url'])
 
         def _write_link_file(extension, template, newline, embed_filename):
-            linkfn = replace_extension(filename, extension, info_dict.get('ext'))
+            linkfn = replace_extension(full_filename, extension, info_dict.get('ext'))
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(linkfn)):
                 self.to_screen('[info] Internet shortcut is already present')
             else:
@@ -2105,9 +2161,27 @@ class YoutubeDL(object):
         must_record_download_archive = False
         if not self.params.get('skip_download', False):
             try:
+
+                def existing_file(filename, temp_filename):
+                    file_exists = os.path.exists(encodeFilename(filename))
+                    tempfile_exists = (
+                        False if temp_filename == filename
+                        else os.path.exists(encodeFilename(temp_filename)))
+                    if not self.params.get('overwrites', False) and (file_exists or tempfile_exists):
+                        existing_filename = temp_filename if tempfile_exists else filename
+                        self.to_screen('[download] %s has already been downloaded and merged' % existing_filename)
+                        return existing_filename
+                    if tempfile_exists:
+                        self.report_file_delete(temp_filename)
+                        os.remove(encodeFilename(temp_filename))
+                    if file_exists:
+                        self.report_file_delete(filename)
+                        os.remove(encodeFilename(filename))
+                    return None
+
+                success = True
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
-                    success = True
                     merger = FFmpegMergerPP(self)
                     if not merger.available:
                         postprocessors = []
@@ -2136,32 +2210,31 @@ class YoutubeDL(object):
                         # TODO: Check acodec/vcodec
                         return False
 
-                    filename_real_ext = os.path.splitext(filename)[1][1:]
-                    filename_wo_ext = (
-                        os.path.splitext(filename)[0]
-                        if filename_real_ext == info_dict['ext']
-                        else filename)
                     requested_formats = info_dict['requested_formats']
+                    old_ext = info_dict['ext']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         info_dict['ext'] = 'mkv'
                         self.report_warning(
                             'Requested formats are incompatible for merge and will be merged into mkv.')
+
+                    def correct_ext(filename):
+                        filename_real_ext = os.path.splitext(filename)[1][1:]
+                        filename_wo_ext = (
+                            os.path.splitext(filename)[0]
+                            if filename_real_ext == old_ext
+                            else filename)
+                        return '%s.%s' % (filename_wo_ext, info_dict['ext'])
+
                     # Ensure filename always has a correct extension for successful merge
-                    filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
-                    file_exists = os.path.exists(encodeFilename(filename))
-                    if not self.params.get('overwrites', False) and file_exists:
-                        self.to_screen(
-                            '[download] %s has already been downloaded and '
-                            'merged' % filename)
-                    else:
-                        if file_exists:
-                            self.report_file_delete(filename)
-                            os.remove(encodeFilename(filename))
+                    full_filename = correct_ext(full_filename)
+                    temp_filename = correct_ext(temp_filename)
+                    dl_filename = existing_file(full_filename, temp_filename)
+                    if dl_filename is None:
                         for f in requested_formats:
                             new_info = dict(info_dict)
                             new_info.update(f)
                             fname = prepend_extension(
-                                self.prepare_filename(new_info),
+                                self.prepare_filepath(self.prepare_filename(new_info), 'temp'),
                                 'f%s' % f['format_id'], new_info['ext'])
                             if not ensure_dir_exists(fname):
                                 return
@@ -2173,14 +2246,17 @@ class YoutubeDL(object):
                         # Even if there were no downloads, it is being merged only now
                         info_dict['__real_download'] = True
                 else:
-                    # Delete existing file with --yes-overwrites
-                    if self.params.get('overwrites', False):
-                        if os.path.exists(encodeFilename(filename)):
-                            self.report_file_delete(filename)
-                            os.remove(encodeFilename(filename))
                     # Just a single file
-                    success, real_download = dl(filename, info_dict)
-                    info_dict['__real_download'] = real_download
+                    dl_filename = existing_file(full_filename, temp_filename)
+                    if dl_filename is None:
+                        success, real_download = dl(temp_filename, info_dict)
+                        info_dict['__real_download'] = real_download
+
+                # info_dict['__temp_filename'] = temp_filename
+                dl_filename = dl_filename or temp_filename
+                info_dict['__dl_filename'] = dl_filename
+                info_dict['__final_filename'] = full_filename
+
             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
                 return
@@ -2206,7 +2282,6 @@ class YoutubeDL(object):
                     elif fixup_policy == 'detect_or_warn':
                         stretched_pp = FFmpegFixupStretchedPP(self)
                         if stretched_pp.available:
-                            info_dict.setdefault('__postprocessors', [])
                             info_dict['__postprocessors'].append(stretched_pp)
                         else:
                             self.report_warning(
@@ -2225,7 +2300,6 @@ class YoutubeDL(object):
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM4aPP(self)
                         if fixup_pp.available:
-                            info_dict.setdefault('__postprocessors', [])
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
@@ -2244,7 +2318,6 @@ class YoutubeDL(object):
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM3u8PP(self)
                         if fixup_pp.available:
-                            info_dict.setdefault('__postprocessors', [])
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
@@ -2254,13 +2327,13 @@ class YoutubeDL(object):
                         assert fixup_policy in ('ignore', 'never')
 
                 try:
-                    self.post_process(filename, info_dict)
+                    self.post_process(dl_filename, info_dict, files_to_move)
                 except (PostProcessingError) as err:
                     self.report_error('postprocessing: %s' % str(err))
                     return
                 try:
                     for ph in self._post_hooks:
-                        ph(filename)
+                        ph(full_filename)
                 except Exception as err:
                     self.report_error('post hooks: %s' % str(err))
                     return
@@ -2326,27 +2399,41 @@ class YoutubeDL(object):
             (k, v) for k, v in info_dict.items()
             if k not in ['requested_formats', 'requested_subtitles'])
 
-    def post_process(self, filename, ie_info):
+    def post_process(self, filename, ie_info, files_to_move={}):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)
         info['filepath'] = filename
-        pps_chain = []
-        if ie_info.get('__postprocessors') is not None:
-            pps_chain.extend(ie_info['__postprocessors'])
-        pps_chain.extend(self._pps)
-        for pp in pps_chain:
+
+        def run_pp(pp):
             files_to_delete = []
+            infodict = info
             try:
-                files_to_delete, info = pp.run(info)
+                files_to_delete, infodict = pp.run(infodict)
             except PostProcessingError as e:
                 self.report_error(e.msg)
-            if files_to_delete and not self.params.get('keepvideo', False):
+            if not files_to_delete:
+                return infodict
+
+            if self.params.get('keepvideo', False):
+                for f in files_to_delete:
+                    files_to_move.setdefault(f, '')
+            else:
                 for old_filename in set(files_to_delete):
                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
                     try:
                         os.remove(encodeFilename(old_filename))
                     except (IOError, OSError):
                         self.report_warning('Unable to remove downloaded original file')
+                    if old_filename in files_to_move:
+                        del files_to_move[old_filename]
+            return infodict
+
+        for pp in ie_info.get('__postprocessors', []) + self._pps:
+            info = run_pp(pp)
+        info = run_pp(MoveFilesAfterDownloadPP(self, files_to_move))
+        files_to_move = {}
+        for pp in self._pps_end:
+            info = run_pp(pp)
 
     def _make_archive_id(self, info_dict):
         video_id = info_dict.get('id')
@@ -2700,14 +2787,11 @@ class YoutubeDL(object):
             if thumbnails:
                 thumbnails = [thumbnails[-1]]
         elif self.params.get('write_all_thumbnails', False):
-            thumbnails = info_dict.get('thumbnails')
+            thumbnails = info_dict.get('thumbnails') or []
         else:
-            return
-
-        if not thumbnails:
-            # No thumbnails present, so return immediately
-            return
+            thumbnails = []
 
+        ret = []
         for t in thumbnails:
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
@@ -2715,6 +2799,7 @@ class YoutubeDL(object):
             t['filename'] = thumb_filename = replace_extension(filename + suffix, thumb_ext, info_dict.get('ext'))
 
             if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
+                ret.append(thumb_filename)
                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
             else:
@@ -2724,8 +2809,10 @@ class YoutubeDL(object):
                     uf = self.urlopen(t['url'])
                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
                         shutil.copyfileobj(uf, thumbf)
+                    ret.append(thumb_filename)
                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                     self.report_warning('Unable to download thumbnail "%s": %s' %
                                         (t['url'], error_to_compat_str(err)))
+        return ret

youtube_dlc/__init__.py

@@ -244,6 +244,7 @@ def _real_main(argv=None):
         parser.error('Cannot download a video and extract audio into the same'
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                      ' template'.format(outtmpl))
+
     for f in opts.format_sort:
         if re.match(InfoExtractor.FormatSort.regex, f) is None:
             parser.error('invalid format sort string "%s" specified' % f)
@@ -318,12 +319,12 @@ def _real_main(argv=None):
             'force': opts.sponskrub_force,
             'ignoreerror': opts.sponskrub is None,
         })
-    # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
-    # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
+    # ExecAfterDownload must be the last PP
     if opts.exec_cmd:
         postprocessors.append({
             'key': 'ExecAfterDownload',
             'exec_cmd': opts.exec_cmd,
+            '_after_move': True
         })
 
     _args_compat_warning = 'WARNING: %s given without specifying name. The arguments will be given to all %s\n'
@@ -372,6 +373,7 @@ def _real_main(argv=None):
         'listformats': opts.listformats,
         'listformats_table': opts.listformats_table,
         'outtmpl': outtmpl,
+        'paths': opts.paths,
         'autonumber_size': opts.autonumber_size,
         'autonumber_start': opts.autonumber_start,
         'restrictfilenames': opts.restrictfilenames,

youtube_dlc/options.py

@@ -14,6 +14,7 @@ from .compat import (
     compat_shlex_split,
 )
 from .utils import (
+    expand_path,
     preferredencoding,
     write_string,
 )
@@ -62,7 +63,7 @@ def parseOpts(overrideArguments=None):
             userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name)
         userConf = _readOptions(userConfFile, default=None)
         if userConf is not None:
-            return userConf
+            return userConf, userConfFile
 
         # appdata
         appdata_dir = compat_getenv('appdata')
@@ -70,19 +71,21 @@ def parseOpts(overrideArguments=None):
             userConfFile = os.path.join(appdata_dir, package_name, 'config')
             userConf = _readOptions(userConfFile, default=None)
             if userConf is None:
-                userConf = _readOptions('%s.txt' % userConfFile, default=None)
+                userConfFile += '.txt'
+                userConf = _readOptions(userConfFile, default=None)
         if userConf is not None:
-            return userConf
+            return userConf, userConfFile
 
         # home
         userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name)
         userConf = _readOptions(userConfFile, default=None)
         if userConf is None:
-            userConf = _readOptions('%s.txt' % userConfFile, default=None)
+            userConfFile += '.txt'
+            userConf = _readOptions(userConfFile, default=None)
         if userConf is not None:
-            return userConf
+            return userConf, userConfFile
 
-        return default
+        return default, None
 
     def _format_option_string(option):
         ''' ('-o', '--option') -> -o, --format METAVAR'''
@@ -187,7 +190,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--config-location',
         dest='config_location', metavar='PATH',
-        help='Location of the configuration file; either the path to the config or its containing directory')
+        help='Location of the main configuration file; either the path to the config or its containing directory')
     general.add_option(
         '--flat-playlist',
         action='store_const', dest='extract_flat', const='in_playlist', default=False,
@@ -641,7 +644,7 @@ def parseOpts(overrideArguments=None):
         metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',
         action='callback', callback=_dict_from_multiple_values_options_callback,
         callback_kwargs={
-            'allowed_keys': '|'.join(list_external_downloaders()), 
+            'allowed_keys': '|'.join(list_external_downloaders()),
             'default_key': 'default', 'process': compat_shlex_split},
         help=(
             'Give these arguments to the external downloader. '
@@ -819,6 +822,21 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--id', default=False,
         action='store_true', dest='useid', help=optparse.SUPPRESS_HELP)
+    filesystem.add_option(
+        '-P', '--paths',
+        metavar='TYPE:PATH', dest='paths', default={}, type='str',
+        action='callback', callback=_dict_from_multiple_values_options_callback,
+        callback_kwargs={
+            'allowed_keys': 'home|temp|config|description|annotation|subtitle|infojson|thumbnail',
+            'process': lambda x: x.strip()},
+        help=(
+            'The paths where the files should be downloaded. '
+            'Specify the type of file and the path separated by a colon ":" '
+            '(supported: description|annotation|subtitle|infojson|thumbnail). '
+            'Additionally, you can also provide "home" and "temp" paths. '
+            'All intermediary files are first downloaded to the temp path and '
+            'then the final files are moved over to the home path after download is finished. '
+            'Note that this option is ignored if --output is an absolute path'))
     filesystem.add_option(
         '-o', '--output',
         dest='outtmpl', metavar='TEMPLATE',
@@ -1171,59 +1189,79 @@ def parseOpts(overrideArguments=None):
             return conf
 
         configs = {
-            'command_line': compat_conf(sys.argv[1:]),
-            'custom': [], 'portable': [], 'user': [], 'system': []}
-        opts, args = parser.parse_args(configs['command_line'])
+            'command-line': compat_conf(sys.argv[1:]),
+            'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []}
+        paths = {'command-line': False}
+        opts, args = parser.parse_args(configs['command-line'])
 
         def get_configs():
-            if '--config-location' in configs['command_line']:
+            if '--config-location' in configs['command-line']:
                 location = compat_expanduser(opts.config_location)
                 if os.path.isdir(location):
                     location = os.path.join(location, 'youtube-dlc.conf')
                 if not os.path.exists(location):
                     parser.error('config-location %s does not exist.' % location)
-                configs['custom'] = _readOptions(location)
-
-            if '--ignore-config' in configs['command_line']:
+                configs['custom'] = _readOptions(location, default=None)
+                if configs['custom'] is None:
+                    configs['custom'] = []
+                else:
+                    paths['custom'] = location
+            if '--ignore-config' in configs['command-line']:
                 return
             if '--ignore-config' in configs['custom']:
                 return
 
+            def read_options(path, user=False):
+                func = _readUserConf if user else _readOptions
+                current_path = os.path.join(path, 'yt-dlp.conf')
+                config = func(current_path, default=None)
+                if user:
+                    config, current_path = config
+                if config is None:
+                    current_path = os.path.join(path, 'youtube-dlc.conf')
+                    config = func(current_path, default=None)
+                    if user:
+                        config, current_path = config
+                if config is None:
+                    return [], None
+                return config, current_path
+
             def get_portable_path():
                 path = os.path.dirname(sys.argv[0])
                 if os.path.abspath(sys.argv[0]) != os.path.abspath(sys.executable):  # Not packaged
                     path = os.path.join(path, '..')
                 return os.path.abspath(path)
 
-            run_path = get_portable_path()
-            configs['portable'] = _readOptions(os.path.join(run_path, 'yt-dlp.conf'), default=None)
-            if configs['portable'] is None:
-                configs['portable'] = _readOptions(os.path.join(run_path, 'youtube-dlc.conf'))
-
+            configs['portable'], paths['portable'] = read_options(get_portable_path())
             if '--ignore-config' in configs['portable']:
                 return
-            configs['system'] = _readOptions('/etc/yt-dlp.conf', default=None)
-            if configs['system'] is None:
-                configs['system'] = _readOptions('/etc/youtube-dlc.conf')
 
+            def get_home_path():
+                opts = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line'])[0]
+                return expand_path(opts.paths.get('home', '')).strip()
+
+            configs['home'], paths['home'] = read_options(get_home_path())
+            if '--ignore-config' in configs['home']:
+                return
+
+            configs['system'], paths['system'] = read_options('/etc')
             if '--ignore-config' in configs['system']:
                 return
-            configs['user'] = _readUserConf('yt-dlp', default=None)
-            if configs['user'] is None:
-                configs['user'] = _readUserConf('youtube-dlc')
+
+            configs['user'], paths['user'] = read_options('', True)
             if '--ignore-config' in configs['user']:
-                configs['system'] = []
+                configs['system'], paths['system'] = [], None
 
         get_configs()
-        argv = configs['system'] + configs['user'] + configs['portable'] + configs['custom'] + configs['command_line']
+        argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line']
         opts, args = parser.parse_args(argv)
         if opts.verbose:
-            for conf_label, conf in (
-                    ('System config', configs['system']),
-                    ('User config', configs['user']),
-                    ('Portable config', configs['portable']),
-                    ('Custom config', configs['custom']),
-                    ('Command-line args', configs['command_line'])):
-                write_string('[debug] %s: %s\n' % (conf_label, repr(_hide_login_info(conf))))
+            for label in ('System', 'User', 'Portable', 'Home', 'Custom', 'Command-line'):
+                key = label.lower()
+                if paths.get(key) is None:
+                    continue
+                if paths[key]:
+                    write_string('[debug] %s config file: %s\n' % (label, paths[key]))
+                write_string('[debug] %s config: %s\n' % (label, repr(_hide_login_info(configs[key]))))
 
     return parser, opts, args

youtube_dlc/postprocessor/__init__.py

@@ -17,6 +17,7 @@ from .ffmpeg import (
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
 from .metadatafromtitle import MetadataFromTitlePP
+from .movefilesafterdownload import MoveFilesAfterDownloadPP
 from .sponskrub import SponSkrubPP
 
 
@@ -39,6 +40,7 @@ __all__ = [
     'FFmpegVideoConvertorPP',
     'FFmpegVideoRemuxerPP',
     'MetadataFromTitlePP',
+    'MoveFilesAfterDownloadPP',
     'SponSkrubPP',
     'XAttrMetadataPP',
 ]

youtube_dlc/postprocessor/movefilesafterdownload.py

@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+import os
+import shutil
+
+from .common import PostProcessor
+from ..utils import (
+    encodeFilename,
+    make_dir,
+    PostProcessingError,
+)
+from ..compat import compat_str
+
+
+class MoveFilesAfterDownloadPP(PostProcessor):
+
+    def __init__(self, downloader, files_to_move):
+        PostProcessor.__init__(self, downloader)
+        self.files_to_move = files_to_move
+
+    @classmethod
+    def pp_key(cls):
+        return 'MoveFiles'
+
+    def run(self, info):
+        if info.get('__dl_filename') is None:
+            return [], info
+        self.files_to_move.setdefault(info['__dl_filename'], '')
+        outdir = os.path.dirname(os.path.abspath(encodeFilename(info['__final_filename'])))
+
+        for oldfile, newfile in self.files_to_move.items():
+            if not os.path.exists(encodeFilename(oldfile)):
+                self.report_warning('File "%s" cannot be found' % oldfile)
+                continue
+            if not newfile:
+                newfile = compat_str(os.path.join(outdir, os.path.basename(encodeFilename(oldfile))))
+            if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)):
+                continue
+            if os.path.exists(encodeFilename(newfile)):
+                if self.get_param('overwrites', True):
+                    self.report_warning('Replacing existing file "%s"' % newfile)
+                    os.path.remove(encodeFilename(newfile))
+                else:
+                    self.report_warning(
+                        'Cannot move file "%s" out of temporary directory since "%s" already exists. '
+                        % (oldfile, newfile))
+                    continue
+            make_dir(newfile, PostProcessingError)
+            self.to_screen('Moving file "%s" to "%s"' % (oldfile, newfile))
+            shutil.move(oldfile, newfile)  # os.rename cannot move between volumes
+
+        info['filepath'] = info['__final_filename']
+        return [], info

youtube_dlc/utils.py

@@ -5893,3 +5893,15 @@ _HEX_TABLE = '0123456789abcdef'
 
 def random_uuidv4():
     return re.sub(r'[xy]', lambda x: _HEX_TABLE[random.randint(0, 15)], 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx')
+
+
+def make_dir(path, to_screen=None):
+    try:
+        dn = os.path.dirname(path)
+        if dn and not os.path.exists(dn):
+            os.makedirs(dn)
+        return True
+    except (OSError, IOError) as err:
+        if callable(to_screen) is not None:
+            to_screen('unable to create directory ' + error_to_compat_str(err))
+        return False