Support using an external command for idgames fetching.

dcc/config.py

@@ -42,6 +42,7 @@ class Base(Command):
 
     def run(self, parsed_args):
         self.init_base(parsed_args)
+        self.finalize()
         self.take_action(parsed_args)
 
     def _init_attr(self, what, default, fn=lambda x: x):

dcc/fetch.py

@@ -2,6 +2,8 @@ import dcc.config
 import io
 import json
 import pathlib
+import shutil
+import subprocess
 import urllib.request
 import zipfile
 
@@ -17,11 +19,10 @@ class Fetch(dcc.config.Base):
         if not parsed_args.id_or_name.isdigit():
             idgames_id = self.search_idgames(parsed_args.id_or_name)
 
-        with urllib.request.urlopen(
+        reply = self.fetch_url(
             "https://www.doomworld.com/idgames/api/" +
             "api.php?action=get&id={}&out=json".format(idgames_id)
-        ) as response:
+        )
-            reply = json.loads(response.read())
         rpath = "/".join([
             self.fetch_mirror,
             reply["content"]["dir"],
@@ -35,13 +36,29 @@ class Fetch(dcc.config.Base):
 
     # TODO: explicit error handling.  Let users choose when >1 result.
     def search_idgames(self, wad):
-        with urllib.request.urlopen(
+        reply = self.fetch_url(
             "https://www.doomworld.com/idgames/api/" +
             "api.php?action=search&query={}&out=json".format(wad)
-        ) as response:
+        )
-            reply = json.loads(response.read())
         files = reply["content"]["file"]
         if type(files) is dict:  # One result.
             return files["id"]
         else:  # More than one.  Zero will raise an error.
             return files[0]["id"]
+
+    # Ideally this would just be urllib.request.open.  However, Cloudflare can
+    # interfere with this.  Therefore we support using a curl_impersonate
+    # binary instead.
+    def fetch_url(self, url):
+        fetcher = self._config.get("url_fetcher")
+        if fetcher is None:
+            with urllib.request.urlopen(url) as response:
+                return json.loads(response.read())
+        
+        fetcher_path = shutil.which(fetcher)
+        if fetcher_path is None:
+            raise Exception(f"Fetch util {fetcher} not found on PATH.")
+
+        proc = subprocess.run([fetcher_path, url], capture_output=True, check = True)
+        return json.loads(proc.stdout)
+