modm_data.dl

24def download_data(url: str, encoding: str = None, errors: str = None) -> str:
25    """
26    Download and decode the data of a URL.
27
28    :param url: URL to download
29    :param encoding: optional encoding to apply (default is `utf-8`)
30    :param errors: optional error handling (default is `ignore`)
31    :return: The data as a decoded string.
32    """
33    LOGGER.debug(f"Downloading data from {url}")
34    cmd = f"curl '{url}' -L -s --max-time 120 -o - " + " ".join(f"-H '{k}: {v}'" for k, v in _hdr.items())
35    data = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE).stdout
36    return data.decode(encoding=encoding or "utf-8", errors=errors or "ignore")

39def download_file(url: str, path: Path, overwrite: bool = False) -> bool:
40    """
41    Download a file from a URL and copy it to a path, potentially overwriting an
42    existing file there. Creates directories if necessary.
43
44    :param url: File URL to download.
45    :param path: Path to copy the downloaded file to.
46    :param overwrite: If the file already exists, overwrite it.
47    :return: Whether the file was downloaded and copied.
48    """
49    if not overwrite and path.exists():
50        LOGGER.error(f"File {path} already exists!")
51        return False
52    if isinstance(path, Path):
53        path.parent.mkdir(parents=True, exist_ok=True)
54    LOGGER.debug(f"Downloading file from {url} to {path}")
55    cmd = f"curl '{url}' -L -s --max-time 60 -o {path} " + " ".join(f"-H '{k}: {v}'" for k, v in _hdr.items())
56    return subprocess.call(cmd, shell=True) == 0
57    # with tempfile.NamedTemporaryFile() as outfile:
58    #     os.system(f'wget -q --user-agent="{_hdr["User-Agent"]}" "{url}" -O {outfile.name}')
59    #     shutil.copy(outfile.name, str(path))
60    # This doesn't work with all PDFs, redirects maybe?
61    # with urlopen(Request(url, headers=_hdr)) as infile, \
62    #      tempfile.NamedTemporaryFile() as outfile:
63    #     shutil.copyfileobj(infile, outfile)
64    #     shutil.copy(outfile.name, str(path))
65    # return True