modm_data.dl

23def download_data(url: str, encoding: str = None, errors: str = None) -> str:
24    """
25    Download and decode the data of a URL.
26
27    :param url: URL to download
28    :param encoding: optional encoding to apply (default is `utf-8`)
29    :param errors: optional error handling (default is `ignore`)
30    :return: The data as a decoded string.
31    """
32    LOGGER.debug(f"Downloading data from {url}")
33    cmd = f"curl '{url}' -L -s --max-time 120 -o - " + " ".join(f"-H '{k}: {v}'" for k, v in _hdr.items())
34    data = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE).stdout
35    return data.decode(encoding=encoding or "utf-8", errors=errors or "ignore")

38def download_file(url: str, path: Path, overwrite: bool = False) -> bool:
39    """
40    Download a file from a URL and copy it to a path, potentially overwriting an
41    existing file there. Creates directories if necessary.
42
43    :param url: File URL to download.
44    :param path: Path to copy the downloaded file to.
45    :param overwrite: If the file already exists, overwrite it.
46    :return: Whether the file was downloaded and copied.
47    """
48    if not overwrite and path.exists():
49        LOGGER.error(f"File {path} already exists!")
50        return False
51    if isinstance(path, Path):
52        path.parent.mkdir(parents=True, exist_ok=True)
53    LOGGER.debug(f"Downloading file from {url} to {path}")
54    cmd = f"curl '{url}' -L -s --max-time 60 -o {path} " + " ".join(f"-H '{k}: {v}'" for k, v in _hdr.items())
55    return subprocess.call(cmd, shell=True) == 0
56    # with tempfile.NamedTemporaryFile() as outfile:
57    #     os.system(f'wget -q --user-agent="{_hdr["User-Agent"]}" "{url}" -O {outfile.name}')
58    #     shutil.copy(outfile.name, str(path))
59    # This doesn't work with all PDFs, redirects maybe?
60    # with urlopen(Request(url, headers=_hdr)) as infile, \
61    #      tempfile.NamedTemporaryFile() as outfile:
62    #     shutil.copyfileobj(infile, outfile)
63    #     shutil.copy(outfile.name, str(path))
64    # return True