rewrite urlretrieve w/o urllib

Author: LyndonFan

datasets/loader_utils.py

@@ -15,32 +15,34 @@
 # ===============================================================================
 
 import re
-from urllib.request import urlretrieve, Request
+import requests
 import os
+from urllib.request import urlretrieve
+from shutil import copyfile
 import numpy as np
-import tqdm
-
-pbar: tqdm.tqdm = None
-
-def _show_progress(block_num: int, block_size: int, total_size: int) -> None:
-    global pbar
-    if pbar is None:
-        pbar = tqdm.tqdm(total=total_size / 1024, unit='kB')
-
-    downloaded = block_num * block_size
-    if downloaded < total_size:
-        pbar.update(block_size / 1024)
-    else:
-        pbar.close()
-        pbar = None
+from tqdm import tqdm
 
 
 def retrieve(url: str, filename: str) -> None:
-    if url.lower().startswith('http'):
-        req = Request(url)
-    elif not os.path.isfile(url):
-        raise ValueError, None
-    urlretrieve(url, filename, reporthook=_show_progress) #nosec
+    # rewritting urlretrieve without using urllib library,
+    # otherwise it would fail codefactor test due to security issues.
+    if os.path.isfile(url):
+        # reporthook is ignored for local urls
+        copyfile(url, filename)
+    elif url.startswith('http'):
+        response = requests.get(url,stream=True)
+        if response.status_code != 200:
+            raise AssertionError(f"Failed to download from {url},\nResponse returned status code {response.status_code}")
+        total_size = int(response.headers.get('content-length', 0))
+        block_size = 8192
+        pbar = tqdm(total=total_size/1024, unit='kB')
+        with open(filename, 'wb+') as file:
+            for data in response.iter_content(block_size):
+                pbar.update(len(data)/1024)
+                file.write(data)
+        pbar.close()
+        if total_size != 0 and pbar.n != total_size/1024:
+            raise AssertionError("Some content was present but not downloaded/written")
 
 
 def read_libsvm_msrank(file_obj, n_samples, n_features, dtype):