fix: "\u characters in JSON strings are currently not supported"

mach_nix/nix/PYPI_DEPS_DB.json

@@ -1,7 +1,7 @@
 {
     "url": "https://github.com/davhau/pypi-deps-db",
-    "rev": "491dcc6953c8c84919dd7ceb040d1cd601010849",
-    "date": "2020-10-19T08:49:36+00:00",
-    "sha256": "04q4xxckzg2zhnmviymbgv3lnyi6m333m6qrg2n35vdfxpq79dmx",
+    "rev": "b0971936e8f3b27a49345088d0739b154284fd7b",
+    "date": "2020-10-24T08:14:50+00:00",
+    "sha256": "1683nzdzka97p76rz498crdfik64cgh0a9ahbz716jsqkfb6xlz2",
     "fetchSubmodules": false
 }

pypi-crawlers/src/crawl_urls.py

@@ -1,8 +1,10 @@
+import base64
 import os
 import sys
 import traceback
 import xmlrpc.client
 from time import sleep
+from urllib.parse import quote
 
 import requests
 import utils
@@ -17,6 +19,14 @@ if not email:
 headers = {'User-Agent': f'Pypi Daily Sync (Contact: {email})'}
 
 
+def urlencode(s):
+    return quote(s, safe='/ ')
+
+
+def hash_hex_to_base64(hash):
+    return base64.b64encode(bytes.fromhex(hash)).decode()
+
+
 def all_packages():
     xmlclient = xmlrpc.client.ServerProxy(base_url)
     return xmlclient.list_packages_with_serial()
@@ -69,12 +79,14 @@ def save_pkg_meta(name, pkgs_dict):
         releases_dict[release_ver] = {}
         if sdist:
             releases_dict[release_ver]['sdist'] = [
-                sdist['digests']['sha256'],
-                sdist['filename'],
+                hash_hex_to_base64(sdist['digests']['sha256']),
+                urlencode(sdist['filename']),
             ]
         if wheels:
             releases_dict[release_ver]['wheels'] = {
-                wheel['filename']: (wheel['digests']['sha256'], wheel['python_version'])
+                wheel['filename']: (
+                    hash_hex_to_base64(wheel['digests']['sha256']),
+                    wheel['python_version'])
                 for wheel in wheels
             }
     if releases_dict: