diff --git a/mach_nix/nix/PYPI_DEPS_DB.json b/mach_nix/nix/PYPI_DEPS_DB.json index 16acd14..f89823d 100644 --- a/mach_nix/nix/PYPI_DEPS_DB.json +++ b/mach_nix/nix/PYPI_DEPS_DB.json @@ -1,7 +1,7 @@ { "url": "https://github.com/davhau/pypi-deps-db", - "rev": "491dcc6953c8c84919dd7ceb040d1cd601010849", - "date": "2020-10-19T08:49:36+00:00", - "sha256": "04q4xxckzg2zhnmviymbgv3lnyi6m333m6qrg2n35vdfxpq79dmx", + "rev": "b0971936e8f3b27a49345088d0739b154284fd7b", + "date": "2020-10-24T08:14:50+00:00", + "sha256": "1683nzdzka97p76rz498crdfik64cgh0a9ahbz716jsqkfb6xlz2", "fetchSubmodules": false } diff --git a/pypi-crawlers/src/crawl_urls.py b/pypi-crawlers/src/crawl_urls.py index 9cdee47..416145a 100644 --- a/pypi-crawlers/src/crawl_urls.py +++ b/pypi-crawlers/src/crawl_urls.py @@ -1,8 +1,10 @@ +import base64 import os import sys import traceback import xmlrpc.client from time import sleep +from urllib.parse import quote import requests import utils @@ -17,6 +19,14 @@ if not email: headers = {'User-Agent': f'Pypi Daily Sync (Contact: {email})'} +def urlencode(s): + return quote(s, safe='/ ') + + +def hash_hex_to_base64(hash): + return base64.b64encode(bytes.fromhex(hash)).decode() + + def all_packages(): xmlclient = xmlrpc.client.ServerProxy(base_url) return xmlclient.list_packages_with_serial() @@ -69,12 +79,14 @@ def save_pkg_meta(name, pkgs_dict): releases_dict[release_ver] = {} if sdist: releases_dict[release_ver]['sdist'] = [ - sdist['digests']['sha256'], - sdist['filename'], + hash_hex_to_base64(sdist['digests']['sha256']), + urlencode(sdist['filename']), ] if wheels: releases_dict[release_ver]['wheels'] = { - wheel['filename']: (wheel['digests']['sha256'], wheel['python_version']) + wheel['filename']: ( + hash_hex_to_base64(wheel['digests']['sha256']), + wheel['python_version']) for wheel in wheels } if releases_dict: