[neon/neon/livecd-rootfs/Neon/release] /: magic-proxy: replace http.client with urllib calls
Thomas Bechtold
null at kde.org
Thu Oct 28 12:37:16 BST 2021
Git commit efd0641f6c5f8ed2244f02243a51ab3081cd6378 by Thomas Bechtold, on behalf of Dimitri John Ledkov.
Committed on 28/09/2021 at 11:48.
Pushed by jriddell into branch 'Neon/release'.
magic-proxy: replace http.client with urllib calls
Initialize passwords from sources.list.
Use urllib everywhere.
This way authentication is added to all the required requests.
And incoming headers, are passed to the outgoing requests.
And all the response headers, are passed to the original client.
And all the TCP & HTTP errors are passed back to the client.
Thus should avoiding hanging requests upon failure.
Also rewrite the URI when requesting things.
This allows to use private-ppa.buildd outside of launchpad.
Signed-off-by: Dimitri John Ledkov <xnox at ubuntu.com>
(cherry picked from commit dc2a472871907bbed3ab89d2a46d924ece80d514)
M +1 -0 live-build/auto/build
M +117 -50 magic-proxy
https://invent.kde.org/neon/neon/livecd-rootfs/commit/efd0641f6c5f8ed2244f02243a51ab3081cd6378
diff --git a/live-build/auto/build b/live-build/auto/build
index d44b1a8a..0ed25a59 100755
--- a/live-build/auto/build
+++ b/live-build/auto/build
@@ -68,6 +68,7 @@ if [ -n "$REPO_SNAPSHOT_STAMP" ]; then
-m owner ! --uid-owner daemon -j REDIRECT --to 8080
# Run proxy as "daemon" to avoid infinite loop.
+ LB_PARENT_MIRROR_BOOTSTRAP=$LB_PARENT_MIRROR_BOOTSTRAP \
/usr/share/livecd-rootfs/magic-proxy \
--address="127.0.0.1" \
--port=8080 \
diff --git a/magic-proxy b/magic-proxy
index e2d0c28d..29d95ab4 100755
--- a/magic-proxy
+++ b/magic-proxy
@@ -68,6 +68,45 @@ class LPInReleaseCacheError(LPInReleaseBaseError):
class LPInReleaseProxyError(LPInReleaseBaseError):
pass
+IN_LP = "http://ftpmaster.internal/ubuntu" in os.environ.get("LB_PARENT_MIRROR_BOOTSTRAP", "")
+
+# We cannot proxy & rewrite https requests Thus apt will talk to us
+# over http But we must upgrade to https for private-ppas, outside of
+# launchpad hence use this helper to re-write urls.
+def get_uri(host, path):
+ if host in ("private-ppa.launchpad.net", "private-ppa.buildd"):
+ if IN_LP:
+ return "http://private-ppa.buildd" + path
+ else:
+ return "https://private-ppa.launchpad.net" + path
+ # TODO add split mirror handling for ftpmaster.internal =>
+ # (ports|archive).ubuntu.com
+ return "http://" + host + path
+
+def initialize_auth():
+ auth_handler = urllib.request.HTTPBasicAuthHandler()
+ with open('/etc/apt/sources.list') as f:
+ for line in f.readlines():
+ for word in line.split():
+ if not word.startswith('http'):
+ continue
+ parse=urllib.parse.urlparse(word)
+ if not parse.username:
+ continue
+ if parse.hostname not in ("private-ppa.launchpad.net", "private-ppa.buildd"):
+ continue
+ auth_handler.add_password(
+ "Token Required", "https://private-ppa.launchpad.net" + parse.path,
+ parse.username, parse.password)
+ auth_handler.add_password(
+ "Token Required", "http://private-ppa.buildd" + parse.path,
+ parse.username, parse.password)
+ print("add password for", parse.path)
+ opener = urllib.request.build_opener(auth_handler)
+ urllib.request.install_opener(opener)
+
+initialize_auth()
+
class InRelease:
"""This class represents an InRelease file."""
@@ -97,7 +136,8 @@ class InRelease:
this is set explicitly to correspond to the Last-Modified header spat
out by the Web server.
"""
- self.mirror = mirror
+ parsed = urllib.parse.urlparse(mirror)
+ self.mirror = get_uri(parsed.hostname, parsed.path)
self.suite = suite
self.data = data
self.dict = {}
@@ -363,7 +403,7 @@ class LPInReleaseCache:
suite."""
with self._lock:
url_obj = urllib.parse.urlparse(mirror)
- address = url_obj.hostname + url_obj.path.rstrip("/")
+ address = url_obj.scheme + url_obj.hostname + url_obj.path.rstrip("/")
inrel_by_hash = self._data\
.get(address, {})\
@@ -403,7 +443,8 @@ class LPInReleaseIndex:
which case all look-ups will first go to the cache and only cache
misses will result in requests to the Web server.
"""
- self._mirror = mirror
+ parsed = urllib.parse.urlparse(mirror)
+ self._mirror = get_uri(parsed.hostname, parsed.path)
self._suite = suite
self._cache = cache
@@ -528,7 +569,8 @@ class LPInReleaseIndex:
return [inrel.hash for inrel in cache_entry]
try:
- with urllib.request.urlopen(self._base_url) as response:
+ request=urllib.request.Request(self._base_url)
+ with urllib.request.urlopen(request) as response:
content_encoding = self._guess_content_encoding_for_response(
response)
@@ -744,6 +786,23 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
"""Process a GET request."""
self.__get_request()
+ def sanitize_requestline(self):
+ requestline = []
+ for word in self.requestline.split():
+ if word.startswith('http'):
+ parse = urllib.parse.urlparse(word)
+ parse = urllib.parse.ParseResult(
+ parse.scheme,
+ parse.hostname, # not netloc, to sanitize username/password
+ parse.path,
+ parse.params,
+ parse.query,
+ parse.fragment)
+ requestline.append(urllib.parse.urlunparse(parse))
+ else:
+ requestline.append(word)
+ self.requestline = ' '.join(requestline)
+
def __get_request(self, verb="GET"):
"""Pass all requests on to the destination server 1:1 except when the
target is an InRelease file or a resource listed in an InRelease files.
@@ -756,15 +815,18 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
happening here, the client does not know that what it receives is not
exactly what it requested."""
- host, path = self.__get_host_path()
+ uri = self.headers.get("host") + self.path
+ parsed = urllib.parse.urlparse(uri)
+
+ self.sanitize_requestline()
m = re.match(
r"^(?P<base>.*?)/dists/(?P<suite>[^/]+)/(?P<target>.*)$",
- path
+ parsed.path
)
if m:
- mirror = "http://" + host + m.group("base")
+ mirror = get_uri(parsed.hostname, m.group("base"))
base = m.group("base")
suite = m.group("suite")
target = m.group("target")
@@ -775,50 +837,49 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
self.server.snapshot_stamp)
if inrelease is None:
- self.__send_error(404, "No InRelease file found for given "
- "mirror, suite and timestamp.")
- return
-
- if target == "InRelease":
- # If target is InRelease, send back contents directly.
- data = inrelease.data.encode("utf-8")
-
self.log_message(
- "Inject InRelease '{}'".format(inrelease.hash))
-
- self.send_response(200)
- self.send_header("Content-Length", len(data))
- self.end_headers()
+ "InRelease not found for {}/{}".format(parsed.hostname, parsed.path))
+ self.send_error(404, "No InRelease file found for given "
+ "mirror, suite and timestamp.")
+ return
- if verb == "GET":
- self.wfile.write(data)
+ hash_ = None
- return
+ if target == "InRelease":
+ hash_ = inrelease.hash
else:
- # If target hash is listed, then redirect to by-hash URL.
hash_ = inrelease.get_hash_for(target)
- if hash_:
- self.log_message(
- "Inject {} for {}".format(hash_, target))
+ if hash_:
+ self.log_message(
+ "Inject {} for {}".format(hash_, target))
- target_path = target.rsplit("/", 1)[0]
+ target_path = target.rsplit("/", 1)[0]
- path = "{}/dists/{}/{}/by-hash/SHA256/{}"\
- .format(base, suite, target_path, hash_)
+ uri = "{}/dists/{}/by-hash/SHA256/{}"\
+ .format(mirror, suite, hash_)
+ else:
+ uri = get_uri(parsed.hostname, parsed.path)
+ ## use requests such that authentication via password database happens
+ ## reuse all the headers that we got asked to provide
try:
- client = http.client.HTTPConnection(host)
- client.request(verb, path)
- except Exception as e:
- self.log_error("Failed to retrieve http://{}{}: {}"
- .format(host, path, str(e)))
- return
+ with urllib.request.urlopen(
+ urllib.request.Request(
+ uri,
+ method=verb,
+ headers=self.headers)) as response:
+ self.__send_response(response)
+ except urllib.error.HTTPError as e:
+ if e.code not in (304,):
+ self.log_message(
+ "urlopen() failed for {} with {}".format(uri, e.reason))
+ self.__send_response(e)
+ except urllib.error.URLError as e:
+ self.log_message(
+ "urlopen() failed for {} with {}".format(uri, e.reason))
+ self.send_error(501, e.reason)
- try:
- self.__send_response(client.getresponse())
- except Exception as e:
- self.log_error("Error delivering response: {}".format(str(e)))
def __get_host_path(self):
"""Figure out the host to contact and the path of the resource that is
@@ -831,20 +892,26 @@ class ProxyingHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
def __send_response(self, response):
"""Pass on upstream response headers and body to the client."""
- self.send_response(response.status)
+ if hasattr(response, "status"):
+ status = response.status
+ elif hassattr(response, "code"):
+ status = response.code
+ elif hasattr(response, "getstatus"):
+ status = response.getstatus()
+
+ if hasattr(response, "headers"):
+ headers = response.headers
+ elif hasattr(response, "info"):
+ headers = response.info()
- for name, value in response.getheaders():
- self.send_header(name, value)
+ self.send_response(status)
- self.end_headers()
- shutil.copyfileobj(response, self.wfile)
+ for name, value in headers.items():
+ self.send_header(name, value)
- def __send_error(self, status, message):
- """Return an HTTP error status and a message in the response body."""
- self.send_response(status)
- self.send_header("Content-Type", "text/plain; charset=utf-8")
self.end_headers()
- self.wfile.write(message.encode("utf-8"))
+ if hasattr(response, "read"):
+ shutil.copyfileobj(response, self.wfile)
class MagicHTTPProxy(socketserver.ThreadingMixIn, http.server.HTTPServer):
More information about the Neon-commits
mailing list