From ea85a3c7605aab81816c2e89479e440e7d8e85de Mon Sep 17 00:00:00 2001 From: Scott Bailey Date: Tue, 10 Dec 2019 12:24:53 -0500 Subject: [PATCH 1/5] python3 support Add some codec twiddling to explicitly convert string-oriented json output to byte-oriented stream input. I believe this remains backwards-compatible with python 2.7. --- stick/repository.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/stick/repository.py b/stick/repository.py index ea056aa..0fdcafb 100644 --- a/stick/repository.py +++ b/stick/repository.py @@ -1,3 +1,4 @@ +import codecs import io import json import logging @@ -176,7 +177,9 @@ def _put_manifest(self, safe_name, project): json_key = '{0}{1}/manifest.json'.format(self.prefix, safe_name) logger.info('Uploading {0}'.format(json_key)) with io.BytesIO() as data: - json.dump(project.get_manifest(), data) + writer = codecs.getwriter("utf-8") + str_data = writer(data) + json.dump(project.get_manifest(), str_data) data.seek(0, 0) return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') @@ -186,7 +189,9 @@ def _put_json(self, safe_name, project, version=None): json_key = '{0}{1}{2}/json'.format(self.prefix, safe_name, version_prefix) logger.info('Uploading {0}'.format(json_key)) with io.BytesIO() as data: - json.dump(project.get_metadata(version), data) + writer = codecs.getwriter("utf-8") + str_data = writer(data) + json.dump(project.get_metadata(version), str_data) data.seek(0, 0) return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') From 43c2be32588991f7dc244f95a0835626e32c9840 Mon Sep 17 00:00:00 2001 From: Scott Bailey Date: Tue, 10 Dec 2019 13:15:26 -0500 Subject: [PATCH 2/5] CloudFront support, if not by name Add an optional `--url` flag; when present, the supplied URL is used in place of the S3 bucket-based URL. It should be in the form "https://myhost.mydomain" (without a trailing slash). --- stick/commands.py | 3 +++ stick/repository.py | 8 ++++++-- stick/settings.py | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/stick/commands.py b/stick/commands.py index 19bfab8..822cbe7 100644 --- a/stick/commands.py +++ b/stick/commands.py @@ -31,6 +31,7 @@ def cli(): @cli.command(context_settings={'max_content_width': 120, 'ignore_unknown_options': True}) @click.option('--bucket', help='S3 Bucket hosting the repository.', required=True) +@click.option('--url', help='Override the S3-based repository URL.', default=None) @click.option('--prefix', help='Prefix within the S3 Bucket that repository objects are stored.', default='simple', show_default=True) @click.option('--profile', help='Use a specific profile from your credential file to access S3.', default=None) @click.option('--skip-existing/--no-skip-existing', help='Skip uploading file if it already exists.', default=True, show_default=True) @@ -71,6 +72,7 @@ def upload(dist, **kwargs): @cli.command(context_settings={'max_content_width': 120}) @click.option('--bucket', help='S3 Bucket hosting the repository.', required=True) +@click.option('--url', help='Override the S3-based repository URL.', default=None) @click.option('--prefix', help='Prefix within the S3 Bucket that repository objects are stored.', default='simple', show_default=True) @click.option('--profile', help='Use a specific profile from your credential file to access S3.', default=None) @click.option('--project', help='Reindex a specific project. May be specified multiple times. [default: all projects]', default=None, multiple=True) @@ -87,6 +89,7 @@ def reindex(project, **kwargs): @cli.command(context_settings={'max_content_width': 120}) @click.option('--bucket', help='S3 Bucket hosting the repository.', required=True) +@click.option('--url', help='Override the S3-based repository URL.', default=None) @click.option('--prefix', help='Prefix within the S3 Bucket that repository objects are stored.', default='simple', show_default=True) @click.option('--profile', help='Use a specific profile from your credential file to access S3.', default=None) @click.option('--project', help='Check a specific project. May be specified multiple times. [default: all projects]', default=None, multiple=True) diff --git a/stick/repository.py b/stick/repository.py index 0fdcafb..82ef9c7 100644 --- a/stick/repository.py +++ b/stick/repository.py @@ -19,8 +19,9 @@ class Repository(object): - def __init__(self, bucket, prefix, profile): + def __init__(self, bucket, url, prefix, profile): self.bucket = bucket + self.url = url self.prefix = prefix self.client = boto3.Session(profile_name=profile).client('s3', config=client_config) self._project_cache = {} @@ -29,7 +30,10 @@ def __init__(self, bucket, prefix, profile): self.prefix += '/' def get_url(self): - url = 'https://{0}.s3.amazonaws.com/{1}'.format(self.bucket, self.prefix) + if self.url is None: + url = 'https://{0}.s3.amazonaws.com/{1}'.format(self.bucket, self.prefix) + else: + url = '{0}/{1}'.format(self.url, self.prefix) return url def reindex(self, projects): diff --git a/stick/settings.py b/stick/settings.py index 9f8a228..f06270b 100644 --- a/stick/settings.py +++ b/stick/settings.py @@ -2,8 +2,9 @@ class Settings(object): - def __init__(self, bucket, prefix, profile=None, skip_existing=True, sign=False, sign_with='gpg', identity=None): + def __init__(self, bucket, url, prefix, profile=None, skip_existing=True, sign=False, sign_with='gpg', identity=None): self.bucket = bucket + self.url = url self.prefix = prefix self.profile = profile self.skip_existing = skip_existing @@ -12,5 +13,5 @@ def __init__(self, bucket, prefix, profile=None, skip_existing=True, sign=False, self.identity = identity def create_repository(self): - repo = Repository(self.bucket, self.prefix, self.profile) + repo = Repository(self.bucket, self.url, self.prefix, self.profile) return repo From 0c4f1f6a326fe663551f9344e53727d015a2a47e Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 10 Dec 2019 21:27:26 +0000 Subject: [PATCH 3/5] Add TextIOWrapper for json dump to fix Py3 compat --- stick/repository.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stick/repository.py b/stick/repository.py index ea056aa..944634a 100644 --- a/stick/repository.py +++ b/stick/repository.py @@ -169,14 +169,14 @@ def _get_manifest(self, safe_name): with io.BytesIO() as data: self.client.download_fileobj(Fileobj=data, Bucket=self.bucket, Key=json_key) data.seek(0, 0) - return json.load(io.TextIOWrapper(data)) + return json.load(io.TextIOWrapper(data, encoding='utf-8')) def _put_manifest(self, safe_name, project): """Dump and upload the project manifest JSON""" json_key = '{0}{1}/manifest.json'.format(self.prefix, safe_name) logger.info('Uploading {0}'.format(json_key)) with io.BytesIO() as data: - json.dump(project.get_manifest(), data) + json.dump(project.get_manifest(), io.TextIOWrapper(data, encoding='utf-8')) data.seek(0, 0) return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') @@ -186,7 +186,7 @@ def _put_json(self, safe_name, project, version=None): json_key = '{0}{1}{2}/json'.format(self.prefix, safe_name, version_prefix) logger.info('Uploading {0}'.format(json_key)) with io.BytesIO() as data: - json.dump(project.get_metadata(version), data) + json.dump(project.get_metadata(version), io.TextIOWrapper(data, encoding='utf-8')) data.seek(0, 0) return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') From 18af1c0edfc8e531de64de41046514bca71d3e4d Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 10 Dec 2019 21:53:19 +0000 Subject: [PATCH 4/5] Fix TextIOWrapper closing buffer on __del__ --- .flake8 | 2 ++ stick/repository.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..1fd4893 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 160 diff --git a/stick/repository.py b/stick/repository.py index 944634a..cefd371 100644 --- a/stick/repository.py +++ b/stick/repository.py @@ -176,9 +176,11 @@ def _put_manifest(self, safe_name, project): json_key = '{0}{1}/manifest.json'.format(self.prefix, safe_name) logger.info('Uploading {0}'.format(json_key)) with io.BytesIO() as data: - json.dump(project.get_manifest(), io.TextIOWrapper(data, encoding='utf-8')) - data.seek(0, 0) - return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') + with io.TextIOWrapper(data, encoding='utf-8') as text: + json.dump(project.get_manifest(), text) + text.flush() + data.seek(0, 0) + return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') def _put_json(self, safe_name, project, version=None): """Regenerate and upload the project or release-level index JSON""" @@ -186,9 +188,11 @@ def _put_json(self, safe_name, project, version=None): json_key = '{0}{1}{2}/json'.format(self.prefix, safe_name, version_prefix) logger.info('Uploading {0}'.format(json_key)) with io.BytesIO() as data: - json.dump(project.get_metadata(version), io.TextIOWrapper(data, encoding='utf-8')) - data.seek(0, 0) - return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') + with io.TextIOWrapper(data, encoding='utf-8') as text: + json.dump(project.get_metadata(version), text) + text.flush() + data.seek(0, 0) + return self.client.put_object(Body=data, Bucket=self.bucket, Key=json_key, ContentType='application/json; charset=utf-8') def _put_index(self, safe_name, project): """Regenerate and upload the project-level index HTML""" From cf378f556a29f4ba81f9c84eb6d41775f6ee980a Mon Sep 17 00:00:00 2001 From: Scott Bailey Date: Tue, 10 Dec 2019 13:15:26 -0500 Subject: [PATCH 5/5] CloudFront support, if not by name Add an optional `--url` flag; when present, the supplied URL is used in place of the S3 bucket-based URL. It should be in the form "https://myhost.mydomain" (without a trailing slash). --- stick/commands.py | 3 +++ stick/repository.py | 8 ++++++-- stick/settings.py | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/stick/commands.py b/stick/commands.py index 19bfab8..822cbe7 100644 --- a/stick/commands.py +++ b/stick/commands.py @@ -31,6 +31,7 @@ def cli(): @cli.command(context_settings={'max_content_width': 120, 'ignore_unknown_options': True}) @click.option('--bucket', help='S3 Bucket hosting the repository.', required=True) +@click.option('--url', help='Override the S3-based repository URL.', default=None) @click.option('--prefix', help='Prefix within the S3 Bucket that repository objects are stored.', default='simple', show_default=True) @click.option('--profile', help='Use a specific profile from your credential file to access S3.', default=None) @click.option('--skip-existing/--no-skip-existing', help='Skip uploading file if it already exists.', default=True, show_default=True) @@ -71,6 +72,7 @@ def upload(dist, **kwargs): @cli.command(context_settings={'max_content_width': 120}) @click.option('--bucket', help='S3 Bucket hosting the repository.', required=True) +@click.option('--url', help='Override the S3-based repository URL.', default=None) @click.option('--prefix', help='Prefix within the S3 Bucket that repository objects are stored.', default='simple', show_default=True) @click.option('--profile', help='Use a specific profile from your credential file to access S3.', default=None) @click.option('--project', help='Reindex a specific project. May be specified multiple times. [default: all projects]', default=None, multiple=True) @@ -87,6 +89,7 @@ def reindex(project, **kwargs): @cli.command(context_settings={'max_content_width': 120}) @click.option('--bucket', help='S3 Bucket hosting the repository.', required=True) +@click.option('--url', help='Override the S3-based repository URL.', default=None) @click.option('--prefix', help='Prefix within the S3 Bucket that repository objects are stored.', default='simple', show_default=True) @click.option('--profile', help='Use a specific profile from your credential file to access S3.', default=None) @click.option('--project', help='Check a specific project. May be specified multiple times. [default: all projects]', default=None, multiple=True) diff --git a/stick/repository.py b/stick/repository.py index cefd371..f3ef0bc 100644 --- a/stick/repository.py +++ b/stick/repository.py @@ -18,8 +18,9 @@ class Repository(object): - def __init__(self, bucket, prefix, profile): + def __init__(self, bucket, url, prefix, profile): self.bucket = bucket + self.url = url self.prefix = prefix self.client = boto3.Session(profile_name=profile).client('s3', config=client_config) self._project_cache = {} @@ -28,7 +29,10 @@ def __init__(self, bucket, prefix, profile): self.prefix += '/' def get_url(self): - url = 'https://{0}.s3.amazonaws.com/{1}'.format(self.bucket, self.prefix) + if self.url is None: + url = 'https://{0}.s3.amazonaws.com/{1}'.format(self.bucket, self.prefix) + else: + url = '{0}/{1}'.format(self.url, self.prefix) return url def reindex(self, projects): diff --git a/stick/settings.py b/stick/settings.py index 9f8a228..f06270b 100644 --- a/stick/settings.py +++ b/stick/settings.py @@ -2,8 +2,9 @@ class Settings(object): - def __init__(self, bucket, prefix, profile=None, skip_existing=True, sign=False, sign_with='gpg', identity=None): + def __init__(self, bucket, url, prefix, profile=None, skip_existing=True, sign=False, sign_with='gpg', identity=None): self.bucket = bucket + self.url = url self.prefix = prefix self.profile = profile self.skip_existing = skip_existing @@ -12,5 +13,5 @@ def __init__(self, bucket, prefix, profile=None, skip_existing=True, sign=False, self.identity = identity def create_repository(self): - repo = Repository(self.bucket, self.prefix, self.profile) + repo = Repository(self.bucket, self.url, self.prefix, self.profile) return repo