pax_global_header 0000666 0000000 0000000 00000000064 15213143633 0014513 g ustar 00root root 0000000 0000000 52 comment=cd8422326f42bf609eababfbea1ce05bbeabeb27
awscurl-0.44/ 0000775 0000000 0000000 00000000000 15213143633 0013122 5 ustar 00root root 0000000 0000000 awscurl-0.44/.dockerignore 0000664 0000000 0000000 00000000014 15213143633 0015571 0 ustar 00root root 0000000 0000000 build/
venv/ awscurl-0.44/.github/ 0000775 0000000 0000000 00000000000 15213143633 0014462 5 ustar 00root root 0000000 0000000 awscurl-0.44/.github/CODEOWNERS 0000664 0000000 0000000 00000000050 15213143633 0016050 0 ustar 00root root 0000000 0000000 # Default owner for all files
* @okigan
awscurl-0.44/.github/FUNDING.yml 0000664 0000000 0000000 00000001317 15213143633 0016301 0 ustar 00root root 0000000 0000000 # These are supported funding model platforms
github: [okigan]# Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
awscurl-0.44/.github/PULL_REQUEST_TEMPLATE.md 0000664 0000000 0000000 00000000413 15213143633 0020261 0 ustar 00root root 0000000 0000000 ## What
## Why
## Testing
- [ ] `pycodestyle awscurl` passes
- [ ] `pytest -v --cov=awscurl --cov-fail-under=77` passes
- [ ] No AWS credentials exposed in code or logs
awscurl-0.44/.github/copilot-instructions.md 0000664 0000000 0000000 00000000167 15213143633 0021223 0 ustar 00root root 0000000 0000000 # Copilot Instructions
See [AGENTS.md](../AGENTS.md) for project context, review priorities, and build/test commands.
awscurl-0.44/.github/dependabot.yml 0000664 0000000 0000000 00000001062 15213143633 0017311 0 ustar 00root root 0000000 0000000 # To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "pip"
directory: "/" # Location of package manifests
schedule:
interval: "monthly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
awscurl-0.44/.github/workflows/ 0000775 0000000 0000000 00000000000 15213143633 0016517 5 ustar 00root root 0000000 0000000 awscurl-0.44/.github/workflows/codeql-analysis.yml 0000664 0000000 0000000 00000004227 15213143633 0022337 0 ustar 00root root 0000000 0000000 # For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
name: "CodeQL"
on:
push:
branches: [master]
pull_request:
# The branches below must be a subset of the branches above
branches: [master]
schedule:
- cron: '0 9 * * 6'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
# Override automatic language detection by changing the below list
# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
language: ['python']
# Learn more...
# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
steps:
- name: Checkout repository
uses: actions/checkout@v6
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v4
# โน๏ธ Command-line programs to run using the OS shell.
# ๐ https://git.io/JvXDl
# โ๏ธ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
awscurl-0.44/.github/workflows/dockerhubpublish.yml 0000664 0000000 0000000 00000002472 15213143633 0022604 0 ustar 00root root 0000000 0000000 name: Publish Docker images
on:
release:
types: [published]
workflow_dispatch:
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v6
- name: Log in to Docker Hub
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v6
with:
images: |
okigan/awscurl
ghcr.io/${{ github.repository }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Build and push Docker image
uses: docker/build-push-action@v7
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} awscurl-0.44/.github/workflows/pythonapp.yml 0000664 0000000 0000000 00000003145 15213143633 0021267 0 ustar 00root root 0000000 0000000 name: CI
on: [push,pull_request,workflow_dispatch]
jobs:
build:
strategy:
matrix:
runs-on: [ubuntu-22.04, ubuntu-24.04, ubuntu-latest, macOS-latest]
python-version: ['3.10', '3.11', '3.12', '3.13']
fail-fast: false
runs-on: ${{ matrix.runs-on }}
env:
AWS_ACCESS_KEY_ID: MOCK_AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: MOCK_AWS_SECRET_ACCESS_KEY
AWS_SESSION_TOKEN: MOCK_AWS_SESSION_TOKEN
steps:
- uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install -r requirements-test.txt
pip install pytest
# - name: Lint with flake8
# run: |
# pip install flake8
# # stop the build if there are Python syntax errors or undefined names
# flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
# flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Check with pycodestyle
run: |
pycodestyle -v awscurl
- name: Check with mypy
run: |
mypy awscurl/ tests/
- name: Test with pytest
run: |
python --version
pytest -v --cov=awscurl --cov-fail-under=77 --cov-report html
awscurl-0.44/.github/workflows/pythonpublish.yml 0000664 0000000 0000000 00000001200 15213143633 0022143 0 ustar 00root root 0000000 0000000 name: Upload Python Package
on:
release:
types: [created]
workflow_dispatch:
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.x'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
- name: Build and publish
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*
awscurl-0.44/.gitignore 0000664 0000000 0000000 00000000175 15213143633 0015115 0 ustar 00root root 0000000 0000000 *.pyc
/build/
/dist/
/*.egg-info
/venv/
/.pytest_cache/v/cache/
/.coverage
/.tox/
**/*,cover
/htmlcov/
/.cache/pip/
/.local/
awscurl-0.44/.python-version 0000664 0000000 0000000 00000000037 15213143633 0016127 0 ustar 00root root 0000000 0000000 3.10.16
3.11.12
3.12.10
3.13.3
awscurl-0.44/.vscode/ 0000775 0000000 0000000 00000000000 15213143633 0014463 5 ustar 00root root 0000000 0000000 awscurl-0.44/.vscode/launch.json 0000664 0000000 0000000 00000001251 15213143633 0016627 0 ustar 00root root 0000000 0000000 {
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "awscurl",
"type": "python",
"request": "launch",
"module": "awscurl",
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"PYTHONPATH": "${workspaceFolder}/awscurl"
},
"args": ["--service","s3", "https://awscurl-sample-bucket.s3.amazonaws.com?q=a{status=b}" ]
}
]
} awscurl-0.44/.vscode/settings.json 0000664 0000000 0000000 00000000177 15213143633 0017223 0 ustar 00root root 0000000 0000000 {
"python.testing.pytestArgs": [],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true
}
awscurl-0.44/AGENTS.md 0000664 0000000 0000000 00000002336 15213143633 0014431 0 ustar 00root root 0000000 0000000 # Agents
## Project
This is `awscurl` โ a curl-like CLI tool that signs requests with AWS Signature Version 4.
## Key Files to Reference
- `setup.cfg` โ code style rules (pycodestyle config)
- `setup.py` โ package metadata, dependencies, supported Python versions
- `.github/workflows/pythonapp.yml` โ CI checks that must pass (lint + test matrix)
- `requirements-test.txt` โ test tooling
- `scripts/ci.sh` โ local CI flow
## Build & Test
See `scripts/ci.sh` for the local CI flow and `.github/workflows/pythonapp.yml` for the CI pipeline.
## Review Priorities
- Changes to request signing (`task_1` through `task_4`, `__normalize_query_string`, `aws_url_encode`) must conform to the [AWS SigV4 spec](https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html)
- Never log or expose AWS credentials (access keys, secret keys, tokens) โ check `__log`, error paths, and `load_aws_config`
- Maintain backward compatibility of CLI arguments and behavior โ see `inner_main` for arg parsing
- CLI arguments should follow curl's style and naming conventions
- Keep dependencies minimal โ check `setup.py` for the current set
- New functionality must include unit tests โ see `tests/` for existing patterns
awscurl-0.44/DEVELOP.md 0000664 0000000 0000000 00000000512 15213143633 0014540 0 ustar 00root root 0000000 0000000 ## Some useful commands for local development
### Build docker image
```sh
docker build -t awscurl .
docker run --rm -ti -v "$HOME/.aws:/root/.aws" -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SECURITY_TOKEN -e AWS_PROFILE -e AWS_REGION awscurl "${api_url_base}/api/rxxxxx"
docker run -it --entrypoint sh awscurl
```
awscurl-0.44/Dockerfile 0000664 0000000 0000000 00000000577 15213143633 0015125 0 ustar 00root root 0000000 0000000 # Build stage
FROM python:3-alpine AS builder
RUN set -ex && \
apk add \
build-base \
libffi-dev \
libxml2-dev \
openssl-dev
RUN pip install --user botocore
COPY . /app-source-dir
RUN pip install -v --user /app-source-dir
# Runtime stage
FROM python:3-alpine
COPY --from=builder /root/.local /root/.local
ENV PATH=/root/.local/bin/:${PATH}
ENTRYPOINT ["awscurl"] awscurl-0.44/LICENSE 0000664 0000000 0000000 00000002073 15213143633 0014131 0 ustar 00root root 0000000 0000000 MIT License
Copyright 2015 by the contributors to awscurl
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
awscurl-0.44/Makefile 0000664 0000000 0000000 00000000450 15213143633 0014561 0 ustar 00root root 0000000 0000000 venv:
python3 -m venv venv
( \
. ./venv/bin/activate; \
which python; \
pip install --upgrade pip; \
pip install --upgrade setuptools; \
pip install -r requirements.txt -r requirements-test.txt; \
)
docker-build:
docker build -t awscurl .
docker-run:
docker run -it --rm awscurl
awscurl-0.44/README.md 0000664 0000000 0000000 00000016736 15213143633 0014416 0 ustar 00root root 0000000 0000000 # awscurl [](https://github.com/sponsors/okigan) [](https://www.paypal.com/donate/?business=UDN4FL55J34QC&amount=25) [](https://www.buymeacoffee.com/okigan)
[](https://pypi.python.org/pypi/awscurl)
[](https://github.com/okigan/awscurl)
[](https://hub.docker.com/r/okigan/awscurl)
[](https://gitpod.io/#https://github.com/okigan/awscurl)
[](https://vscode.dev/github/okigan/awscurl)
[](https://github.dev/okigan/awscurl)
curl-like tool with AWS Signature Version 4 request signing.
## Features
* performs requests to AWS services with request signing using curl interface
* supports IAM profile credentials
## Overview
Requests to AWS API must be signed (see [Signing AWS API Requests](http://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html))
automates the process of signing and makes requests to AWS as simple as a standard curl command.
## Installation
```sh
pip install awscurl
```
### Installation from source (bleeding edge)
```sh
pip install git+https://github.com/okigan/awscurl
```
### Installation via Homebrew for MacOS
```sh
brew install awscurl
```
#### Running via Docker
```sh
docker pull okigan/awscurl # or via docker pull ghcr.io/okigan/awscurl
```
or via Github docker registry
```sh
docker pull ghcr.io/okigan/awscurl
```
then
```sh
$ docker run --rm -it okigan/awscurl --access_key ACCESS_KEY --secret_key SECRET_KEY --service s3 s3://...
# or allow access to local credentials as following
$ docker run --rm -it -v "$HOME/.aws:/root/.aws" okigan/awscurl --service s3 s3://...
```
To shorten the length of docker commands use the following alias:
```sh
alias awscurl='docker run --rm -ti -v "$HOME/.aws:/root/.aws" -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SECURITY_TOKEN -e AWS_PROFILE okigan/awscurl'
```
This will allow you to run awscurl from within a Docker container as if it was installed on the host system:
```sh
awscurl
```
## Examples
* Call S3: List bucket content
```sh
$ awscurl --service s3 'https://awscurl-sample-bucket.s3.amazonaws.com' | tidy -xml -iq
awscurl-sample-bucket
1000
false
awscurl-sample-file.txt
2017-07-25T21:27:38.000Z
"d41d8cd98f00b204e9800998ecf8427e"
0
STANDARD
```
* Call EC2:
```sh
$ awscurl --service ec2 'https://ec2.amazonaws.com?Action=DescribeRegions&Version=2013-10-15' | tidy -xml -iq
96511ccd-2d6d-4d63-ad9b-6be6f2c9874d
-
eu-north-1
ec2.eu-north-1.amazonaws.com
-
ap-south-1
ec2.ap-south-1.amazonaws.com
```
* Call API Gateway:
```sh
$ awscurl --service execute-api -X POST -d @request.json \
https://.execute-api.us-east-1.amazonaws.com/
```
## Options
```sh
usage: __main__.py [-h] [-v] [-i] [-X REQUEST] [-d DATA] [-H HEADER] [-k] [--fail-with-body] [--data-binary] [--region REGION] [--profile PROFILE] [--service SERVICE]
[--access_key ACCESS_KEY] [--secret_key SECRET_KEY] [--security_token SECURITY_TOKEN] [--session_token SESSION_TOKEN] [-L] [-o ]
uri
Curl AWS request signing
positional arguments:
uri
options:
-h, --help show this help message and exit
-v, --verbose verbose flag (default: False)
-i, --include include headers in the output (default: False)
-X REQUEST, --request REQUEST
Specify request command to use (default: GET)
-d DATA, --data DATA HTTP POST data (default: )
-H HEADER, --header HEADER
HTTP header (default: None)
-k, --insecure Allow insecure server connections when using SSL (default: False)
--fail-with-body Fail on HTTP errors but save the body (default: False)
--data-binary Process HTTP POST data exactly as specified with no extra processing whatsoever. (default: False)
--region REGION AWS region [env var: AWS_DEFAULT_REGION] (default: us-east-1)
--profile PROFILE AWS profile [env var: AWS_PROFILE] (default: default)
--service SERVICE AWS service (default: execute-api)
--access_key ACCESS_KEY
[env var: AWS_ACCESS_KEY_ID] (default: None)
--secret_key SECRET_KEY
[env var: AWS_SECRET_ACCESS_KEY] (default: None)
--security_token SECURITY_TOKEN
[env var: AWS_SECURITY_TOKEN] (default: None)
--session_token SESSION_TOKEN
[env var: AWS_SESSION_TOKEN] (default: None)
-L, --location Follow redirects (default: False)
-o , --output
Write to file instead of stdout (default: )
In general, command-line values override environment variables which override defaults.
```
If you do not specify the `--access_key` or `--secret_key`
(or environment variables), `awscurl` will attempt to use
the credentials you set in `~/.aws/credentials`. If you
do not specify a `--profile` or `AWS_PROFILE`, `awscurl`
uses `default`.
## Who uses awscurl
* [AWS Documentation](https://docs.aws.amazon.com/apigateway/latest/developerguide/apigateway-how-to-call-websocket-api-connections.html)
* [Onica blog](https://onica.com/blog/how-to/how-to-kibana-default-index-pattern/)
* QnA on [StackOverflow](https://stackoverflow.com/search?q=awscurl)
* QnA on [DevOps StackExchange](https://devops.stackexchange.com/search?q=awscurl)
* Examples on [Golfbert](https://golfbert.com/api/samples)
## Star History
[](https://star-history.com/#okigan/awscurl&Date)
## Related projects
* awscurl in Go:
*
*
* awscurl in Lisp:
* awscurl on DockerHub:
* [aws-signature-proxy](https://github.com/sverch/aws-signature-proxy) and related [blog post](https://shaunverch.com/butter/open-source/2019/09/27/butter-days-6.html)
* [aws-sigv4-proxy](https://github.com/awslabs/aws-sigv4-proxy) on awslabs
## Last but not least
* [Sponsor awscurl](https://github.com/sponsors/okigan)
awscurl-0.44/awscurl/ 0000775 0000000 0000000 00000000000 15213143633 0014602 5 ustar 00root root 0000000 0000000 awscurl-0.44/awscurl/__init__.py 0000664 0000000 0000000 00000000000 15213143633 0016701 0 ustar 00root root 0000000 0000000 awscurl-0.44/awscurl/__main__.py 0000664 0000000 0000000 00000000267 15213143633 0016701 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
"""The main entry point. Invoke as `awscurl' or `python -m awscurl'.
"""
import sys
from .awscurl import main
if __name__ == '__main__':
sys.exit(main())
awscurl-0.44/awscurl/awscurl.py 0000775 0000000 0000000 00000064321 15213143633 0016645 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
"""
Awscurl implementation
"""
from __future__ import print_function
from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Tuple, Union
import datetime
import hashlib
import hmac
import os
import pprint
import ssl
import sys
import re
from botocore import crt, awsrequest
from botocore.credentials import Credentials
from typing import Dict
import urllib
from urllib.parse import quote, urljoin
from urllib3.util.ssl_ import create_urllib3_context
import configparser
import configargparse # type: ignore[import-untyped]
import requests # type: ignore[import-untyped]
from requests.adapters import HTTPAdapter # type: ignore[import-untyped]
from requests.structures import CaseInsensitiveDict # type: ignore[import-untyped]
from .utils import sha256_hash, sha256_hash_for_binary_data, sign
__author__ = 'iokulist'
IS_VERBOSE = False
TLS_VERSIONS = {
"1.0": ssl.TLSVersion.TLSv1,
"1.1": ssl.TLSVersion.TLSv1_1,
"1.2": ssl.TLSVersion.TLSv1_2,
"1.3": ssl.TLSVersion.TLSv1_3,
}
def __log(*args, **kwargs):
if not IS_VERBOSE:
return
stderr_pp = pprint.PrettyPrinter(stream=sys.stderr)
stderr_pp.pprint(*args, **kwargs)
def url_path_to_dict(path):
"""http://stackoverflow.com/a/17892757/142207"""
pattern = (r'^'
r'((?P.+?)://)?'
r'((?P[^/]+?)(:(?P[^/]*?))?@)?'
r'(?P.*?)'
r'(:(?P\d+?))?'
r'(?P/.*?)?'
r'(\?(?P.*?))?'
r'$')
regex = re.compile(pattern)
url_match = regex.match(path)
url_dict = url_match.groupdict() if url_match is not None else None
if url_dict['path'] is None:
url_dict['path'] = '/'
if url_dict['query'] is None:
url_dict['query'] = ''
return url_dict
# pylint: disable=too-many-arguments,too-many-locals
def make_request(method: str,
service: str,
region: str,
uri: str,
headers: MutableMapping[str, str],
data: Union[str, bytes],
access_key: str,
secret_key: str,
security_token: str,
data_binary: bool,
verify: bool = True,
allow_redirects: bool = False,
tls_min: Optional[str] = None,
tls_max: Optional[str] = None) -> Any:
"""
# Make HTTP request with AWS Version 4 signing
:return: http request object
:param method: str
:param service: str
:param region: str
:param uri: str
:param headers: dict
:param data: str
:param access_key: str
:param secret_key: str
:param security_token: str
:param data_binary: bool
:param verify: bool
:param allow_redirects: false
:param tls_min: str
:param tls_max: str
See also: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
"""
uri_dict = url_path_to_dict(uri)
host = uri_dict['host']
query = uri_dict['query']
canonical_uri = uri_dict['path']
port = uri_dict['port']
# Create a date for headers and the credential string
current_time = __now()
amzdate = current_time.strftime('%Y%m%dT%H%M%SZ')
datestamp = current_time.strftime('%Y%m%d') # Date w/o time, used in credential scope
if region == "*":
# support sigv4a
auth_headers = get_sigv4a_headers(
service,
region,
method,
uri,
access_key,
secret_key,
security_token)
headers.update(auth_headers)
else:
canonical_request, payload_hash, signed_headers = task_1_create_a_canonical_request(
query,
headers,
port,
host,
amzdate,
method,
data,
security_token,
data_binary,
canonical_uri)
string_to_sign, algorithm, credential_scope = task_2_create_the_string_to_sign(
amzdate,
datestamp,
canonical_request,
service,
region)
signature = task_3_calculate_the_signature(
datestamp,
string_to_sign,
service,
region,
secret_key)
auth_headers = task_4_build_auth_headers_for_the_request(
amzdate,
payload_hash,
algorithm,
credential_scope,
signed_headers,
signature,
access_key,
security_token)
headers.update(auth_headers)
if data_binary:
return __send_request(uri, data, headers, method, verify, allow_redirects, tls_min, tls_max)
else:
encoded_data = data.encode('utf-8') if isinstance(data, str) else data
return __send_request(uri, encoded_data, headers, method, verify, allow_redirects, tls_min, tls_max)
def remove_default_port(parsed_url):
default_ports = {'http': 80, 'https': 443}
if any(parsed_url.scheme == scheme and parsed_url.port == port
for scheme, port in default_ports.items()):
host = parsed_url.hostname
else:
host = parsed_url.netloc
return host
# pylint: disable=too-many-arguments,too-many-locals
def task_1_create_a_canonical_request(
query,
headers: MutableMapping[str, str],
port,
host,
amzdate,
method,
data,
security_token,
data_binary,
canonical_uri):
"""
************* TASK 1: CREATE A CANONICAL REQUEST *************
http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
Step 1 is to define the verb (GET, POST, etc.)--already done.
Step 2: Create canonical URI--the part of the URI from domain to query
string (use '/' if no path)
canonical_uri = '/'
Step 3: Create the canonical query string. In this example (a GET
request),
request parameters are in the query string. Query string values must
be URL-encoded (space=%20). The parameters must be sorted by name.
For this example, the query string is pre-formatted in the
request_parameters variable.
"""
canonical_querystring = __normalize_query_string(query)
__log(canonical_querystring)
# If the host was specified in the HTTP header, ensure that the canonical
# headers are set accordingly
ci_headers = requests.structures.CaseInsensitiveDict(headers)
if 'host' in ci_headers:
fullhost = ci_headers['host']
else:
fullhost = host + ':' + port if port else host
fullhost = remove_default_port(urllib.parse.urlparse('//' + fullhost))
# Step 4: Create payload hash (hash of the request body content). For GET
# requests, the payload is an empty string ("").
# Only use binary hash if data is present AND data_binary flag is set.
payload_hash = sha256_hash_for_binary_data(data) if (data_binary and data) else sha256_hash(data or '')
# Step 5: Create the canonical headers and signed headers. Header names
# and value must be trimmed and lowercase, and sorted in ASCII order.
# Note that there is a trailing \n.
canonical_headers_dict = {'host': fullhost.lower(),
'x-amz-content-sha256': payload_hash,
'x-amz-date': amzdate}
if security_token:
canonical_headers_dict['x-amz-security-token'] = security_token
if 'content-type' in ci_headers:
canonical_headers_dict['content-type'] = ci_headers['content-type'].strip()
# Step 6: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers lists those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
# "content-type" must be included if present in the request.
# Headers starting with "x-amz-" (e.g. x-amz-content-sha256) are also required.
# Step 6.5: Add custom signed headers into the canonical_headers and signed_headers lists.
# Header names must be lowercase, values trimmed, and sorted in ASCII order.
for header, value in sorted(ci_headers.items()):
if header.lower().startswith("x-amz-"):
canonical_headers_dict[header.lower()] = value.strip()
sorted_canonical_headers_items = sorted(canonical_headers_dict.items())
canonical_headers = ''.join(['%s:%s\n' % (k, v) for k, v in sorted_canonical_headers_items])
signed_headers = ';'.join(k for k, v in sorted_canonical_headers_items)
# Step 7: Combine elements to create canonical request
canonical_request = (method + '\n' +
requests.utils.quote(canonical_uri) + '\n' +
canonical_querystring + '\n' +
canonical_headers + '\n' +
signed_headers + '\n' +
payload_hash)
__log('\nCANONICAL REQUEST = ' + canonical_request)
return canonical_request, payload_hash, signed_headers
def task_2_create_the_string_to_sign(
amzdate,
datestamp,
canonical_request,
service,
region):
"""
************* TASK 2: CREATE THE STRING TO SIGN*************
Match the algorithm to the hashing algorithm you use, either SHA-1 or
SHA-256 (recommended)
"""
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = (datestamp + '/' +
region + '/' +
service + '/' +
'aws4_request')
string_to_sign = (algorithm + '\n' +
amzdate + '\n' +
credential_scope + '\n' +
sha256_hash(canonical_request))
__log('\nSTRING_TO_SIGN = ' + string_to_sign)
return string_to_sign, algorithm, credential_scope
def task_3_calculate_the_signature(
datestamp,
string_to_sign,
service,
region,
secret_key):
"""
************* TASK 3: CALCULATE THE SIGNATURE *************
"""
def get_signature_key(key, date_stamp, region_name, service_name):
"""
See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
In AWS Signature Version 4, instead of using your AWS access keys to sign a request, you
first create a signing key that is scoped to a specific region and service. For more
information about signing keys, see Introduction to Signing Requests.
"""
k_date = sign(('AWS4' + key).encode('utf-8'), date_stamp)
k_region = sign(k_date, region_name)
k_service = sign(k_region, service_name)
k_signing = sign(k_service, 'aws4_request')
return k_signing
# Create the signing key using the function defined above.
signing_key = get_signature_key(secret_key, datestamp, region, service)
# Sign the string_to_sign using the signing_key
encoded = string_to_sign.encode('utf-8')
signature = hmac.new(signing_key, encoded, hashlib.sha256).hexdigest()
return signature
def task_4_build_auth_headers_for_the_request(
amzdate,
payload_hash,
algorithm,
credential_scope,
signed_headers,
signature,
access_key,
security_token):
"""
************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST ***********
The signing information can be either in a query string value or in a header
named Authorization. This function shows how to use the header. It returns
a headers dict with all the necessary signing headers.
"""
# Create authorization header and add to request headers
authorization_header = (
algorithm + ' ' +
'Credential=' + access_key + '/' + credential_scope + ', ' +
'SignedHeaders=' + signed_headers + ', ' +
'Signature=' + signature
)
# The request can include any headers, but MUST include "host",
# "x-amz-date", and (for this scenario) "Authorization". "host" and
# "x-amz-date" must be included in the canonical_headers and
# signed_headers, as noted earlier. Order here is not significant.
# Python note: The 'host' header is added automatically by the Python
# 'requests' library.
headers = {
'Authorization': authorization_header,
'x-amz-date': amzdate,
'x-amz-content-sha256': payload_hash
}
if security_token is not None:
headers['x-amz-security-token'] = security_token
return headers
def get_sigv4a_headers(service, region, method, url, access_key, secret_key, security_token):
sig_v4a = crt.auth.CrtS3SigV4AsymAuth(
Credentials(access_key, secret_key, security_token), service, region)
request = awsrequest.AWSRequest(method=method, url=url)
sig_v4a.add_auth(request)
return request.prepare().headers
def __normalize_query_string(query):
parameter_pairs = (list(map(str.strip, s.split("=")))
for s in query.split('&')
if len(s) > 0)
normalized = '&'.join('%s=%s' % (aws_url_encode(p[0]), aws_url_encode(p[1]) if len(p) > 1 else '')
for p in sorted(parameter_pairs))
return normalized
def aws_url_encode(text: str) -> str:
"""
URI-encode each parameter name and value according to the following rules:
- Do not URI-encode any of the unreserved characters that RFC 3986 defines: A-Z, a-z, 0-9, hyphen (-),
underscore (_), period (.), and tilde (~).
- Percent-encode all other characters with %XY, where X and Y are hexadecimal characters (0-9 and uppercase A-F).
For example, the space character must be encoded as %20 (not using '+', as some encoding schemes do) and
extended UTF-8 characters must be in the form %XY%ZA%BC.
- Double-encode any equals (=) characters in parameter values.
"""
return quote(text, safe='~=').replace('=', '==')
def __now():
return datetime.datetime.utcnow()
class _TLSAdapter(HTTPAdapter):
def __init__(self, tls_min=None, tls_max=None, verify=True, **kwargs):
self._tls_min = tls_min
self._tls_max = tls_max
self._verify = verify
super().__init__(**kwargs)
def init_poolmanager(self, *args, **kwargs):
ctx = create_urllib3_context()
ctx.load_default_certs()
tls_min_ver = TLS_VERSIONS.get(self._tls_min)
tls_max_ver = TLS_VERSIONS.get(self._tls_max)
if tls_min_ver is not None:
ctx.minimum_version = tls_min_ver
if tls_max_ver is not None:
ctx.maximum_version = tls_max_ver
if not self._verify:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
kwargs['ssl_context'] = ctx
super().init_poolmanager(*args, **kwargs)
def __send_request(uri, data, headers, method, verify, allow_redirects, tls_min, tls_max):
__log('\nHEADERS++++++++++++++++++++++++++++++++++++')
__log(headers)
__log('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
__log('Request URL = ' + uri)
if (verify is False):
import urllib3
urllib3.disable_warnings()
if tls_min is not None and tls_max is not None:
min_ver = TLS_VERSIONS[tls_min]
max_ver = TLS_VERSIONS[tls_max]
if min_ver > max_ver:
raise ValueError('--tls-min ({}) must not exceed --tls-max ({})'.format(tls_min, tls_max))
# Always disable automatic redirects for signed requests
# We'll handle redirects manually to avoid re-signing presigned URLs
with requests.Session() as session:
if tls_min is not None or tls_max is not None or not verify:
session.mount('https://', _TLSAdapter(tls_min=tls_min, tls_max=tls_max, verify=verify))
response = session.request(method, uri, headers=headers, data=data, verify=verify, allow_redirects=False)
__log('\nRESPONSE++++++++++++++++++++++++++++++++++++')
__log('Response code: %d\n' % response.status_code)
# If user wants to follow redirects and we got a redirect response
if allow_redirects and response.status_code in (301, 302, 303, 307, 308):
redirect_url = response.headers.get('Location')
if redirect_url:
__log('\nFOLLOWING REDIRECT (unsigned)+++++++++++++++++++++++++++')
__log('Redirect URL = ' + redirect_url)
# Resolve relative redirect targets against the response URL
# to avoid MissingSchema errors (e.g. Location: /login)
redirect_url = urljoin(response.url, redirect_url)
# Follow redirect WITHOUT signing (important for presigned URLs)
# Strip only Authorization and AWS signing headers, keep user headers
redirect_headers = {k: v for k, v in headers.items() if k not in
('Authorization', 'x-amz-date',
'x-amz-content-sha256', 'x-amz-security-token')}
# 301/302/303: follow with GET (303 explicitly changes method to GET)
# 307/308: preserve original method and body
if response.status_code in (301, 302, 303):
follow_method = 'GET'
follow_data = None
else: # 307, 308
follow_method = method
follow_data = data
response = requests.request(follow_method, redirect_url,
headers=redirect_headers, data=follow_data,
verify=verify, allow_redirects=True)
__log('\nREDIRECT RESPONSE++++++++++++++++++++++++++++++++++++')
__log('Response code: %d\n' % response.status_code)
return response
# pylint: disable=too-many-branches
def load_aws_config(access_key, secret_key, security_token, credentials_path, profile):
# type: (str, str, str, str, str) -> Tuple[str, str, str]
"""
Load aws credential configuration, by parsing credential file, then try to fall back to
botocore, by checking (access_key,secret_key) are not (None,None)
"""
if access_key is None or secret_key is None:
try:
exists = os.path.exists(credentials_path)
__log('Credentials file \'{0}\' exists \'{1}\''.format(credentials_path, exists))
config = configparser.ConfigParser()
config.read(credentials_path)
while True:
if access_key is None and config.has_option(profile, "aws_access_key_id"):
access_key = config.get(profile, "aws_access_key_id")
else:
break
if secret_key is None and config.has_option(profile, "aws_secret_access_key"):
secret_key = config.get(profile, "aws_secret_access_key")
else:
break
if security_token is None and config.has_option(profile, "aws_session_token"):
security_token = config.get(profile, "aws_session_token")
break
except configparser.NoSectionError as exception:
__log('AWS profile \'{0}\' not found'.format(exception.args))
raise exception
except configparser.NoOptionError as exception:
__log('AWS profile \'{0}\' is missing \'{1}\''.format(profile, exception.args))
raise exception
except ValueError as exception:
__log(exception)
raise exception
# try to load instance credentials using botocore
if access_key is None or secret_key is None:
try:
__log("loading botocore package")
import botocore
except ImportError:
__log("botocore package could not be loaded")
botocore = None
if botocore:
import botocore.session
session = botocore.session.get_session()
cred = session.get_credentials()
access_key, secret_key, security_token = cred.access_key, cred.secret_key, cred.token
return access_key, secret_key, security_token
def normalize_args(args):
if args.access_key == "":
args.access_key = None
if args.secret_key == "":
args.secret_key = None
if args.security_token == "":
args.security_token = None
if args.session_token == "":
args.session_token = None
def parse_data(data: Optional[str], binary: bool) -> Optional[Union[str, bytes]]:
if data is None:
return None
# if data is not a file identifier, return it
if not data.startswith("@"):
return data
# if data is the stdin `@-` identifier, read from stdin
if data == "@-":
return sys.stdin.read()
# otherwise read from the file
filename = data[1:]
read_mode = "rb" if binary else "r"
with open(filename, read_mode) as post_data_file:
return post_data_file.read()
def inner_main(argv: List[str]) -> int:
"""
Awscurl CLI main entry point
"""
# note EC2 ignores Accept header and responds in xml
default_headers = ['Accept: application/xml',
'Content-Type: application/json']
parser = configargparse.ArgumentParser(
description='Curl AWS request signing',
formatter_class=configargparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('-v', '--verbose', action='store_true',
help='verbose flag', default=False)
parser.add_argument('-i', '--include', action='store_true',
help='include headers in the output', default=False)
parser.add_argument('-X', '--request',
help='Specify request command to use',
default='GET')
parser.add_argument('-d', '--data', help='HTTP POST data', default='')
parser.add_argument('-H', '--header', help='HTTP header', action='append')
parser.add_argument('-k', '--insecure', action='store_true', default=False,
help='Allow insecure server connections when using SSL')
parser.add_argument('--fail-with-body', action='store_true', help='Fail on HTTP errors but save the body', default=False)
parser.add_argument('--data-binary', action='store_true',
help='Process HTTP POST data exactly as specified with '
'no extra processing whatsoever.', default=False)
parser.add_argument('--region', help='AWS region', default='us-east-1',
env_var='AWS_DEFAULT_REGION')
parser.add_argument('--profile', help='AWS profile', default='default', env_var='AWS_PROFILE')
parser.add_argument('--service', help='AWS service', default='execute-api')
parser.add_argument('--access_key', env_var='AWS_ACCESS_KEY_ID')
parser.add_argument('--secret_key', env_var='AWS_SECRET_ACCESS_KEY')
# AWS_SECURITY_TOKEN is deprecated, but kept for backward compatibility
# https://github.com/boto/botocore/blob/c76553d3158b083d818f88c898d8f6d7918478fd/botocore/credentials.py#L260-262
parser.add_argument('--security_token', env_var='AWS_SECURITY_TOKEN')
parser.add_argument('--session_token', env_var='AWS_SESSION_TOKEN')
parser.add_argument('-L', '--location', action='store_true', default=False,
help="Follow redirects")
parser.add_argument('-o', '--output', metavar="", help='Write to file instead of stdout', default='')
parser.add_argument('--tls-min', help='Set minimum allowed TLS version', choices=TLS_VERSIONS.keys())
parser.add_argument('--tls-max', help='Set maximum allowed TLS version', choices=TLS_VERSIONS.keys())
parser.add_argument('uri')
args = parser.parse_args(argv)
normalize_args(args)
# pylint: disable=global-statement
global IS_VERBOSE
IS_VERBOSE = args.verbose
if args.verbose:
__log(vars(args))
data = parse_data(args.data, args.data_binary)
if data is None:
data = ''
if args.header is None:
args.header = default_headers
if args.security_token is not None:
args.session_token = args.security_token
del args.security_token
# pylint: disable=deprecated-lambda
headers = CaseInsensitiveDict({k: v for (k, v) in map(lambda s: s.split(": "), args.header)})
credentials_path = os.path.expanduser("~") + "/.aws/credentials"
args.access_key, args.secret_key, args.session_token = load_aws_config(args.access_key,
args.secret_key,
args.session_token,
credentials_path,
args.profile)
if args.access_key is None:
raise ValueError('No access key is available')
if args.secret_key is None:
raise ValueError('No secret key is available')
response = make_request(args.request,
args.service,
args.region,
args.uri,
headers,
data,
args.access_key,
args.secret_key,
args.session_token,
args.data_binary,
verify=not args.insecure,
allow_redirects=args.location,
tls_min=args.tls_min,
tls_max=args.tls_max)
if args.include:
print(response.headers, end='\n\n')
elif IS_VERBOSE:
pprint.PrettyPrinter(stream=sys.stderr).pprint(response.headers)
pprint.PrettyPrinter(stream=sys.stderr).pprint('')
# Write response body to stdout as raw bytes (matching curl behavior)
# Flush first so any text-mode output (e.g. --include headers) is written before the binary body
sys.stdout.flush()
sys.stdout.buffer.write(response.content)
sys.stdout.buffer.flush()
if args.output:
filename = args.output
# Always write raw bytes to file (matching curl behavior)
# --data-binary affects request body hashing, not response encoding
with open(filename, "wb") as f:
f.write(response.content)
exit_code = 0 if response.ok or not args.fail_with_body else 22
return exit_code
def main():
return inner_main(sys.argv[1:])
if __name__ == '__main__':
sys.exit(main())
awscurl-0.44/awscurl/utils.py 0000664 0000000 0000000 00000001127 15213143633 0016315 0 ustar 00root root 0000000 0000000 """
Utilities needed during the signing process
"""
import hashlib
import hmac
def sha256_hash(val):
"""
Sha256 hash of text data.
"""
return hashlib.sha256(val.encode('utf-8')).hexdigest()
def sha256_hash_for_binary_data(val):
"""
Sha256 hash of binary data.
"""
return hashlib.sha256(val).hexdigest()
def sign(key, msg):
"""
Key derivation functions.
See: http://docs.aws.amazon.com
/general/latest/gr/signature-v4-examples.html
#signature-v4-examples-python
"""
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
awscurl-0.44/ci/ 0000775 0000000 0000000 00000000000 15213143633 0013515 5 ustar 00root root 0000000 0000000 awscurl-0.44/ci/ci-alpine/ 0000775 0000000 0000000 00000000000 15213143633 0015356 5 ustar 00root root 0000000 0000000 awscurl-0.44/ci/ci-alpine/Dockerfile 0000664 0000000 0000000 00000002257 15213143633 0017356 0 ustar 00root root 0000000 0000000 FROM alpine
# RUN echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections && \
# echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections && \
# apk add --no-cache tzdata && \
# cp /usr/share/zoneinfo/Europe/Paris /etc/localtime && \
# echo "Europe/Paris" > /etc/timezone && \
# apk del tzdata
RUN apk update && \
apk add sudo curl git build-base autoconf automake libtool \
openssl-dev readline-dev zlib-dev sqlite-dev ncurses-dev \
xz-dev tk-dev libffi-dev bzip2-dev bash gcc make musl-dev \
libffi-dev openssl-dev zlib-dev readline-dev sqlite-dev
RUN curl https://pyenv.run | bash
ENV PATH="/root/.pyenv/bin:$PATH"
RUN eval "$(pyenv init -)" && \
eval "$(pyenv virtualenv-init -)"
WORKDIR /root/workdir
COPY .python-version .python-version
RUN for version in $(cat .python-version); do \
/root/.pyenv/bin/pyenv install $version; \
done
COPY setup.cfg setup.cfg
COPY awscurl awscurl
COPY setup.py setup.py
COPY scripts/ci.sh scripts/ci.sh
COPY requirements.txt requirements.txt
COPY requirements-test.txt requirements-test.txt
COPY tests tests
# RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"
awscurl-0.44/ci/ci-amazonlinux/ 0000775 0000000 0000000 00000000000 15213143633 0016453 5 ustar 00root root 0000000 0000000 awscurl-0.44/ci/ci-amazonlinux/Dockerfile 0000664 0000000 0000000 00000001267 15213143633 0020453 0 ustar 00root root 0000000 0000000 FROM amazonlinux:2023
RUN yum update -y
RUN yum install -y sudo tar gzip make gcc openssl-devel bzip2-devel \
libffi-devel zlib-devel readline-devel sqlite-devel xz-devel \
git
RUN curl https://pyenv.run | bash
ENV PATH="/root/.pyenv/bin:$PATH"
RUN eval "$(pyenv init -)" && \
eval "$(pyenv virtualenv-init -)"
WORKDIR /root/workdir
COPY .python-version .python-version
RUN for version in $(cat .python-version); do \
/root/.pyenv/bin/pyenv install $version; \
done
COPY setup.cfg setup.cfg
COPY awscurl awscurl
COPY setup.py setup.py
COPY scripts/ci.sh scripts/ci.sh
COPY requirements.txt requirements.txt
COPY requirements-test.txt requirements-test.txt
COPY tests tests
awscurl-0.44/ci/ci-centos/ 0000775 0000000 0000000 00000000000 15213143633 0015401 5 ustar 00root root 0000000 0000000 awscurl-0.44/ci/ci-centos/Dockerfile 0000664 0000000 0000000 00000001401 15213143633 0017367 0 ustar 00root root 0000000 0000000 FROM tgagor/centos
RUN yum update -y
RUN yum group install -y "Development Tools"
RUN yum install -y libffi-devel readline-devel zlib-devel bzip2-devel sqlite-devel openssl-devel git
RUN curl https://pyenv.run | bash
ENV PATH="/root/.pyenv/bin:$PATH"
RUN eval "$(pyenv init -)" && \
eval "$(pyenv virtualenv-init -)"
WORKDIR /root/workdir
COPY .python-version .python-version
RUN for version in $(cat .python-version); do \
/root/.pyenv/bin/pyenv install $version; \
done
COPY setup.cfg setup.cfg
COPY awscurl awscurl
COPY setup.py setup.py
COPY scripts/ci.sh scripts/ci.sh
COPY requirements.txt requirements.txt
COPY requirements-test.txt requirements-test.txt
COPY tests tests
# RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"
awscurl-0.44/ci/ci-ubuntu/ 0000775 0000000 0000000 00000000000 15213143633 0015430 5 ustar 00root root 0000000 0000000 awscurl-0.44/ci/ci-ubuntu/Dockerfile 0000664 0000000 0000000 00000002131 15213143633 0017417 0 ustar 00root root 0000000 0000000 FROM ubuntu
RUN apt update
RUN apt install -y sudo
RUN echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections
RUN echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections
RUN DEBIAN_FRONTEND="noninteractive" apt install -y tzdata
RUN apt install -y curl git \
build-essential \
autoconf \
automake \
libtool \
libffi-dev libreadline-dev libz-dev libsqlite3-dev libssl-dev \
wget curl libncurses5-dev libncursesw5-dev \
xz-utils tk-dev libffi-dev libbz2-dev liblzma-dev git
RUN curl https://pyenv.run | bash
ENV PATH="/root/.pyenv/bin:$PATH"
RUN eval "$(pyenv init -)" && \
eval "$(pyenv virtualenv-init -)"
WORKDIR /root/workdir
COPY .python-version .python-version
RUN for version in $(cat .python-version); do \
/root/.pyenv/bin/pyenv install $version; \
done
COPY setup.cfg setup.cfg
COPY awscurl awscurl
COPY setup.py setup.py
COPY scripts/ci.sh scripts/ci.sh
COPY requirements.txt requirements.txt
COPY requirements-test.txt requirements-test.txt
COPY tests tests
# RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"
awscurl-0.44/pyrightconfig.json 0000664 0000000 0000000 00000000111 15213143633 0016662 0 ustar 00root root 0000000 0000000 {
"typeCheckingMode": "basic",
"reportMissingTypeStubs": false
}
awscurl-0.44/requirements-test.txt 0000664 0000000 0000000 00000000116 15213143633 0017361 0 ustar 00root root 0000000 0000000 pycodestyle
mock
pytest
pytest-cov
wheel
setuptools
setuptools-rust
build
mypy awscurl-0.44/requirements.txt 0000664 0000000 0000000 00000000056 15213143633 0016407 0 ustar 00root root 0000000 0000000 requests
configargparse
configparser
botocore
awscurl-0.44/scripts/ 0000775 0000000 0000000 00000000000 15213143633 0014611 5 ustar 00root root 0000000 0000000 awscurl-0.44/scripts/ci-in-docker.sh 0000775 0000000 0000000 00000001051 15213143633 0017411 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
set -o errexit
set -o pipefail
set -o nounset
set -o xtrace
docker_run() {
local image_type=$1
local script_file=$2
echo "building ${image_type} image -- first time it could take a few minutes"
docker build -t "awscurl-ci-${image_type}" -f "./ci/ci-${image_type}/Dockerfile" . &&
docker run --rm -t "awscurl-ci-${image_type}" bash -c "${script_file}"
}
docker_run "ubuntu" "./scripts/ci.sh"
docker_run "alpine" "./scripts/ci.sh"
docker_run "centos" "./scripts/ci.sh"
docker_run "amazonlinux" "./scripts/ci.sh"
awscurl-0.44/scripts/ci.sh 0000775 0000000 0000000 00000003142 15213143633 0015543 0 ustar 00root root 0000000 0000000 #!/bin/bash
DETOX_ROOT_DIR=./build/detox
grep -v '^ *#' < .python-version | while IFS= read -r PYENV_VERSION
do
# echo PYENV_VERSION="$PYENV_VERSION"
# echo SHELL="$SHELL"
# echo $PATH
# pyenv install -sv "${PYENV_VERSION}"
# https://github.com/pyenv/pyenv/issues/1819#issuecomment-780803524
# /root/.pyenv/bin/pyenv shell "${PYENV_VERSION}"
export PYENV_VERSION="$PYENV_VERSION"
eval "$(pyenv init -)"
PER_VER_DIR=${DETOX_ROOT_DIR}/v${PYENV_VERSION}
VENV_DIR=${PER_VER_DIR}/venv${PYENV_VERSION}
(
echo "##### NEW DETOX ENV: " "$(uname) " "${PER_VER_DIR}" " #####"
python3 -m venv "${VENV_DIR}"
source "${VENV_DIR}"/bin/activate
echo which python="$(command -v python)"
echo python --version="$(python --version)"
echo pip --version="$(pip --version)"
PS4='[$(date "+%Y-%m-%d %H:%M:%S")] '
set -o errexit -o pipefail -o nounset -o xtrace
pip -q -q install --upgrade pip
# python -m ensurepip --upgrade
# pip install -r requirements.txt
pip -q -q install -r requirements-test.txt
pycodestyle .
# python -m build .
pip -q install .
export AWS_ACCESS_KEY_ID=MOCK_AWS_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY=MOCK_AWS_SECRET_ACCESS_KEY
export AWS_SESSION_TOKEN=MOCK_AWS_SESSION_TOKEN
pytest \
--cov=awscurl \
--cov-fail-under=77 \
--cov-report html \
--cov-report=html:"${PER_VER_DIR}"/htmlcov \
--durations=2 \
--strict-config
)
done
awscurl-0.44/scripts/install.sh 0000775 0000000 0000000 00000002374 15213143633 0016624 0 ustar 00root root 0000000 0000000 #!/bin/env bash
set -o errexit
set -o pipefail
set -o nounset
set -o xtrace
OS_RELEASE=$(. /etc/os-release; echo "${NAME}")
if [ "${OS_RELEASE}" = "Ubuntu" ]; then
apt update
apt install -y sudo
echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections
echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections
DEBIAN_FRONTEND="noninteractive" apt install -y tzdata
apt install -y curl git \
build-essential \
autoconf \
automake \
libtool \
libffi-dev libreadline-dev libz-dev libsqlite-dev libssl-dev \
libreadline-dev libsqlite3-dev wget curl libncurses5-dev libncursesw5-dev \
xz-utils tk-dev libffi-dev libbz2-dev liblzma-dev git
elif [ "${OS_RELEASE}" = "CentOS Linux" ]; then
yum update -y
yum group install -y "Development Tools"
yum install -y libffi-devel readline-devel zlib-devel bzip2-devel sqlite-devel openssl-devel git
fi
curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
export PATH="/root/.pyenv/bin:$PATH"
if [ -z "${PROMPT_COMMAND:-}" ]; then
export PROMPT_COMMAND=""
fi
eval "$(pyenv init -)"
eval "$(pyenv virtualenv-init -)"
grep -v '^ *#' < .python-version | while IFS= read -r line
do
pyenv install -s "${line}"
done
awscurl-0.44/scripts/pypi_publish.sh 0000775 0000000 0000000 00000000135 15213143633 0017656 0 ustar 00root root 0000000 0000000 #!/bin/bash
set -e
pip install twine
python setup.py sdist bdist_wheel
twine upload dist/* awscurl-0.44/setup.cfg 0000664 0000000 0000000 00000000423 15213143633 0014742 0 ustar 00root root 0000000 0000000 [pycodestyle]
count = False
max-line-length = 120
statistics = True
ignore = E501,W291,E225,E123,E266,W504
exclude = build,venv,venv3.6.15,venv3.8.16,.tox
[mypy]
ignore_missing_imports = True
disable_error_code = import-untyped
[mypy-tests.*]
disable_error_code = arg-type
awscurl-0.44/setup.py 0000664 0000000 0000000 00000002216 15213143633 0014635 0 ustar 00root root 0000000 0000000 __author__ = 'iokulist'
from setuptools import setup # type: ignore[import-untyped]
# https://github.com/okigan/awscurl/issues/167
# with open("requirements.txt", "r", encoding="utf-8") as f:
# requirements = f.read().splitlines()
with open("README.md", "r", encoding="utf-8") as f:
long_description = f.read()
# https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html#summary
setup(
name='awscurl',
version='0.44',
description='Curl like tool with AWS request signing',
long_description=long_description,
long_description_content_type='text/markdown',
url='http://github.com/okigan/awscurl',
author='Igor Okulist',
author_email='okigan@gmail.com',
license='MIT',
packages=['awscurl'],
# package_data={
# 'tests': ['*.py'],
# },
entry_points={
'console_scripts': [
'awscurl = awscurl.__main__:main',
],
},
zip_safe=False,
install_requires=[
'requests',
'configargparse',
'configparser',
'urllib3',
"boto3",
"botocore",
"awscrt"
],
extras_require={
'awslibs': []
}
)
awscurl-0.44/tests/ 0000775 0000000 0000000 00000000000 15213143633 0014264 5 ustar 00root root 0000000 0000000 awscurl-0.44/tests/__init__.py 0000664 0000000 0000000 00000000000 15213143633 0016363 0 ustar 00root root 0000000 0000000 awscurl-0.44/tests/basic_test.py 0000664 0000000 0000000 00000000235 15213143633 0016756 0 ustar 00root root 0000000 0000000 import unittest
def increment(x):
return x + 1
class ShallPassTest(unittest.TestCase):
def test(self):
self.assertEqual(increment(3), 4)
awscurl-0.44/tests/data/ 0000775 0000000 0000000 00000000000 15213143633 0015175 5 ustar 00root root 0000000 0000000 awscurl-0.44/tests/data/credentials 0000664 0000000 0000000 00000000126 15213143633 0017414 0 ustar 00root root 0000000 0000000 [default]
aws_access_key_id = access_key_id
aws_secret_access_key = secret_access_key
awscurl-0.44/tests/integration_test.py 0000664 0000000 0000000 00000012730 15213143633 0020223 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: utf-8 -*-
import base64
from unittest import TestCase
import sys
import os
# this block resolves issues with pytest/tox, overall project dir structure
# should be updated, some hints at can be found here:
# https://stackoverflow.com/questions/55737714/how-does-a-tox-environment-set-its-sys-path
print(f'sys.path={sys.path}')
this_script_dir=os.path.dirname(os.path.abspath(__file__))
extra_path=os.path.join(this_script_dir, '..', 'awscurl')
if not os.path.exists(extra_path):
print(f'extra_path does not exist: {extra_path}')
sys.path.append(extra_path)
print(f'sys.path2={sys.path}')
from awscurl.awscurl import make_request, inner_main # nopep8: E402
__author__ = 'iokulist'
class TestMakeRequestWithToken(TestCase):
maxDiff = None
def test_make_request(self, *args, **kvargs):
headers = {}
access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8")
secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8")
params = {'method': 'GET',
'service': 's3',
'region': 'us-east-1',
'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b',
'headers': headers,
'data': '',
'access_key': access_key,
'secret_key': secret_key,
'security_token': None,
'data_binary': False}
r = make_request(**params)
self.assertEqual(r.status_code, 200)
class TestMakeRequestWithTokenAndBinaryData(TestCase):
maxDiff = None
def test_make_request(self, *args, **kvargs):
headers = {}
access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8")
secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8")
params = {'method': 'GET',
'service': 's3',
'region': 'us-east-1',
'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b',
'headers': headers,
'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
'access_key': access_key,
'secret_key': secret_key,
'security_token': None,
'data_binary': True}
r = make_request(**params)
self.assertEqual(r.status_code, 200)
class TestMakeRequestWithTokenAndEnglishData(TestCase):
maxDiff = None
def test_make_request(self, *args, **kvargs):
headers = {}
access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8")
secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8")
params = {'method': 'GET',
'service': 's3',
'region': 'us-east-1',
'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b',
'headers': headers,
'data': 'Test',
'access_key': access_key,
'secret_key': secret_key,
'security_token': None,
'data_binary': False}
r = make_request(**params)
self.assertEqual(r.status_code, 200)
class TestMakeRequestWithTokenAndNonEnglishData(TestCase):
maxDiff = None
def test_make_request(self, *args, **kvargs):
headers = {}
access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8")
secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8")
params = {'method': 'GET',
'service': 's3',
'region': 'us-east-1',
'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b',
'headers': headers,
'data': u'ใในใ',
'access_key': access_key,
'secret_key': secret_key,
'security_token': None,
'data_binary': False}
r = make_request(**params)
self.assertEqual(r.status_code, 200)
class TestInnerMainMethod(TestCase):
maxDiff = None
def test_exit_code_without_fail_option(self, *args, **kwargs):
self.assertEqual(
inner_main(['--verbose', '--service', 's3', 'https://awscurl-sample-bucket.s3.amazonaws.com']),
0
)
def test_exit_code_with_fail_option(self, *args, **kwargs):
self.assertEqual(
inner_main(['--verbose', '--fail-with-body', '--service', 's3', 'https://awscurl-sample-bucket.s3.amazonaws.com']),
22
)
class TestInnerMainMethodEmptyCredentials(TestCase):
maxDiff = None
def test_exit_code_without_fail_option(self, *args, **kwargs):
self.assertEqual(
inner_main(['--verbose', '--access_key', '', '--secret_key', '', '--session_token', '', '--service', 's3',
'https://awscurl-sample-bucket.s3.amazonaws.com']),
0
)
def test_exit_code_with_fail_option(self, *args, **kwargs):
self.assertEqual(
inner_main(['--verbose', '--fail-with-body', '--access_key', '', '--secret_key', '', '--session_token', '', '--service', 's3',
'https://awscurl-sample-bucket.s3.amazonaws.com']),
22
)
awscurl-0.44/tests/load_aws_config_test.py 0000664 0000000 0000000 00000003224 15213143633 0021014 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
from unittest import TestCase
from awscurl.awscurl import load_aws_config
__author__ = 'iokulist'
class Test__load_aws_config(TestCase):
def test(self):
access_key, secret_access, token = load_aws_config(None,
None,
None,
"./tests/data/credentials",
"default")
self.assertEqual([access_key, secret_access, token], ['access_key_id', 'secret_access_key', None])
access_key, secret_access, token = load_aws_config(None,
None,
"ttt",
"./tests/data/credentials",
"default")
self.assertEqual([access_key, secret_access, token], ['access_key_id', 'secret_access_key', 'ttt'])
# TODO: remove this test as I think it's not valid to loads secret_key if session_key was already provided
# access_key, secret_access, token = load_aws_config('aaa',
# None,
# "ttt",
# "./tests/data/credentials",
# "default")
#
# self.assertEquals([access_key, secret_access, token], ['aaa', None, 'ttt'])
awscurl-0.44/tests/stages_test.py 0000664 0000000 0000000 00000016750 15213143633 0017174 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
"""
Test cases for seprate header calculation stages.
"""
import json
from unittest import TestCase
from awscurl.awscurl import (
task_1_create_a_canonical_request,
task_2_create_the_string_to_sign,
task_3_calculate_the_signature,
task_4_build_auth_headers_for_the_request)
class TestStages(TestCase):
"""
Suite to test all stages.
"""
maxDiff = None
def test_task_1_create_a_canonical_request(self):
"""
Test the function to create the "canonical" request to match the thing that AWS is hashing
on the server side.
"""
canonical_request, payload_hash, signed_headers = task_1_create_a_canonical_request(
query="Action=DescribeInstances&Version=2013-10-15",
headers=json.loads('{"Content-Type": "application/json", "Accept": "application/xml"}'),
port=None,
host="ec2.amazonaws.com",
amzdate="20190921T022008Z",
method="GET",
data="",
security_token=None,
data_binary=False,
canonical_uri="/")
self.assertEqual(canonical_request, "GET\n"
"/\n"
"Action=DescribeInstances&Version=2013-10-15\n"
"content-type:application/json\n"
"host:ec2.amazonaws.com\n"
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date:20190921T022008Z\n"
"\n"
"content-type;host;x-amz-content-sha256;x-amz-date\n"
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
self.assertEqual(payload_hash,
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
self.assertEqual(signed_headers, "content-type;host;x-amz-content-sha256;x-amz-date")
def test_task_1_create_a_canonical_request_url_encode_querystring(self):
"""
Test that canonical requests correctly sort and url encode querystring parameters.
"""
canonical_request, payload_hash, signed_headers = task_1_create_a_canonical_request(
query="arg1=true&arg3=c,b,a&arg2=false&noEncoding=ABC-abc_1.23~tilde/slash",
headers=json.loads('{"Content-Type": "application/json", "Accept": "application/xml"}'),
port=None,
host="my-gateway-id.execute-api.us-east-1.amazonaws.com",
amzdate="20190921T022008Z",
method="GET",
data="",
security_token=None,
data_binary=False,
canonical_uri="/stage/my-path")
self.assertEqual(canonical_request, "GET\n"
"/stage/my-path\n"
"arg1=true&arg2=false&arg3=c%2Cb%2Ca&noEncoding=ABC-abc_1.23~tilde%2Fslash\n"
"content-type:application/json\n"
"host:my-gateway-id.execute-api.us-east-1.amazonaws.com\n"
"x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
"x-amz-date:20190921T022008Z\n"
"\n"
"content-type;host;x-amz-content-sha256;x-amz-date\n"
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
self.assertEqual(payload_hash,
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
self.assertEqual(signed_headers, "content-type;host;x-amz-content-sha256;x-amz-date")
def test_task_1_header_with_amz_substring_not_signed(self):
"""
Test that headers containing 'x-amz-' as a substring (not prefix) are not
included in canonical/signed headers.
"""
canonical_request, payload_hash, signed_headers = task_1_create_a_canonical_request(
query="",
headers={"Not-x-amz-fake": "should-not-be-signed"},
port=None,
host="example.amazonaws.com",
amzdate="20190921T022008Z",
method="GET",
data="",
security_token=None,
data_binary=False,
canonical_uri="/")
self.assertNotIn("not-x-amz-fake", signed_headers)
def test_task_2_create_the_string_to_sign(self):
"""
Test the next function that is creating a string in exactly the same way as AWS is on the
server side, to make sure our signature matches.
"""
string_to_sign, algorithm, credential_scope = task_2_create_the_string_to_sign(
amzdate="20190921T022008Z",
datestamp="20190921",
canonical_request="GET\n"
"/\n"
"Action=DescribeInstances&Version=2013-10-15\n"
"host:ec2.amazonaws.com\n"
"x-amz-date:20190921T022008Z\n"
"\n"
"host;x-amz-date\n"
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
service="ec2",
region="us-east-1",
)
self.assertEqual(string_to_sign, "AWS4-HMAC-SHA256\n"
"20190921T022008Z\n"
"20190921/us-east-1/ec2/aws4_request\n"
"4a3b77321aca7e671d4945f0b3b826112e5ca3f2a10c4357e54f518798e7c8ff")
self.assertEqual(algorithm, "AWS4-HMAC-SHA256")
self.assertEqual(credential_scope, "20190921/us-east-1/ec2/aws4_request")
def test_task_3_calculate_the_signature(self):
"""
Test that we calculate the correct signature from our carefully prepared strings.
"""
signature = task_3_calculate_the_signature(
datestamp="20190921",
string_to_sign="AWS4-HMAC-SHA256\n"
"20190921T022008Z\n"
"20190921/us-east-1/ec2/aws4_request\n"
"4a3b77321aca7e671d4945f0b3b826112e5ca3f2a10c4357e54f518798e7c8ff",
service="ec2",
region="us-east-1",
secret_key="dummytestsecretkey",
)
self.assertEqual(signature,
"9164aea23e266890838ff6e51eea552e2ee39c63896ac61d91990f200bb16362")
def test_task_4_build_auth_headers_for_the_request(self):
"""
Test that we are adding the proper headers based on all our calculated information.
"""
new_headers = task_4_build_auth_headers_for_the_request(
amzdate="20190921T022008Z",
payload_hash="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
algorithm="AWS4-HMAC-SHA256",
credential_scope="20190921/us-east-1/ec2/aws4_request",
signed_headers="host;x-amz-date",
signature="9164aea23e266890838ff6e51eea552e2ee39c63896ac61d91990f200bb16362",
access_key="AKIAIJLPLDILMJV53HCQ",
security_token=None,
)
self.assertEqual(
new_headers['x-amz-content-sha256'],
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
self.assertNotIn('x-amz-security-token', new_headers)
self.assertEqual(
new_headers['x-amz-date'],
'20190921T022008Z')
self.assertEqual(
new_headers['Authorization'],
'AWS4-HMAC-SHA256 '
'Credential=AKIAIJLPLDILMJV53HCQ/20190921/us-east-1/ec2/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=9164aea23e266890838ff6e51eea552e2ee39c63896ac61d91990f200bb16362')
awscurl-0.44/tests/tls_test.py 0000664 0000000 0000000 00000007426 15213143633 0016510 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""TLS-related tests for awscurl.
Unit tests verify SSL context configuration (no network needed).
Integration tests verify real HTTPS connections work.
Regression context (issue #235):
On Amazon Linux 2023 the RPM ``python3-requests`` replaces the
``certifi`` import with a hard-coded path to the OS CA bundle
(``/etc/pki/tls/certs/ca-bundle.crt``). ``certifi`` is **not
installed** โ ``pip3 list`` never shows it, even after downgrading
awscurl. Because ``_TLSAdapter.init_poolmanager`` creates a bare
``ssl.SSLContext`` via ``create_urllib3_context()`` without loading
CA certificates, HTTPS requests fail with ``SSLCertVerificationError``
on any system where ``certifi`` is absent.
"""
from typing import Any
from unittest import TestCase
from unittest.mock import patch
from awscurl.awscurl import _TLSAdapter
from requests.adapters import HTTPAdapter
import requests
class TestTLSAdapterSSLContext(TestCase):
"""Regression test for #235: _TLSAdapter must produce an SSL context
with system CA certificates loaded, otherwise HTTPS fails on systems
without certifi (e.g. Amazon Linux with distro-packaged requests)."""
def test_ssl_context_has_ca_certs(self):
"""The SSL context created by _TLSAdapter should load default certs.
Note: we verify load_default_certs() is called rather than checking
get_ca_certs() count, because on capath-based systems (e.g. ubuntu-22.04)
certs aren't enumerable until an actual TLS connection uses them.
"""
adapter = _TLSAdapter(tls_min=None, tls_max=None, verify=True)
with patch('awscurl.awscurl.create_urllib3_context') as mock_create:
mock_ctx = mock_create.return_value
with patch.object(HTTPAdapter, 'init_poolmanager'):
adapter.init_poolmanager(1, 1)
mock_ctx.load_default_certs.assert_called_once()
def test_ssl_context_has_ca_certs_with_tls_versions(self):
"""CA certs should also be loaded when TLS versions are specified."""
adapter = _TLSAdapter(tls_min='1.2', tls_max='1.3', verify=True)
with patch('awscurl.awscurl.create_urllib3_context') as mock_create:
mock_ctx = mock_create.return_value
with patch.object(HTTPAdapter, 'init_poolmanager'):
adapter.init_poolmanager(1, 1)
mock_ctx.load_default_certs.assert_called_once()
class TestHTTPSDefaultTLS(TestCase):
"""Integration test: verify real HTTPS connections work with and
without _TLSAdapter. Any HTTP status is acceptable โ we only test
that the TLS handshake completes (SSLError would be raised otherwise)."""
def test_https_no_ssl_error_without_tls_adapter(self):
"""Baseline: plain requests.Session completes TLS handshake."""
with requests.Session() as session:
response = session.get('https://awscurl-sample-bucket.s3.amazonaws.com')
self.assertIsNotNone(response.status_code)
def test_https_no_ssl_error_with_tls_adapter(self):
"""_TLSAdapter with explicit TLS versions completes TLS handshake."""
with requests.Session() as session:
session.mount('https://', _TLSAdapter(tls_min='1.2', tls_max='1.3', verify=True))
response = session.get('https://awscurl-sample-bucket.s3.amazonaws.com')
self.assertIsNotNone(response.status_code)
def test_https_no_ssl_error_with_default_tls_adapter(self):
"""_TLSAdapter with no TLS args (v0.40 bug path) completes TLS handshake."""
with requests.Session() as session:
session.mount('https://', _TLSAdapter(tls_min=None, tls_max=None, verify=True))
response = session.get('https://awscurl-sample-bucket.s3.amazonaws.com')
self.assertIsNotNone(response.status_code)
awscurl-0.44/tests/unit_test.py 0000775 0000000 0000000 00000051226 15213143633 0016665 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: UTF-8 -*-
# mypy: disable-error-code="arg-type"
import datetime
import json
import sys
from types import SimpleNamespace
from typing import Any
from unittest import TestCase
try:
from mock import patch # type: ignore[import-untyped]
except ImportError:
from unittest.mock import patch
from awscurl.awscurl import aws_url_encode, make_request, parse_data
from requests.exceptions import SSLError
from requests import Response
from io import StringIO
import pytest
from _pytest.monkeypatch import MonkeyPatch
__author__ = 'iokulist'
def my_mock_get() -> Any:
def ss(*args: Any, **kargs: Any) -> SimpleNamespace:
print("in mock")
response = SimpleNamespace(status_code=200, text='some text')
return response
return ss
def my_mock_send_request() -> Any:
def ss(*args: Any, **kargs: Any) -> SimpleNamespace:
print("in mock")
response = SimpleNamespace(status_code=200, text='some text')
return response
return ss
def my_mock_send_request_verify() -> Any:
def ss(uri: Any, data: Any, headers: Any, method: Any,
verify: Any, allow_redirects: Any,
tls_min: Any, tls_max: Any, **kargs: Any) -> SimpleNamespace:
print("in mock")
if not verify:
raise SSLError
response = SimpleNamespace(status_code=200, text='some text')
return response
return ss
def my_mock_utcnow() -> Any:
def ss(*args: Any, **kargs: Any) -> datetime.datetime:
print("in mock")
return datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc)
return ss
class TestMakeRequest(TestCase):
maxDiff = None
@patch('requests.get', new_callable=my_mock_get)
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': '',
'access_key': '',
'secret_key': '',
'security_token': '',
'data_binary': False}
make_request(**params)
expected = {'x-amz-date': '19700101T000000Z',
'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=591b7ad3b09e1a58bfb44a2cce310a3474643d2feb56bae3fe707638e6001fd9',
'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
'x-amz-security-token': ''}
self.assertEqual(expected, headers)
pass
@patch('requests.get', new_callable=my_mock_get)
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request2(self, *args: Any, **kvargs: Any):
payload = json.dumps({
"key": "",
})
creds = {
"access_key": "",
"secret_key": "",
"token": ""
}
headers = {
"Content-Type": "application/json; charset:UTF-8",
"Connection": "keep-alive",
"Content-Encoding": "amz-1.0",
"x-amz-requestsupertrace": "true"
}
params = {
'method': 'POST',
'service': 'service-',
'region': "region-",
'uri': "",
'headers': headers,
'data': payload,
'data_binary': False,
'access_key': creds['access_key'],
'secret_key': creds['secret_key'],
'security_token': creds['token'],
}
make_request(**params)
expected = {
"Content-Type": "application/json; charset:UTF-8",
"Connection": "keep-alive",
"Content-Encoding": "amz-1.0",
"x-amz-requestsupertrace": "true",
"Authorization": "AWS4-HMAC-SHA256 Credential=/19700101/region-/service-/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-requestsupertrace;x-amz-security-token, Signature=b7346445ac29a9e7ee32b37b12b40295fe715b023c32579fedaf8518c472560b",
"x-amz-date": "19700101T000000Z",
"x-amz-content-sha256": "4930e13bdc55bb30accf137260ec8fa65b35658360e92a5f8498def3f8ab6144",
"x-amz-security-token": ""
}
self.assertEqual(expected, headers)
pass
class TestMakeRequestVerifySSLRaises(TestCase):
maxDiff = None
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request_verify)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': '',
'access_key': '',
'secret_key': '',
'security_token': '',
'data_binary': False,
'verify': False,
'allow_redirects': False}
with pytest.raises(SSLError):
make_request(**params)
pass
class TestMakeRequestVerifySSLPass(TestCase):
maxDiff = None
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request_verify)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': '',
'access_key': '',
'secret_key': '',
'security_token': '',
'data_binary': False,
'verify': True,
'allow_redirects': False}
make_request(**params)
expected = {'x-amz-date': '19700101T000000Z',
'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=591b7ad3b09e1a58bfb44a2cce310a3474643d2feb56bae3fe707638e6001fd9',
'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
'x-amz-security-token': ''}
self.assertEqual(expected, headers)
pass
class TestMakeRequestWithTLSVersions(TestCase):
maxDiff = None
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': '',
'access_key': '',
'secret_key': '',
'security_token': '',
'data_binary': False,
'verify': True,
'allow_redirects': False,
'tls_min': '1.1',
'tls_max': '1.3'}
with patch('awscurl.awscurl.__send_request') as mock_send:
make_request(**params)
mock_send.assert_called_once()
_, kwargs = mock_send.call_args
self.assertEqual('1.1', kwargs.get('tls_min', mock_send.call_args[0][6]))
self.assertEqual('1.3', kwargs.get('tls_max', mock_send.call_args[0][7]))
class TestMakeRequestWithoutTLSVersions(TestCase):
maxDiff = None
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': '',
'access_key': '',
'secret_key': '',
'security_token': '',
'data_binary': False,
'verify': True,
'allow_redirects': False}
with patch('awscurl.awscurl.__send_request') as mock_send:
make_request(**params)
mock_send.assert_called_once()
_, kwargs = mock_send.call_args
self.assertIsNone(kwargs.get('tls_min', mock_send.call_args[0][6]))
self.assertIsNone(kwargs.get('tls_max', mock_send.call_args[0][7]))
class TestMakeRequestWithBinaryData(TestCase):
maxDiff = None
@patch('requests.get', new_callable=my_mock_get)
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
'access_key': '',
'secret_key': '',
'security_token': '',
'data_binary': True}
make_request(**params)
expected = {'x-amz-date': '19700101T000000Z',
'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=acd1d93620304cf08a36002b2f83fdca1e61a1d76d9621d24a7fe65360c09d56',
'x-amz-content-sha256': '3f514228bd64bbff67daaa80e482aee0e0b0c51891d3a64e4abfa145f4364b99',
'x-amz-security-token': ''}
self.assertEqual(expected, headers)
pass
class TestMakeRequestWithToken(TestCase):
maxDiff = None
@patch('requests.get', new_callable=my_mock_get)
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': '',
'access_key': 'ABC',
'secret_key': 'DEF',
'security_token': 'GHI',
'data_binary': False}
make_request(**params)
expected = {'x-amz-date': '19700101T000000Z',
'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
'Authorization': 'AWS4-HMAC-SHA256 Credential=ABC/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=ea6289aa4b2d606a25398ae763129ae05f8767240215ab605d8f0c728b02a94b',
'x-amz-security-token': 'GHI'}
self.assertEqual(expected, headers)
pass
class TestMakeRequestWithTokenAndBinaryData(TestCase):
maxDiff = None
@patch('requests.get', new_callable=my_mock_get)
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
'access_key': 'ABC',
'secret_key': 'DEF',
'security_token': 'GHI',
'data_binary': True}
make_request(**params)
expected = {'x-amz-date': '19700101T000000Z',
'x-amz-content-sha256': '3f514228bd64bbff67daaa80e482aee0e0b0c51891d3a64e4abfa145f4364b99',
'Authorization': 'AWS4-HMAC-SHA256 Credential=ABC/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=67ef2ee6583f88829575af9dec721aab9255e0a87a53fb439df86ee763ebbfd3',
'x-amz-security-token': 'GHI'}
self.assertEqual(expected, headers)
pass
class TestHostFromHeaderUsedInCanonicalHeader(TestCase):
maxDiff = None
@patch('requests.get', new_callable=my_mock_get)
@patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
@patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
def test_make_request(self, *args: Any, **kvargs: Any):
headers = {'host': 'some.other.host.address.com'}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': '',
'access_key': 'ABC',
'secret_key': 'DEF',
'security_token': 'GHI',
'data_binary': False}
make_request(**params)
expected = {'host': 'some.other.host.address.com',
'x-amz-date': '19700101T000000Z',
'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
'Authorization': 'AWS4-HMAC-SHA256 Credential=ABC/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=d470f8a8988b7de8ed1a9a9bf29b68706100fc8bdadd2db5fb9f602de4b49778',
'x-amz-security-token': 'GHI'}
self.assertEqual(expected, headers)
pass
class TestRequestResponse(TestCase):
maxDiff = None
@patch('awscurl.awscurl.__send_request')
def test_make_request(self, mocked_resp: Any) -> None:
resp = Response()
resp.status_code=200
resp._content = b'{"file_name": "test.yml", "env": "staging", "hash": "\xe5\xad\x97"}'
resp.encoding = 'UTF-8'
mocked_resp.return_value = resp
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'ec2',
'region': 'region',
'uri': 'https://user:pass@host:123/path/?a=b&c=d',
'headers': headers,
'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
'access_key': '',
'secret_key': '',
'security_token': '',
'data_binary': True}
r = make_request(**params)
expected = u'\u5b57'
### assert that the unicode character is in the response.text output
self.assertTrue(expected in r.text)
### assert that the unicode character is _not_ in the response.text.encode('utf-8')
### which has been converted to 8-bit string with unicode characters escaped
### in py2 this raises an exception on the assertion (`expected in x` below)
### in py3 we can compare the two directly, and the assertion should be false
if sys.version_info[0] == 2:
with self.assertRaises(UnicodeDecodeError):
x = str(r.text.encode('utf-8'))
expected in x
else:
self.assertFalse(expected in str(r.text.encode('utf-8')))
pass
class TestAwsUrlEncode(TestCase):
def test_aws_url_encode(self):
self.assertEqual(aws_url_encode(""), "")
self.assertEqual(aws_url_encode("AZaz09-_.~"), "AZaz09-_.~")
self.assertEqual(aws_url_encode(" /:@[`{"), "%20%2F%3A%40%5B%60%7B")
self.assertEqual(aws_url_encode("a=,=b"), "a==%2C==b")
self.assertEqual(aws_url_encode("\u0394-\u30a1"), "%CE%94-%E3%82%A1")
pass
@pytest.fixture(scope="class")
def monkeypatch_for_class(request):
request.cls.monkeypatch = MonkeyPatch()
@pytest.mark.usefixtures("monkeypatch_for_class")
class TestMakeRequestWithDataFromStdin(TestCase):
monkeypatch: MonkeyPatch
def setUp(self):
pass
def test_input_data(self):
expected = '{"hello": "world"}'
data = parse_data(expected, False)
self.assertEqual(expected, data)
pass
def test_input_stdin(self):
expected = json.dumps({
"my": "arbitrary-json-data",
"another": {
"random": "json-object"
},
"and": ["a", "list", "too"]
})
self.monkeypatch.setattr('sys.stdin', StringIO(expected))
data = parse_data("@-", False)
self.assertEqual(expected, data)
pass
class TestBinaryResponseOutput(TestCase):
"""Test that binary response content is written as raw bytes, matching curl behavior.
See: https://github.com/okigan/awscurl/issues/242
"""
@patch('awscurl.awscurl.__send_request')
def test_binary_response_content_preserved_in_output(self, mocked_request) -> None:
"""Binary response content (with non-UTF-8 bytes) should be written as raw bytes."""
# Simulate a gzip file: 0x1f 0x8b are gzip magic bytes
# 0x91 is NOT valid UTF-8 start byte
binary_content = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x91\xab\xcd\xef'
resp = Response()
resp.status_code = 200
resp._content = binary_content
resp.encoding = 'UTF-8'
resp.headers = {} # type: ignore[assignment]
mocked_request.return_value = resp
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'execute-api',
'region': 'us-east-1',
'uri': 'https://api.execute-api.us-east-1.amazonaws.com/v1/file.gz',
'headers': headers,
'data': '',
'access_key': 'AKIAIOSFODNN7EXAMPLE',
'secret_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
'security_token': '',
'data_binary': False}
r = make_request(**params)
# response.content should be raw bytes - unchanged
self.assertEqual(r.content, binary_content)
# response.text decodes with replacement chars - not what we want for output
self.assertNotEqual(r.text.encode('utf-8'), binary_content)
@patch('awscurl.awscurl.__send_request')
def test_gzip_magic_bytes_preserved(self, mocked_request) -> None:
"""gzip magic bytes (0x1f 0x8b) must be preserved exactly in raw output.
The core issue #242: non-UTF-8 byte 0x8b was being replaced with U+FFFD
(ef bf bd) when response.text was used, corrupting binary files.
"""
gzip_content = b'\x1f\x8b\x08\x00\xff\xab\xcd\xef\x00'
resp = Response()
resp.status_code = 200
resp._content = gzip_content
resp.encoding = 'UTF-8'
resp.headers = {} # type: ignore[assignment]
mocked_request.return_value = resp
headers: dict[str, str] = {}
params = {'method': 'GET',
'service': 'execute-api',
'region': 'us-east-1',
'uri': 'https://api.execute-api.us-east-1.amazonaws.com/v1/data.gz',
'headers': headers,
'data': '',
'access_key': 'AKIAIOSFODNN7EXAMPLE',
'secret_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
'security_token': '',
'data_binary': False}
r = make_request(**params)
# The raw content must be byte-identical
self.assertEqual(r.content, gzip_content)
# Verify magic bytes at position 0,1 (0x1f 0x8b) - these are non-UTF-8 start bytes
self.assertEqual(r.content[0:2], b'\x1f\x8b')
# Byte 0xff at position 4 is invalid UTF-8 - must be preserved
self.assertEqual(r.content[4], 0xff)
# Byte 0xab at position 5 is invalid UTF-8 - must be preserved
self.assertEqual(r.content[5], 0xab)
awscurl-0.44/tests/url_parsing_test.py 0000664 0000000 0000000 00000003356 15213143633 0020231 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
from unittest import TestCase
from awscurl import awscurl
__author__ = 'iokulist'
class TestUriParsing(TestCase):
maxDiff = None
def test(self, *args, **kvargs):
self.assertEqual(awscurl.url_path_to_dict("http://google.com"),
{'host': 'google.com',
'password': None,
'path': '/',
'port': None,
'query': '',
'schema': 'http',
'user': None})
self.assertEqual(awscurl.url_path_to_dict("http://user:password@google.com"),
{'host': 'google.com',
'password': 'password',
'path': '/',
'port': None,
'query': '',
'schema': 'http',
'user': 'user'})
self.assertEqual(awscurl.url_path_to_dict("http://user:password@google.com/path1/path2"),
{'host': 'google.com',
'password': 'password',
'path': '/path1/path2',
'port': None,
'query': '',
'schema': 'http',
'user': 'user'})
self.assertEqual(awscurl.url_path_to_dict("http://google.com/path1/path2/@weird"),
{'host': 'google.com',
'password': None,
'path': '/path1/path2/@weird',
'port': None,
'query': '',
'schema': 'http',
'user': None})