pax_global_header00006660000000000000000000000064145201761370014520gustar00rootroot0000000000000052 comment=7506f382e5a44e69767ce7a48e9b20689cce2f0e curl-trurl-4f06332/000077500000000000000000000000001452017613700141045ustar00rootroot00000000000000curl-trurl-4f06332/.checksrc000066400000000000000000000000221452017613700156640ustar00rootroot00000000000000disable FOPENMODE curl-trurl-4f06332/.github/000077500000000000000000000000001452017613700154445ustar00rootroot00000000000000curl-trurl-4f06332/.github/workflows/000077500000000000000000000000001452017613700175015ustar00rootroot00000000000000curl-trurl-4f06332/.github/workflows/codeql.yml000066400000000000000000000027251452017613700215010ustar00rootroot00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: ["master"] pull_request: # The branches below must be a subset of the branches above branches: ["master"] jobs: analyze: name: Analyze runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: ['cpp', 'python'] steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: install libcurl run: | sudo apt-get update sudo apt-get install libcurl4-openssl-dev - name: build run: make - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: category: "/language:${{matrix.language}}" curl-trurl-4f06332/.github/workflows/codespell.yml000066400000000000000000000006251452017613700222010ustar00rootroot00000000000000name: "Codespell" on: [push, pull_request] jobs: codespell: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3 - name: install codespell run: | sudo apt-get update sudo apt-get install codespell - name: Perform spelling checks run: codespell README.md RELEASE-NOTES CONTRIBUTING.md trurl.1 trurl.c curl-trurl-4f06332/.github/workflows/makefile.yml000066400000000000000000000032201452017613700217760ustar00rootroot00000000000000name: build on: push: branches: ["master"] pull_request: branches: ["master"] jobs: ubuntu: runs-on: ubuntu-latest strategy: matrix: build: - name: default install_packages: valgrind test: test-memory - name: clang sanitizers install_packages: clang test: test make_opts: > CC=clang CFLAGS="-fsanitize=address,undefined,signed-integer-overflow -Wformat -Werror=format-security -Werror=array-bounds -g" LDFLAGS="-fsanitize=address,undefined,signed-integer-overflow -g" steps: - uses: actions/checkout@v3 - name: install libcurl run: | sudo apt-get update sudo apt-get install libcurl4-openssl-dev ${{ matrix.build.install_packages }} - name: code style check run: make checksrc - name: make run: make ${{ matrix.build.make_opts }} - name: sanity test run: ./trurl -v - name: test run: make ${{matrix.build.test}} cygwin: runs-on: windows-latest steps: - uses: actions/checkout@v3 - name: install cygwin uses: cygwin/cygwin-install-action@master with: packages: curl, libcurl-devel, libcurl4, make, gcc-core, python39 - name: make run: make - name: sanity test run: ./trurl -v - name: test run: make test macos: runs-on: macos-latest steps: - uses: actions/checkout@v3 - name: make run: make - name: sanity test run: ./trurl -v - name: test run: make test curl-trurl-4f06332/.github/workflows/reuse.yml000066400000000000000000000011251452017613700213460ustar00rootroot00000000000000# Copyright (C) Daniel Stenberg, , et al. # SPDX-FileCopyrightText: 2022 Free Software Foundation Europe e.V. # # SPDX-License-Identifier: curl name: REUSE compliance on: push: branches: - master - '*/ci' pull_request: branches: - master concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true permissions: {} jobs: check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: REUSE Compliance Check uses: fsfe/reuse-action@v1 curl-trurl-4f06332/.gitignore000066400000000000000000000010751452017613700160770ustar00rootroot00000000000000# compiled program /trurl # Prerequisites *.d # Object files *.o *.ko *.obj *.elf # Linker output *.ilk *.map *.exp # Precompiled Headers *.gch *.pch # Libraries *.lib *.a *.la *.lo # Shared objects (inc. Windows DLLs) *.dll *.so *.so.* *.dylib # Executables *.exe *.out *.app *.i*86 *.x86_64 *.hex # Debug files *.dSYM/ *.su *.idb *.pdb # Kernel Module Compile Results *.mod* *.cmd .tmp_versions/ modules.order Module.symvers Mkfile.old dkms.conf # Output files from msvc winbuild/bin/ winbuild/obj/ # Dependencies for msvc from vcpkg winbuild/vcpkg_installed/ curl-trurl-4f06332/.reuse/000077500000000000000000000000001452017613700153055ustar00rootroot00000000000000curl-trurl-4f06332/.reuse/dep5000066400000000000000000000014711452017613700160700ustar00rootroot00000000000000Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: trurl Upstream-Contact: Daniel Stenberg Source: https://curl.se # Tests Files: tests.json testfiles/* Copyright: Daniel Stenberg, , et al. License: curl # Docs Files: CONTRIBUTING.md README.md RELEASE-NOTES THANKS URL-QUIRKS.md Copyright: Daniel Stenberg, , et al. License: curl # Meta files Files: .checksrc .github/workflows/codeql.yml .github/workflows/codespell.yml .github/workflows/makefile.yml .gitignore Copyright: Daniel Stenberg, , et al. License: curl # winbuild files Files: winbuild/.vcpkg winbuild/trurl.sln winbuild/trurl.vcxproj winbuild/vcpkg-configuration.json winbuild/vcpkg.json Copyright: Daniel Stenberg, , et al. License: curl curl-trurl-4f06332/CONTRIBUTING.md000066400000000000000000000116671452017613700163500ustar00rootroot00000000000000# Contributing to trurl This document is intended to provide a framework for contributing to trurl. This document will go over requesting new features, fixing existing bugs and effectively using the internal tooling to help PRs merge quickly. ## Opening an issue trurl uses GitHubs issue tracking to track upcoming work. If you have a feature you want to add or find a bug simply open an issue in the [issues tab](https://github.com/curl/trurl/issues). Briefly describe the feature you are requesting and why you think it may be valuable for trurl. If you are reporting a bug be prepared for questions as we will want to reproduce it locally. In general providing the output of `trurl --version` along with the operating system / Distro you are running is a good starting point. ## Writing a good PR trurl is a relatively straightforward code base, so it is best to keep your PRs straightforward as well. Avoid trying to fix many bugs in one PR, and instead use many smaller PRs as this avoids potential conflicts when merging. trurl is written in C and uses the [curl code style](https://curl.se/dev/code-style.html). PRs that do not follow to code style will not be merged in. trurl is in its early stages, so it's important to open a PR against a recent version of the source code, as a lot can change over a few days. Preferably you would open a PR against the most recent commit in master. If you are implementing a new feature, it must be submitted with tests and documentation. The process for writing tests is explained below in the tooling section. Documentation exists in two locations, the man page ([trurl.1](https://github.com/curl/trurl/blob/master/trurl.1)) and the help prompt when running `trurl -h`. Most documentation changes will go in the man page, but if you add a new command line argument then it must be documented in the help page. It is also important to be prepared for feedback on your PR and adjust it promptly. ## Tooling The trurl repository has a few small helper tools to make development easier. **checksrc.pl** is used to ensure the code style is correct. It accepts C files as command line arguments, and returns nothing if the code style is valid. If the code style is incorrect, checksrc.pl will provide the line the error is on and a brief description of what is wrong. You may run `make checksrc` to scan the entire repository for style compliance. **test.py** is used to run automated tests for trurl. It loads in tests from `test.json` (described below) and reports the number of tests passed. You may specify the tests to run by passing a list of comma-separated numbers as command line arguments, such as `4,8,15,16,23,42` Note there is no space between the numbers. `test.py` may also use valgrind to test for memory errors by passing `--with-valgrind` as a command line argument, it should be noted that this may take a while to run all the tests. `test.py` will also skip tests that require a specific curl runtime or buildtime. ### Adding tests tests are located in [tests.json](https://github.com/curl/trurl/blob/master/tests.json). This file is an array of json objects when outline an input and what the expected output should be. Below is a simple example of a single test: ```json { "input": { "arguments": [ "https://example.com" ] }, "expected": { "stdout": "https://example.com/\n", "stderr": "", "returncode": 0 } } ``` `"arguments"` is an array of the arguments to run in the test, so if you wanted to pass multiple arguments it would look something like: ```json { "input": { "arguments": [ "https://curl.se:22/", "-s", "port=443", "--get", "{url}" ] }, "expected": { "stdout": "https://curl.se/\n", "stderr": "", "returncode": 0 } } ``` trurl may also return json. It you are adding a test that returns json to stdout, write the json directly instead of a string in the examples above. Below is an example of what stdout should be if it is a json test, where `"input"` is what trul accepts from the command line and `"expected"` is what trurl should return. ```json "expected": { "stdout": [ { "url": "https://curl.se/", "scheme": "https", "host": "curl.se", "port": "443", "raw_port": "", "path": "/", "query": "", "params": [] } ], "returncode": 0, "stderr": "" } ``` # Tips to make opening a PR easier - Run `make checksrc` and `make test-memory` locally before opening a PR. These ran automatically when a PR is opened so you might as well make sure they pass before-hand. - Update the man page and the help prompt accordingly. Documentation is annoying but if everyone writes a little it's not bad. - Add tests to cover new features or the bug you fixed. curl-trurl-4f06332/COPYING000066400000000000000000000020141452017613700151340ustar00rootroot00000000000000COPYRIGHT AND PERMISSION NOTICE Copyright (c) 2023, Daniel Stenberg, All rights reserved. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization of the copyright holder. curl-trurl-4f06332/LICENSES/000077500000000000000000000000001452017613700153115ustar00rootroot00000000000000curl-trurl-4f06332/LICENSES/curl.txt000066400000000000000000000020631452017613700170200ustar00rootroot00000000000000COPYRIGHT AND PERMISSION NOTICE Copyright (C) Daniel Stenberg, , and many contributors, see the THANKS file. All rights reserved. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization of the copyright holder. curl-trurl-4f06332/Makefile000066400000000000000000000034531452017613700155510ustar00rootroot00000000000000########################################################################## # _ _ ____ _ # Project ___| | | | _ \| | # / __| | | | |_) | | # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # # Copyright (C) Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms # are also available at https://curl.se/docs/copyright.html. # # You may opt to use, copy, modify, merge, publish, distribute and/or sell # copies of the Software, and permit persons to whom the Software is # furnished to do so, under the terms of the COPYING file. # # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY # KIND, either express or implied. # # SPDX-License-Identifier: curl # ########################################################################## TARGET = trurl OBJS = trurl.o LDLIBS = $$(curl-config --libs) CFLAGS += $$(curl-config --cflags) -W -Wall -Wshadow -Werror -pedantic -g -std=gnu99 MANUAL = trurl.1 PREFIX ?= /usr/local BINDIR ?= $(PREFIX)/bin MANDIR ?= $(PREFIX)/share/man/man1 INSTALL ?= install PYTHON3 ?= python3 $(TARGET): $(OBJS) $(CC) $(OBJS) -o $(TARGET) $(LDLIBS) $(LDFLAGS) trurl.o:trurl.c version.h .PHONY: install install: $(INSTALL) -d $(DESTDIR)$(BINDIR) $(INSTALL) -m 0755 $(TARGET) $(DESTDIR)$(BINDIR) $(INSTALL) -d $(DESTDIR)$(MANDIR) $(INSTALL) -m 0644 $(MANUAL) $(DESTDIR)$(MANDIR) .PHONY: clean clean: rm -f $(OBJS) $(TARGET) .PHONY: test test: $(TARGET) @$(PYTHON3) test.py .PHONY: test-memory test-memory: $(TARGET) @$(PYTHON3) test.py --with-valgrind .PHONY: checksrc checksrc: ./checksrc.pl trurl.c version.h curl-trurl-4f06332/README.md000066400000000000000000000070771452017613700153760ustar00rootroot00000000000000# trurl Command line tool for URL parsing and manipulation [Video presentation](https://youtu.be/oDL7DVszr2w) ## Examples **Replace the host name of a URL:** ```text $ trurl --url https://curl.se --set host=example.com https://example.com/ ``` **Create a URL by setting components:** ```text $ trurl --set host=example.com --set scheme=ftp ftp://example.com/ ``` **Redirect a URL:** ```text $ trurl --url https://curl.se/we/are.html --redirect here.html https://curl.se/we/here.html ``` **Change port number:** ```text $ trurl --url https://curl.se/we/../are.html --set port=8080 https://curl.se:8080/are.html ``` **Extract the path from a URL:** ```text $ trurl --url https://curl.se/we/are.html --get '{path}' /we/are.html ``` **Extract the port from a URL:** ```text $ trurl --url https://curl.se/we/are.html --get '{port}' 443 ``` **Append a path segment to a URL:** ```text $ trurl --url https://curl.se/hello --append path=you https://curl.se/hello/you ``` **Append a query segment to a URL:** ```text $ trurl --url "https://curl.se?name=hello" --append query=search=string https://curl.se/?name=hello&search=string ``` **Read URLs from stdin:** ```text $ cat urllist.txt | trurl --url-file - ... ``` **Output JSON:** ```text $ trurl "https://fake.host/hello#frag" --set user=::moo:: --json [ { "url": "https://%3a%3amoo%3a%3a@fake.host/hello#frag", "parts": { "scheme": "https", "user": "::moo::", "host": "fake.host", "path": "/hello", "fragment": "frag" } } ] ``` **Remove tracking tuples from query:** ```text $ trurl "https://curl.se?search=hey&utm_source=tracker" --trim query="utm_*" https://curl.se/?search=hey ``` **Show a specific query key value:** ```text $ trurl "https://example.com?a=home&here=now&thisthen" -g '{query:a}' home ``` **Sort the key/value pairs in the query component:** ```text $ trurl "https://example.com?b=a&c=b&a=c" --sort-query https://example.com?a=c&b=a&c=b ``` **Work with a query that uses a semicolon separator:** ```text $ trurl "https://curl.se?search=fool;page=5" --trim query="search" --query-separator ";" https://curl.se?page=5 ``` **Accept spaces in the URL path:** ```text $ trurl "https://curl.se/this has space/index.html" --accept-space https://curl.se/this%20has%20space/index.html ``` ## Install ### Linux It's quite easy to compile the C source with GCC: ```text $ make cc -W -Wall -pedantic -g -c -o trurl.o trurl.c cc trurl.o -lcurl -o trurl ``` trurl is also available in [some Linux distributions](https://github.com/curl/trurl/discussions/51). You can try searching for it using the package manager of your preferred distribution. ### Windows 1. Download and run [Cygwin installer.](https://www.cygwin.com/install.html) 2. Follow the instructions provided by the installer. When prompted to select packages, make sure to choose the following: curl, libcurl-devel, libcurl4, make and gcc-core. 3. (optional) Add the Cygwin bin directory to your system PATH variable. 4. Use `make`, just like on Linux. ## Prerequisites Development files of libcurl (e.g. `libcurl4-openssl-dev` or `libcurl4-gnutls-dev`) are needed for compilation. Requires libcurl version 7.62.0 or newer (the first libcurl to ship the URL parsing API). trurl also uses `CURLUPART_ZONEID` added in libcurl 7.81.0 and `curl_url_strerror()` added in libcurl 7.80.0 It would certainly be possible to make trurl work with older libcurl versions if someone wanted to. ### Older libcurls trurl builds with libcurl older than 7.81.0 but will then not work as good. For all the documented goodness, use a more modern libcurl. curl-trurl-4f06332/RELEASE-NOTES000066400000000000000000000007751452017613700160060ustar00rootroot00000000000000trurl 0.9 Changes since previous release o add --as-idn and punycode to IDN conversion o add --curl to only count as valid URLs supported by libcurl o add vs2022 project files Bugfixes since previous release o accept \* as a trim name to trim a literal asterisk name o format null as \u0000 for --json o run --trim query before --append query Contributors to this release: Daniel Stenberg, Ehsan, Emanuele Torre, Jacob Mealey, Krishean Draconis, Michael Ablassmeier, 積丹尼 Dan Jacobson curl-trurl-4f06332/THANKS000066400000000000000000000006021452017613700150150ustar00rootroot00000000000000This project exists only thanks to the awesome people who make it happen. The following friends have contributed: Dan Fandrich Daniel Stenberg Emanuele Torre Enno Tensing Gustavo Costa Håvard Bønes Jacob Mealey Jeremy Lecour Krishean Draconis Luca Barbato ma Marian Posaceanu Martin Hauke Michael Ablassmeier Nekobit Olaf Alders Pascal Knecht Paul Roub Ruoyu Zhong Sajad F. Maghrebi curl-trurl-4f06332/URL-QUIRKS.md000066400000000000000000000034771452017613700161170ustar00rootroot00000000000000# URL Quirks This is a collection peculiarities to may find in trurl due to bugs or changes/improvements in libcurl's URL handling. ## The URL API Was introduced in libcurl 7.62.0. No older libcurl versions can be used. Build-time requirement. ## Extracting zone id Added in libcurl 7.65.0. The `CURLUE_NO_ZONEID` error code was added in 7.81.0. Build-time requirement. ## Normalizing IPv4 addresses Added in libcurl 7.77.0. Before that, the source formatting was kept. Run-time requirement. ## Allow space The libcurl URL parser was given the ability to allow spaces in libcurl 7.78.0. trurl therefore cannot offer this feature with older libcurl versions. Build-time requirement. ## `curl_url_strerror()` This API call was added in 7.80.0, using a libcurl version older than this will make trurl output less good error messages. Build-time requirement. ## Normalizing IPv6 addresses Implemented in libcurl 7.81.0. Before this, the source formatting was kept. Run-time requirement. ## `CURLU_PUNYCODE` Added in libcurl 7.88.0. Build-time requirement. ## Accepting % in host names The host name parser has been made stricter over time, with the most recent enhancement merged for libcurl 8.0.0. Run-time requirement. ## Parsing IPv6 literals when libcurl does not support IPv6 Before libcurl 8.0.0 the URL parser was not able to parse IPv6 addresses if libcurl itself was built without IPv6 capabilities. Run-time requirement. ## URL encoding of fragments This was a libcurl bug, fixed in libcurl 8.1.0 Run-time requirement. ## Bad IPv4 numerical address The normalization of IPv4 addresses would just ignore bad addresses, while newer libcurl versions will reject host names using invalid IPv4 addresses. Fixed in 8.1.0 Run-time requirement. ## Set illegal scheme Permitted before libcurl 8.1.0 Run-time requirement. curl-trurl-4f06332/checksrc.pl000077500000000000000000000774461452017613700162530ustar00rootroot00000000000000#!/usr/bin/env perl #*************************************************************************** # _ _ ____ _ # Project ___| | | | _ \| | # / __| | | | |_) | | # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # # Copyright (C) Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms # are also available at https://curl.se/docs/copyright.html. # # You may opt to use, copy, modify, merge, publish, distribute and/or sell # copies of the Software, and permit persons to whom the Software is # furnished to do so, under the terms of the COPYING file. # # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY # KIND, either express or implied. # # SPDX-License-Identifier: curl # ########################################################################### use strict; use warnings; my $max_column = 79; my $indent = 2; my $warnings = 0; my $swarnings = 0; my $errors = 0; my $serrors = 0; my $suppressed; # skipped problems my $file; my $dir="."; my $wlist=""; my @alist; my $windows_os = $^O eq 'MSWin32' || $^O eq 'cygwin' || $^O eq 'msys'; my $verbose; my %skiplist; my %ignore; my %ignore_set; my %ignore_used; my @ignore_line; my %warnings_extended = ( 'COPYRIGHTYEAR' => 'copyright year incorrect', 'STRERROR', => 'strerror() detected', ); my %warnings = ( 'ASSIGNWITHINCONDITION' => 'assignment within conditional expression', 'ASTERISKNOSPACE' => 'pointer declared without space before asterisk', 'ASTERISKSPACE' => 'pointer declared with space after asterisk', 'BADCOMMAND' => 'bad !checksrc! instruction', 'BANNEDFUNC' => 'a banned function was used', 'BRACEELSE' => '} else on the same line', 'BRACEPOS' => 'wrong position for an open brace', 'BRACEWHILE' => 'A single space between open brace and while', 'COMMANOSPACE' => 'comma without following space', 'COMMENTNOSPACEEND' => 'no space before */', 'COMMENTNOSPACESTART' => 'no space following /*', 'COPYRIGHT' => 'file missing a copyright statement', 'CPPCOMMENTS' => '// comment detected', 'DOBRACE' => 'A single space between do and open brace', 'EMPTYLINEBRACE' => 'Empty line before the open brace', 'EQUALSNOSPACE' => 'equals sign without following space', 'EQUALSNULL' => 'if/while comparison with == NULL', 'EXCLAMATIONSPACE' => 'Whitespace after exclamation mark in expression', 'FOPENMODE' => 'fopen needs a macro for the mode string', 'INCLUDEDUP', => 'same file is included again', 'INDENTATION' => 'wrong start column for code', 'LONGLINE' => "Line longer than $max_column", 'MULTISPACE' => 'multiple spaces used when not suitable', 'NOSPACEEQUALS' => 'equals sign without preceding space', 'NOTEQUALSZERO', => 'if/while comparison with != 0', 'ONELINECONDITION' => 'conditional block on the same line as the if()', 'OPENCOMMENT' => 'file ended with a /* comment still "open"', 'PARENBRACE' => '){ without sufficient space', 'RETURNNOSPACE' => 'return without space', 'SEMINOSPACE' => 'semicolon without following space', 'SIZEOFNOPAREN' => 'use of sizeof without parentheses', 'SNPRINTF' => 'use of snprintf', 'SPACEAFTERPAREN' => 'space after open parenthesis', 'SPACEBEFORECLOSE' => 'space before a close parenthesis', 'SPACEBEFORECOMMA' => 'space before a comma', 'SPACEBEFOREPAREN' => 'space before an open parenthesis', 'SPACESEMICOLON' => 'space before semicolon', 'SPACESWITCHCOLON' => 'space before colon of switch label', 'TABS' => 'TAB characters not allowed', 'TRAILINGSPACE' => 'Trailing whitespace on the line', 'TYPEDEFSTRUCT' => 'typedefed struct', 'UNUSEDIGNORE' => 'a warning ignore was not used', ); sub readskiplist { open(my $W, '<', "$dir/checksrc.skip") or return; my @all=<$W>; for(@all) { $windows_os ? $_ =~ s/\r?\n$// : chomp; $skiplist{$_}=1; } close($W); } # Reads the .checksrc in $dir for any extended warnings to enable locally. # Currently there is no support for disabling warnings from the standard set, # and since that's already handled via !checksrc! commands there is probably # little use to add it. sub readlocalfile { my $i = 0; open(my $rcfile, "<", "$dir/.checksrc") or return; while(<$rcfile>) { $i++; # Lines starting with '#' are considered comments if (/^\s*(#.*)/) { next; } elsif (/^\s*enable ([A-Z]+)$/) { if(!defined($warnings_extended{$1})) { print STDERR "invalid warning specified in .checksrc: \"$1\"\n"; next; } $warnings{$1} = $warnings_extended{$1}; } elsif (/^\s*disable ([A-Z]+)$/) { if(!defined($warnings{$1})) { print STDERR "invalid warning specified in .checksrc: \"$1\"\n"; next; } # Accept-list push @alist, $1; } else { die "Invalid format in $dir/.checksrc on line $i\n"; } } close($rcfile); } sub checkwarn { my ($name, $num, $col, $file, $line, $msg, $error) = @_; my $w=$error?"error":"warning"; my $nowarn=0; #if(!$warnings{$name}) { # print STDERR "Dev! there's no description for $name!\n"; #} # checksrc.skip if($skiplist{$line}) { $nowarn = 1; } # !checksrc! controlled elsif($ignore{$name}) { $ignore{$name}--; $ignore_used{$name}++; $nowarn = 1; if(!$ignore{$name}) { # reached zero, enable again enable_warn($name, $num, $file, $line); } } if($nowarn) { $suppressed++; if($w) { $swarnings++; } else { $serrors++; } return; } if($w) { $warnings++; } else { $errors++; } $col++; print "$file:$num:$col: $w: $msg ($name)\n"; print " $line\n"; if($col < 80) { my $pref = (' ' x $col); print "${pref}^\n"; } } $file = shift @ARGV; while(defined $file) { if($file =~ /-D(.*)/) { $dir = $1; $file = shift @ARGV; next; } elsif($file =~ /-W(.*)/) { $wlist .= " $1 "; $file = shift @ARGV; next; } elsif($file =~ /-A(.+)/) { push @alist, $1; $file = shift @ARGV; next; } elsif($file =~ /-i([1-9])/) { $indent = $1 + 0; $file = shift @ARGV; next; } elsif($file =~ /-m([0-9]+)/) { $max_column = $1 + 0; $file = shift @ARGV; next; } elsif($file =~ /^(-h|--help)/) { undef $file; last; } last; } if(!$file) { print "checksrc.pl [option] [file2] ...\n"; print " Options:\n"; print " -A[rule] Accept this violation, can be used multiple times\n"; print " -D[DIR] Directory to prepend file names\n"; print " -h Show help output\n"; print " -W[file] Skip the given file - ignore all its flaws\n"; print " -i Indent spaces. Default: 2\n"; print " -m Maximum line length. Default: 79\n"; print "\nDetects and warns for these problems:\n"; my @allw = keys %warnings; push @allw, keys %warnings_extended; for my $w (sort @allw) { if($warnings{$w}) { printf (" %-18s: %s\n", $w, $warnings{$w}); } else { printf (" %-18s: %s[*]\n", $w, $warnings_extended{$w}); } } print " [*] = disabled by default\n"; exit; } readskiplist(); readlocalfile(); do { if("$wlist" !~ / $file /) { my $fullname = $file; $fullname = "$dir/$file" if ($fullname !~ '^\.?\.?/'); scanfile($fullname); } $file = shift @ARGV; } while($file); sub accept_violations { for my $r (@alist) { if(!$warnings{$r}) { print "'$r' is not a warning to accept!\n"; exit; } $ignore{$r}=999999; $ignore_used{$r}=0; } } sub checksrc_clear { undef %ignore; undef %ignore_set; undef @ignore_line; } sub checksrc_endoffile { my ($file) = @_; for(keys %ignore_set) { if($ignore_set{$_} && !$ignore_used{$_}) { checkwarn("UNUSEDIGNORE", $ignore_set{$_}, length($_)+11, $file, $ignore_line[$ignore_set{$_}], "Unused ignore: $_"); } } } sub enable_warn { my ($what, $line, $file, $l) = @_; # switch it back on, but warn if not triggered! if(!$ignore_used{$what}) { checkwarn("UNUSEDIGNORE", $line, length($what) + 11, $file, $l, "No warning was inhibited!"); } $ignore_set{$what}=0; $ignore_used{$what}=0; $ignore{$what}=0; } sub checksrc { my ($cmd, $line, $file, $l) = @_; if($cmd =~ / *([^ ]*) *(.*)/) { my ($enable, $what) = ($1, $2); $what =~ s: *\*/$::; # cut off end of C comment # print "ENABLE $enable WHAT $what\n"; if($enable eq "disable") { my ($warn, $scope)=($1, $2); if($what =~ /([^ ]*) +(.*)/) { ($warn, $scope)=($1, $2); } else { $warn = $what; $scope = 1; } # print "IGNORE $warn for SCOPE $scope\n"; if($scope eq "all") { $scope=999999; } # Comparing for a literal zero rather than the scalar value zero # covers the case where $scope contains the ending '*' from the # comment. If we use a scalar comparison (==) we induce warnings # on non-scalar contents. if($scope eq "0") { checkwarn("BADCOMMAND", $line, 0, $file, $l, "Disable zero not supported, did you mean to enable?"); } elsif($ignore_set{$warn}) { checkwarn("BADCOMMAND", $line, 0, $file, $l, "$warn already disabled from line $ignore_set{$warn}"); } else { $ignore{$warn}=$scope; $ignore_set{$warn}=$line; $ignore_line[$line]=$l; } } elsif($enable eq "enable") { enable_warn($what, $line, $file, $l); } else { checkwarn("BADCOMMAND", $line, 0, $file, $l, "Illegal !checksrc! command"); } } } sub nostrings { my ($str) = @_; $str =~ s/\".*\"//g; return $str; } sub scanfile { my ($file) = @_; my $line = 1; my $prevl=""; my $prevpl=""; my $l = ""; my $prep = 0; my $prevp = 0; open(my $R, '<', $file) || die "failed to open $file"; my $incomment=0; my @copyright=(); my %includes; checksrc_clear(); # for file based ignores accept_violations(); while(<$R>) { $windows_os ? $_ =~ s/\r?\n$// : chomp; my $l = $_; my $ol = $l; # keep the unmodified line for error reporting my $column = 0; # check for !checksrc! commands if($l =~ /\!checksrc\! (.*)/) { my $cmd = $1; checksrc($cmd, $line, $file, $l) } # check for a copyright statement and save the years if($l =~ /\* +copyright .* (\d\d\d\d|)/i) { my $count = 0; while($l =~ /([\d]{4})/g) { push @copyright, { year => $1, line => $line, col => index($l, $1), code => $l }; $count++; } if(!$count) { # year-less push @copyright, { year => -1, line => $line, col => index($l, $1), code => $l }; } } # detect long lines if(length($l) > $max_column) { checkwarn("LONGLINE", $line, length($l), $file, $l, "Longer than $max_column columns"); } # detect TAB characters if($l =~ /^(.*)\t/) { checkwarn("TABS", $line, length($1), $file, $l, "Contains TAB character", 1); } # detect trailing whitespace if($l =~ /^(.*)[ \t]+\z/) { checkwarn("TRAILINGSPACE", $line, length($1), $file, $l, "Trailing whitespace"); } # no space after comment start if($l =~ /^(.*)\/\*\w/) { checkwarn("COMMENTNOSPACESTART", $line, length($1) + 2, $file, $l, "Missing space after comment start"); } # no space at comment end if($l =~ /^(.*)\w\*\//) { checkwarn("COMMENTNOSPACEEND", $line, length($1) + 1, $file, $l, "Missing space end comment end"); } # ------------------------------------------------------------ # Above this marker, the checks were done on lines *including* # comments # ------------------------------------------------------------ # strip off C89 comments comment: if(!$incomment) { if($l =~ s/\/\*.*\*\// /g) { # full /* comments */ were removed! } if($l =~ s/\/\*.*//) { # start of /* comment was removed $incomment = 1; } } else { if($l =~ s/.*\*\///) { # end of comment */ was removed $incomment = 0; goto comment; } else { # still within a comment $l=""; } } # ------------------------------------------------------------ # Below this marker, the checks were done on lines *without* # comments # ------------------------------------------------------------ # prev line was a preprocessor **and** ended with a backslash if($prep && ($prevpl =~ /\\ *\z/)) { # this is still a preprocessor line $prep = 1; goto preproc; } $prep = 0; # crude attempt to detect // comments without too many false # positives if($l =~ /^(([^"\*]*)[^:"]|)\/\//) { checkwarn("CPPCOMMENTS", $line, length($1), $file, $l, "\/\/ comment"); } if($l =~ /^(\#\s*include\s+)([\">].*[>}"])/) { my ($pre, $path) = ($1, $2); if($includes{$path}) { checkwarn("INCLUDEDUP", $line, length($1), $file, $l, "duplicated include"); } $includes{$path} = $l; } # detect and strip preprocessor directives if($l =~ /^[ \t]*\#/) { # preprocessor line $prep = 1; goto preproc; } my $nostr = nostrings($l); # check spaces after for/if/while/function call if($nostr =~ /^(.*)(for|if|while|switch| ([a-zA-Z0-9_]+)) \((.)/) { my ($leading, $word, $extra, $first)=($1,$2,$3,$4); if($1 =~ / *\#/) { # this is a #if, treat it differently } elsif(defined $3 && $3 eq "return") { # return must have a space } elsif(defined $3 && $3 eq "case") { # case must have a space } elsif(($first eq "*") && ($word !~ /(for|if|while|switch)/)) { # A "(*" beginning makes the space OK because it wants to # allow function pointer declared } elsif($1 =~ / *typedef/) { # typedefs can use space-paren } else { checkwarn("SPACEBEFOREPAREN", $line, length($leading)+length($word), $file, $l, "$word with space"); } } # check for '== NULL' in if/while conditions but not if the thing on # the left of it is a function call if($nostr =~ /^(.*)(if|while)(\(.*?)([!=]= NULL|NULL [!=]=)/) { checkwarn("EQUALSNULL", $line, length($1) + length($2) + length($3), $file, $l, "we prefer !variable instead of \"== NULL\" comparisons"); } # check for '!= 0' in if/while conditions but not if the thing on # the left of it is a function call if($nostr =~ /^(.*)(if|while)(\(.*[^)]) != 0[^x]/) { checkwarn("NOTEQUALSZERO", $line, length($1) + length($2) + length($3), $file, $l, "we prefer if(rc) instead of \"rc != 0\" comparisons"); } # check spaces in 'do {' if($nostr =~ /^( *)do( *)\{/ && length($2) != 1) { checkwarn("DOBRACE", $line, length($1) + 2, $file, $l, "one space after do before brace"); } # check spaces in 'do {' elsif($nostr =~ /^( *)\}( *)while/ && length($2) != 1) { checkwarn("BRACEWHILE", $line, length($1) + 2, $file, $l, "one space between brace and while"); } if($nostr =~ /^((.*\s)(if) *\()(.*)\)(.*)/) { my $pos = length($1); my $postparen = $5; my $cond = $4; if($cond =~ / = /) { checkwarn("ASSIGNWITHINCONDITION", $line, $pos+1, $file, $l, "assignment within conditional expression"); } my $temp = $cond; $temp =~ s/\(//g; # remove open parens my $openc = length($cond) - length($temp); $temp = $cond; $temp =~ s/\)//g; # remove close parens my $closec = length($cond) - length($temp); my $even = $openc == $closec; if($l =~ / *\#/) { # this is a #if, treat it differently } elsif($even && $postparen && ($postparen !~ /^ *$/) && ($postparen !~ /^ *[,{&|\\]+/)) { checkwarn("ONELINECONDITION", $line, length($l)-length($postparen), $file, $l, "conditional block on the same line"); } } # check spaces after open parentheses if($l =~ /^(.*[a-z])\( /i) { checkwarn("SPACEAFTERPAREN", $line, length($1)+1, $file, $l, "space after open parenthesis"); } # check spaces before close parentheses, unless it was a space or a # close parenthesis! if($l =~ /(.*[^\) ]) \)/) { checkwarn("SPACEBEFORECLOSE", $line, length($1)+1, $file, $l, "space before close parenthesis"); } # check spaces before comma! if($l =~ /(.*[^ ]) ,/) { checkwarn("SPACEBEFORECOMMA", $line, length($1)+1, $file, $l, "space before comma"); } # check for "return(" without space if($l =~ /^(.*)return\(/) { if($1 =~ / *\#/) { # this is a #if, treat it differently } else { checkwarn("RETURNNOSPACE", $line, length($1)+6, $file, $l, "return without space before paren"); } } # check for "sizeof" without parenthesis if(($l =~ /^(.*)sizeof *([ (])/) && ($2 ne "(")) { if($1 =~ / *\#/) { # this is a #if, treat it differently } else { checkwarn("SIZEOFNOPAREN", $line, length($1)+6, $file, $l, "sizeof without parenthesis"); } } # check for comma without space if($l =~ /^(.*),[^ \n]/) { my $pref=$1; my $ign=0; if($pref =~ / *\#/) { # this is a #if, treat it differently $ign=1; } elsif($pref =~ /\/\*/) { # this is a comment $ign=1; } elsif($pref =~ /[\"\']/) { $ign = 1; # There is a quote here, figure out whether the comma is # within a string or '' or not. if($pref =~ /\"/) { # within a string } elsif($pref =~ /\'$/) { # a single letter } else { $ign = 0; } } if(!$ign) { checkwarn("COMMANOSPACE", $line, length($pref)+1, $file, $l, "comma without following space"); } } # check for "} else" if($l =~ /^(.*)\} *else/) { checkwarn("BRACEELSE", $line, length($1), $file, $l, "else after closing brace on same line"); } # check for "){" if($l =~ /^(.*)\)\{/) { checkwarn("PARENBRACE", $line, length($1)+1, $file, $l, "missing space after close paren"); } # check for "^{" with an empty line before it if(($l =~ /^\{/) && ($prevl =~ /^[ \t]*\z/)) { checkwarn("EMPTYLINEBRACE", $line, 0, $file, $l, "empty line before open brace"); } # check for space before the semicolon last in a line if($l =~ /^(.*[^ ].*) ;$/) { checkwarn("SPACESEMICOLON", $line, length($1), $file, $ol, "no space before semicolon"); } # check for space before the colon in a switch label if($l =~ /^( *(case .+|default)) :/) { checkwarn("SPACESWITCHCOLON", $line, length($1), $file, $ol, "no space before colon of switch label"); } # scan for use of banned functions if($l =~ /^(.*\W) (gmtime|localtime| gets| strtok| v?sprintf| (str|_mbs|_tcs|_wcs)n?cat| LoadLibrary(Ex)?(A|W)?) \s*\( /x) { checkwarn("BANNEDFUNC", $line, length($1), $file, $ol, "use of $2 is banned"); } if($warnings{"STRERROR"}) { # scan for use of banned strerror. This is not a BANNEDFUNC to # allow for individual enable/disable of this warning. if($l =~ /^(.*\W)(strerror)\s*\(/x) { if($1 !~ /^ *\#/) { # skip preprocessor lines checkwarn("STRERROR", $line, length($1), $file, $ol, "use of $2 is banned"); } } } # scan for use of snprintf for curl-internals reasons if($l =~ /^(.*\W)(v?snprintf)\s*\(/x) { checkwarn("SNPRINTF", $line, length($1), $file, $ol, "use of $2 is banned"); } # scan for use of non-binary fopen without the macro if($l =~ /^(.*\W)fopen\s*\([^,]*, *\"([^"]*)/) { my $mode = $2; if($mode !~ /b/) { checkwarn("FOPENMODE", $line, length($1), $file, $ol, "use of non-binary fopen without FOPEN_* macro: $mode"); } } # check for open brace first on line but not first column only alert # if previous line ended with a close paren and it wasn't a cpp line if(($prevl =~ /\)\z/) && ($l =~ /^( +)\{/) && !$prevp) { checkwarn("BRACEPOS", $line, length($1), $file, $ol, "badly placed open brace"); } # if the previous line starts with if/while/for AND ends with an open # brace, or an else statement, check that this line is indented $indent # more steps, if not a cpp line if(!$prevp && ($prevl =~ /^( *)((if|while|for)\(.*\{|else)\z/)) { my $first = length($1); # this line has some character besides spaces if($l =~ /^( *)[^ ]/) { my $second = length($1); my $expect = $first+$indent; if($expect != $second) { my $diff = $second - $first; checkwarn("INDENTATION", $line, length($1), $file, $ol, "not indented $indent steps (uses $diff)"); } } } # if the previous line starts with if/while/for AND ends with a closed # parenthesis and there's an equal number of open and closed # parentheses, check that this line is indented $indent more steps, if # not a cpp line elsif(!$prevp && ($prevl =~ /^( *)(if|while|for)(\(.*\))\z/)) { my $first = length($1); my $op = $3; my $cl = $3; $op =~ s/[^(]//g; $cl =~ s/[^)]//g; if(length($op) == length($cl)) { # this line has some character besides spaces if($l =~ /^( *)[^ ]/) { my $second = length($1); my $expect = $first+$indent; if($expect != $second) { my $diff = $second - $first; checkwarn("INDENTATION", $line, length($1), $file, $ol, "not indented $indent steps (uses $diff)"); } } } } # check for 'char * name' if(($l =~ /(^.*(char|int|long|void|CURL|CURLM|CURLMsg|[cC]url_[A-Za-z_]+|struct [a-zA-Z_]+) *(\*+)) (\w+)/) && ($4 !~ /^(const|volatile)$/)) { checkwarn("ASTERISKSPACE", $line, length($1), $file, $ol, "space after declarative asterisk"); } # check for 'char*' if(($l =~ /(^.*(char|int|long|void|curl_slist|CURL|CURLM|CURLMsg|curl_httppost|sockaddr_in|FILE)\*)/)) { checkwarn("ASTERISKNOSPACE", $line, length($1)-1, $file, $ol, "no space before asterisk"); } # check for 'void func() {', but avoid false positives by requiring # both an open and closed parentheses before the open brace if($l =~ /^((\w).*)\{\z/) { my $k = $1; $k =~ s/const *//; $k =~ s/static *//; if($k =~ /\(.*\)/) { checkwarn("BRACEPOS", $line, length($l)-1, $file, $ol, "wrongly placed open brace"); } } # check for equals sign without spaces next to it if($nostr =~ /(.*)\=[a-z0-9]/i) { checkwarn("EQUALSNOSPACE", $line, length($1)+1, $file, $ol, "no space after equals sign"); } # check for equals sign without spaces before it elsif($nostr =~ /(.*)[a-z0-9]\=/i) { checkwarn("NOSPACEEQUALS", $line, length($1)+1, $file, $ol, "no space before equals sign"); } # check for plus signs without spaces next to it if($nostr =~ /(.*)[^+]\+[a-z0-9]/i) { checkwarn("PLUSNOSPACE", $line, length($1)+1, $file, $ol, "no space after plus sign"); } # check for plus sign without spaces before it elsif($nostr =~ /(.*)[a-z0-9]\+[^+]/i) { checkwarn("NOSPACEPLUS", $line, length($1)+1, $file, $ol, "no space before plus sign"); } # check for semicolons without space next to it if($nostr =~ /(.*)\;[a-z0-9]/i) { checkwarn("SEMINOSPACE", $line, length($1)+1, $file, $ol, "no space after semicolon"); } # typedef struct ... { if($nostr =~ /^(.*)typedef struct.*{/) { checkwarn("TYPEDEFSTRUCT", $line, length($1)+1, $file, $ol, "typedef'ed struct"); } if($nostr =~ /(.*)! +(\w|\()/) { checkwarn("EXCLAMATIONSPACE", $line, length($1)+1, $file, $ol, "space after exclamation mark"); } # check for more than one consecutive space before open brace or # question mark. Skip lines containing strings since they make it hard # due to artificially getting multiple spaces if(($l eq $nostr) && $nostr =~ /^(.*(\S)) + [{?]/i) { checkwarn("MULTISPACE", $line, length($1)+1, $file, $ol, "multiple spaces"); } preproc: $line++; $prevp = $prep; $prevl = $ol if(!$prep); $prevpl = $ol if($prep); } if(!scalar(@copyright)) { checkwarn("COPYRIGHT", 1, 0, $file, "", "Missing copyright statement", 1); } # COPYRIGHTYEAR is an extended warning so we must first see if it has been # enabled in .checksrc if(defined($warnings{"COPYRIGHTYEAR"})) { # The check for updated copyrightyear is overly complicated in order to # not punish current hacking for past sins. The copyright years are # right now a bit behind, so enforcing copyright year checking on all # files would cause hundreds of errors. Instead we only look at files # which are tracked in the Git repo and edited in the workdir, or # committed locally on the branch without being in upstream master. # # The simple and naive test is to simply check for the current year, # but updating the year even without an edit is against project policy # (and it would fail every file on January 1st). # # A rather more interesting, and correct, check would be to not test # only locally committed files but inspect all files wrt the year of # their last commit. Removing the `git rev-list origin/master..HEAD` # condition below will enforce copyright year checks against the year # the file was last committed (and thus edited to some degree). my $commityear = undef; @copyright = sort {$$b{year} cmp $$a{year}} @copyright; # if the file is modified, assume commit year this year if(`git status -s -- $file` =~ /^ [MARCU]/) { $commityear = (localtime(time))[5] + 1900; } else { # min-parents=1 to ignore wrong initial commit in truncated repos my $grl = `git rev-list --max-count=1 --min-parents=1 --timestamp HEAD -- $file`; if($grl) { chomp $grl; $commityear = (localtime((split(/ /, $grl))[0]))[5] + 1900; } } if(defined($commityear) && scalar(@copyright) && $copyright[0]{year} != $commityear) { checkwarn("COPYRIGHTYEAR", $copyright[0]{line}, $copyright[0]{col}, $file, $copyright[0]{code}, "Copyright year out of date, should be $commityear, " . "is $copyright[0]{year}", 1); } } if($incomment) { checkwarn("OPENCOMMENT", 1, 0, $file, "", "Missing closing comment", 1); } checksrc_endoffile($file); close($R); } if($errors || $warnings || $verbose) { printf "checksrc: %d errors and %d warnings\n", $errors, $warnings; if($suppressed) { printf "checksrc: %d errors and %d warnings suppressed\n", $serrors, $swarnings; } exit 5; # return failure } curl-trurl-4f06332/test.py000066400000000000000000000175631452017613700154510ustar00rootroot00000000000000#!/usr/bin/env python3 ########################################################################## # _ _ ____ _ # Project ___| | | | _ \| | # / __| | | | |_) | | # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # # Copyright (C) Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms # are also available at https://curl.se/docs/copyright.html. # # You may opt to use, copy, modify, merge, publish, distribute and/or sell # copies of the Software, and permit persons to whom the Software is # furnished to do so, under the terms of the COPYING file. # # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY # KIND, either express or implied. # # SPDX-License-Identifier: curl # ########################################################################## import sys from os import getcwd, path import json import shlex from subprocess import PIPE, run from dataclasses import dataclass, asdict from typing import Any, Optional, TextIO PROGNAME = "trurl" TESTFILE = "tests.json" VALGRINDTEST = "valgrind" VALGRINDARGS = ["--error-exitcode=1", "--leak-check=full", "-q"] RED = "\033[91m" # used to mark unsuccessful tests NOCOLOR = "\033[0m" @dataclass class CommandOutput: stdout: Any returncode: int stderr: str def testComponent(value, exp): if isinstance(exp, bool): result = value == 0 or value not in ("", []) if exp: return result else: return not result return value == exp class TestCase: def __init__(self, testIndex, baseCmd, **testCase): self.testIndex = testIndex self.baseCmd = baseCmd self.arguments = testCase["input"]["arguments"] self.expected = testCase["expected"] self.commandOutput: CommandOutput = None self.testPassed: bool = False def runCommand(self, cmdfilter: Optional[str], runWithValgrind: bool): # Skip test if none of the arguments contain the keyword if cmdfilter and all(cmdfilter not in arg for arg in self.arguments): return False cmd = [self.baseCmd] args = self.arguments if runWithValgrind: cmd = [VALGRINDTEST] args = VALGRINDARGS + [self.baseCmd] + self.arguments output = run( cmd + args, stdout=PIPE, stderr=PIPE, encoding="utf-8" ) if isinstance(self.expected["stdout"], list): # if we don't expect string, parse to json try: stdout = json.loads(output.stdout) except json.decoder.JSONDecodeError: stdout = None else: stdout = output.stdout # assume stderr is always going to be string stderr = output.stderr self.commandOutput = CommandOutput(stdout, output.returncode, stderr) return True def test(self): # return true only if stdout, stderr and errorcode # are equal to the ones found in the testfile self.testPassed = all( testComponent(asdict(self.commandOutput)[k], exp) for k, exp in self.expected.items()) return self.testPassed def _printVerbose(self, output: TextIO): self._printConcise(output) for component, exp in self.expected.items(): value = asdict(self.commandOutput)[component] itemFail = self.commandOutput.returncode == 1 or \ not testComponent(value, exp) print(f"--- {component} --- ", file=output) print("expected:", file=output) print("nothing" if exp is False else "something" if exp is True else f"{exp!r}",file=output) print("got:", file=output) header = RED if itemFail else "" footer = NOCOLOR if itemFail else "" print(f"{header}{value!r}{footer}", file=output) print() def _printConcise(self, output: TextIO): if self.testPassed: header = "" result = "passed" footer = "" else: header = RED result = "failed" footer = NOCOLOR text = f"{self.testIndex}: {result}\t{shlex.join(self.arguments)}" print(f"{header}{text}{footer}", file=output) def printDetail(self, verbose: bool = False, failed: bool = False): output: TextIO = sys.stderr if failed else sys.stdout if verbose: self._printVerbose(output) else: self._printConcise(output) def main(argc, argv): ret = 0 baseDir = path.dirname(path.realpath(argv[0])) # python on windows does not always seem to find the # executable if it is in a different output directory than # the python script, even if it is in the current working # directory, using absolute paths to the executable and json # file makes it reliably find the executable baseCmd = path.join(getcwd(), PROGNAME) # the .exe on the end is necessary when using absolute paths if sys.platform == "win32" or sys.platform == "cygwin": baseCmd += ".exe" # check if the trurl executable exists if path.isfile(baseCmd): # get the version info for the feature list output = run( [baseCmd, "--version"], stdout=PIPE, stderr=PIPE, encoding="utf-8" ) features = output.stdout.split('\n')[1].split()[1:] with open(path.join(baseDir, TESTFILE), "r") as file: allTests = json.load(file) testIndexesToRun = [] # if argv[1] exists and starts with int cmdfilter = "" testIndexesToRun = list(range(len(allTests))) runWithValgrind = False verboseDetail = False if argc > 1: for arg in argv[1:]: if arg[0].isnumeric(): # run only test cases separated by "," testIndexesToRun = [] for caseIndex in arg.split(","): testIndexesToRun.append(int(caseIndex)) elif arg == "--with-valgrind": runWithValgrind = True elif arg == "--verbose": verboseDetail = True else: cmdfilter = argv[1] numTestsFailed = 0 numTestsPassed = 0 numTestsSkipped = 0 for testIndex in testIndexesToRun: # skip tests if required features are not met if "required" in allTests[testIndex]: required = allTests[testIndex]["required"] if not set(required).issubset(set(features)): print(f"Missing feature, skipping test {testIndex + 1}.") numTestsSkipped += 1 continue test = TestCase(testIndex + 1, baseCmd, **allTests[testIndex]) if test.runCommand(cmdfilter, runWithValgrind): if test.test(): # passed test.printDetail(verbose=verboseDetail) numTestsPassed += 1 else: test.printDetail(verbose=True, failed=True) numTestsFailed += 1 # finally print the results to terminal print("Finished:") result = ", ".join([ f"Failed: {numTestsFailed}", f"Passed: {numTestsPassed}", f"Skipped: {numTestsSkipped}", f"Total: {len(testIndexesToRun)}" ]) if (numTestsFailed == 0): print("Passed! - ", result) else: ret = f"Failed! - {result}" else: ret = f" error: File \"{baseCmd}\" not found!" return ret if __name__ == "__main__": sys.exit(main(len(sys.argv), sys.argv)) curl-trurl-4f06332/testfiles/000077500000000000000000000000001452017613700161065ustar00rootroot00000000000000curl-trurl-4f06332/testfiles/test0000.txt000066400000000000000000000100471452017613700201300ustar00rootroot00000000000000https://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/ http://example.org curl-trurl-4f06332/testfiles/test0001.txt000066400000000000000000000002061452017613700201250ustar00rootroot00000000000000https://curl.se/ https://docs.python.org/ git://github.com/curl/curl.git http://example.org xyz://hello/?hicurl-trurl-4f06332/testfiles/test0002.txt000066400000000000000000000000001452017613700201160ustar00rootroot00000000000000curl-trurl-4f06332/tests.json000066400000000000000000001516371452017613700161560ustar00rootroot00000000000000[ { "input": { "arguments": [ "example.com" ] }, "expected": { "stdout": "http://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "http://example.com" ] }, "expected": { "stdout": "http://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com" ] }, "expected": { "stdout": "https://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "hp://example.com" ] }, "expected": { "stdout": "hp://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [] }, "expected": { "stdout": "", "stderr": "trurl error: not enough input for a URL\ntrurl error: Try trurl -h for help\n", "returncode": 7 } }, { "input": { "arguments": [ "ftp.example.com" ] }, "expected": { "stdout": "ftp://ftp.example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com/../moo" ] }, "expected": { "stdout": "https://example.com/moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com/.././moo" ] }, "expected": { "stdout": "https://example.com/moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com/test/../moo" ] }, "expected": { "stdout": "https://example.com/moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "localhost", "--append", "path=moo" ] }, "expected": { "stdout": "http://localhost/moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "localhost", "-a", "path=moo" ] }, "expected": { "stdout": "http://localhost/moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--set", "host=moo", "--set", "scheme=http" ] }, "expected": { "stdout": "http://moo/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "-s", "host=moo", "-s", "scheme=http" ] }, "expected": { "stdout": "http://moo/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--set", "host=moo", "--set", "scheme=https", "--set", "port=999" ] }, "expected": { "stdout": "https://moo:999/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--set", "host=moo", "--set", "scheme=ftps", "--set", "path=/hello" ] }, "expected": { "stdout": "ftps://moo/hello\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se", "--set", "host=example.com" ] }, "expected": { "stdout": "https://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--set", "host=example.com", "--set", "scheme=ftp" ] }, "expected": { "stdout": "ftp://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html", "--redirect", "here.html" ] }, "expected": { "stdout": "https://curl.se/we/here.html\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/../are.html", "--set", "port=8080" ] }, "expected": { "stdout": "https://curl.se:8080/are.html\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://curl.se:22/", "-s", "port=443" ] }, "expected": { "stdout": "https://curl.se/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://curl.se:22/", "-s", "port=443", "--get", "{url}" ] }, "expected": { "stdout": "https://curl.se/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html", "--get", "{path}" ] }, "expected": { "stdout": "/we/are.html\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--default-port", "--url", "https://curl.se/we/are.html", "--get", "{port}" ] }, "expected": { "stdout": "443\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html", "--get", "{scheme}" ] }, "expected": { "stdout": "https\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://hello@curl.se/we/are.html", "--get", "{user}" ] }, "expected": { "stdout": "hello\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://hello:secret@curl.se/we/are.html", "--get", "{password}" ] }, "expected": { "stdout": "secret\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "imap://hello:secret;crazy@curl.se/we/are.html", "--get", "{options}" ] }, "expected": { "stdout": "crazy\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html", "--get", "{host}" ] }, "expected": { "stdout": "curl.se\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://10.1/we/are.html", "--get", "{host}" ] }, "required": ["normalize-ipv4"], "expected": { "stdout": "10.0.0.1\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://[fe80::0000:20c:29ff:fe9c:409b]:8080/we/are.html", "--get", "{host}" ] }, "required": ["zone-id"], "expected": { "stdout": "[fe80::20c:29ff:fe9c:409b]\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://[fe80::0000:20c:29ff:fe9c:409b%euth0]:8080/we/are.html", "--get", "{zoneid}" ] }, "expected": { "stdout": "euth0\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://[fe80::0000:20c:29ff:fe9c:409b%eth0]:8080/we/are.html", "--get", "{zoneid}" ] }, "expected": { "stdout": "eth0\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html?user=many#more", "--get", "{query}" ] }, "expected": { "stdout": "user=many\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html?user=many#more", "--get", "{fragment}" ] }, "expected": { "stdout": "more\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html", "-g", "{default:port}" ] }, "expected": { "stdout": "443\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--append", "path=you" ] }, "expected": { "stdout": "https://curl.se/hello/you\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--append", "path=you index.html" ] }, "expected": { "stdout": "https://curl.se/hello/you%20index.html\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se?name=hello", "--append", "query=search=string" ] }, "expected": { "stdout": "https://curl.se/?name=hello&search=string\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--set", "user=:hej:" ] }, "expected": { "stdout": "https://%3ahej%3a@curl.se/hello\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--set", "user=hej", "--set", "password=secret" ] }, "expected": { "stdout": "https://hej:secret@curl.se/hello\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--set", "query:=user=me" ] }, "expected": { "stdout": "https://curl.se/hello?user=me\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--set", "query=user=me" ] }, "expected": { "stdout": "https://curl.se/hello?user%3dme\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--set", "fragment= hello" ] }, "expected": { "stdout": "https://curl.se/hello#%20hello\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://curl.se/hello", "--set", "fragment:=%20hello" ] }, "expected": { "stdout": "https://curl.se/hello#%20hello\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "localhost", "--append", "query=hello=foo" ] }, "expected": { "stdout": "http://localhost/?hello=foo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "localhost", "-a", "query=hello=foo" ] }, "expected": { "stdout": "http://localhost/?hello=foo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?search=hello&utm_source=tracker", "--trim", "query=utm_*" ] }, "expected": { "stdout": "https://example.com/?search=hello\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?search=hello&utm_source=tracker&more=data", "--trim", "query=utm_*" ] }, "expected": { "stdout": "https://example.com/?search=hello&more=data\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?search=hello&more=data", "--trim", "query=utm_*" ] }, "expected": { "stdout": "https://example.com/?search=hello&more=data\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?utm_source=tracker", "--trim", "query=utm_*" ] }, "expected": { "stdout": "https://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?search=hello&utm_source=tracker&more=data", "--trim", "query=utm_source" ] }, "expected": { "stdout": "https://example.com/?search=hello&more=data\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?search=hello&utm_source=tracker&more=data", "--trim", "query=utm_source", "--trim", "query=more", "--trim", "query=search" ] }, "expected": { "stdout": "https://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--accept-space", "--url", "gopher://localhost/ with space" ] }, "required": ["white-space"], "expected": { "stdout": "gopher://localhost/%20with%20space\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--accept-space", "--url", "https://localhost/?with space" ] }, "required": ["white-space"], "expected": { "stdout": "https://localhost/?with+space\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://daniel@curl.se:22/", "-s", "port=", "-s", "user=" ] }, "expected": { "stdout": "https://curl.se/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?moo&search=hello", "--trim", "query=search" ] }, "expected": { "stdout": "https://example.com/?moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?search=hello&moo", "--trim", "query=search" ] }, "expected": { "stdout": "https://example.com/?moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?search=hello", "--trim", "query=search", "--append", "query=moo" ] }, "expected": { "stdout": "https://example.com/?moo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://hello:443/foo" ] }, "expected": { "stdout": "https://hello/foo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "ftp://hello:21/foo" ] }, "expected": { "stdout": "ftp://hello/foo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://hello:443/foo", "-s", "scheme=ftp" ] }, "expected": { "stdout": "ftp://hello:443/foo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "ftp://hello:443/foo", "-s", "scheme=https" ] }, "expected": { "stdout": "https://hello/foo\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?utm_source=tra%20cker&address%20=home&here=now&thisthen", "-g", "{query:utm_source}" ] }, "expected": { "stdout": "tra cker\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?utm_source=tra%20cker&address%20=home&here=now&thisthen", "-g", "{:query:utm_source}" ] }, "expected": { "stdout": "tra%20cker\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?utm_source=tra%20cker&address%20=home&here=now&thisthen", "-g", "{:query:utm_}" ] }, "expected": { "stdout": "\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?utm_source=tra%20cker&address%20=home&here=now&thisthen", "-g", "{:query:UTM_SOURCE}" ] }, "expected": { "stdout": "\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?utm_source=tracker&monkey=123", "--sort-query" ] }, "expected": { "stdout": "https://example.com/?monkey=123&utm_source=tracker\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?a=b&c=d&", "--sort-query" ] }, "expected": { "stdout": "https://example.com/?a=b&c=d\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?a=b&c=d&", "--sort-query", "--trim", "query=a" ] }, "expected": { "stdout": "https://example.com/?c=d\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "example.com:29", "--set", "port=" ] }, "expected": { "stdout": "http://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "HTTPS://example.com" ] }, "expected": { "stdout": "https://example.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://EXAMPLE.com" ] }, "expected": { "stdout": "https://EXAMPLE.com/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "https://example.com/FOO/BAR" ] }, "expected": { "stdout": "https://example.com/FOO/BAR\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "[2001:0db8:0000:0000:0000:ff00:0042:8329]" ] }, "required": ["zone-id"], "expected": { "stdout": "http://[2001:db8::ff00:42:8329]/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "https://example.com?utm=tra%20cker:address%20=home:here=now:thisthen", "--sort-query", "--query-separator", ":" ] }, "expected": { "stdout": "https://example.com/?address%20=home:here=now:thisthen:utm=tra%20cker\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "foo?a=bCd=eCe=f", "--query-separator", "C", "--trim", "query=d" ] }, "expected": { "stdout": "http://foo/?a=bCe=f\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "localhost", "-g", "{scheme} {host" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "http {host\n" } }, { "input": { "arguments": [ "localhost", "-g", "[scheme] [host" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "http [host\n" } }, { "input": { "arguments": [ "localhost", "-g", "\\{{scheme}\\[" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "{http[\n" } }, { "input": { "arguments": [ "localhost", "-g", "\\\\[" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "\\[\n" } }, { "input": { "arguments": [ "https://u:s@foo?moo", "-g", "[scheme][user][password][query]" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "httpsusmoo\n" } }, { "input": { "arguments": [ "hej?a=b&a=c&a=d&b=a", "-g", "{query-all:a}" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "b c d\n" } }, { "input": { "arguments": [ "https://curl.se?name=mr%00smith", "--get", "{query:name}" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "mr.smith\n" } }, { "input": { "arguments": [ "https://curl.se", "--iterate", "port=80 81 443" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "https://curl.se:80/\nhttps://curl.se:81/\nhttps://curl.se/\n" } }, { "input": { "arguments": [ "https://curl.se", "--iterate", "port=81 443", "--iterate", "scheme=sftp moo" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "sftp://curl.se:81/\nmoo://curl.se:81/\nsftp://curl.se:443/\nmoo://curl.se:443/\n" } }, { "input": { "arguments": [ "https://curl.se", "--iterate", "port=81 443", "--iterate", "scheme=sftp moo", "--iterate", "port=2 1" ] }, "expected": { "stderr": "trurl error: duplicate component for iterate: port\ntrurl error: Try trurl -h for help\n", "returncode": 11, "stdout": "" } }, { "input": { "arguments": [ "https://curl.se", "-s", "host=localhost", "--iterate", "port=22 23" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "https://localhost:22/\nhttps://localhost:23/\n" } }, { "input": { "arguments": [ "hello@localhost", "--iterate", "host=one two", "-g", "{host} {user}" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": "one hello\ntwo hello\n" } }, { "input": { "arguments": [ "https://example.com?utm=tra%20cker&address%20=home&here=now&thisthen", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "https://example.com/?utm=tra%20cker&address%20=home&here=now&thisthen", "parts": { "scheme": "https", "host": "example.com", "path": "/", "query": "utm=tra cker&address =home&here=now&thisthen" }, "params": [ { "key": "utm", "value": "tra cker" }, { "key": "address ", "value": "home" }, { "key": "here", "value": "now" }, { "key": "thisthen", "value": "" } ] } ] } }, { "input": { "arguments": [ "ftp://smith:secret@example.com:33/path?search=me#where", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "ftp://smith:secret@example.com:33/path?search=me#where", "parts": { "scheme": "ftp", "user": "smith", "password": "secret", "host": "example.com", "port": "33", "path": "/path", "query": "search=me", "fragment": "where" }, "params": [ { "key": "search", "value": "me" } ] } ] } }, { "input": { "arguments": [ "example.com", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "http://example.com/", "parts": { "scheme": "http", "host": "example.com", "path": "/" } } ] } }, { "input": { "arguments": [ "example.com", "other.com", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "http://example.com/", "parts": { "scheme": "http", "host": "example.com", "path": "/" } }, { "url": "http://other.com/", "parts": { "scheme": "http", "host": "other.com", "path": "/" } } ] } }, { "input": { "arguments": [ "localhost", "--iterate", "host=one two", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "http://one/", "parts": { "scheme": "http", "host": "one", "path": "/" } }, { "url": "http://two/", "parts": { "scheme": "http", "host": "two", "path": "/" } } ] } }, { "input": { "arguments": [ "--json", "-s", "scheme=irc", "-s", "host=curl.se" ] }, "expected": { "stdout": [ { "url": "irc://curl.se/", "parts": { "scheme": "irc", "host": "curl.se", "path": "/" } } ], "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--json", "-s", "host=curl.se" ] }, "expected": { "stdout": [], "returncode": 7, "stderr": "trurl error: not enough input for a URL\ntrurl error: Try trurl -h for help\n" } }, { "input": { "arguments": [ "--verify", "--json", "ftp://example.org", "", "git://curl.se/" ] }, "expected": { "stdout": [ { "url": "ftp://example.org/", "parts": { "scheme": "ftp", "host": "example.org", "path": "/" } } ], "returncode": 9, "stderr": true } }, { "input": { "arguments": [ "-s", "scheme=imap" ] }, "expected": { "stdout": "", "returncode": 7, "stderr": "trurl error: not enough input for a URL\ntrurl error: Try trurl -h for help\n" } }, { "input": { "arguments": [ "-g", "{query:}", "http://localhost/?=bar" ] }, "expected": { "stdout": "bar\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--json", "https://curl.se/?&&&" ] }, "expected": { "stdout": [ { "url": "https://curl.se/", "parts": { "scheme": "https", "host": "curl.se", "path": "/", "query": "" }, "params": [] } ], "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--json", "--trim", "query=f*", "localhost?foo&bar=ar" ] }, "expected": { "stdout": [ { "url": "http://localhost/?bar=ar", "parts": { "scheme": "http", "host": "localhost", "path": "/", "query": "bar=ar" }, "params": [ { "key": "bar", "value": "ar" } ] } ], "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "https://example.com?search=hello&utm_source=tracker&utm_block&testing", "--trim", "query=utm_*", "--json" ] }, "expected": { "stdout": [ { "url": "https://example.com/?search=hello&testing", "parts": { "scheme": "https", "host": "example.com", "path": "/", "query": "search=hello&testing" }, "params": [ { "key": "search", "value": "hello" }, { "key": "testing", "value": "" } ] } ], "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "https://räksmörgås.se", "-g", "{default:puny:url}" ] }, "required": ["punycode"], "expected": { "stderr": "", "returncode": 0, "stdout": "https://xn--rksmrgs-5wao1o.se:443/\n" } }, { "input": { "arguments": [ "https://räksmörgås.se", "-g", "{puny:url}" ] }, "required": ["punycode"], "expected": { "stderr": "", "returncode": 0, "stdout": "https://xn--rksmrgs-5wao1o.se/\n" } }, { "input": { "arguments": [ "https://räksmörgås.se", "-g", "{puny:host}" ] }, "required": ["punycode"], "expected": { "stderr": "", "returncode": 0, "stdout": "xn--rksmrgs-5wao1o.se\n" } }, { "input": { "arguments": [ "imaps://user:password;crazy@[ff00::1234%hello]:1234/path?a=b&c=d#fragment", "--json" ] }, "expected": { "returncode": 0, "stdout": [ { "url": "imaps://user:password;crazy@[ff00::1234%25hello]:1234/path?a=b&c=d#fragment", "parts": { "scheme": "imaps", "user": "user", "password": "password", "options": "crazy", "host": "[ff00::1234]", "port": "1234", "path": "/path", "query": "a=b&c=d", "fragment": "fragment", "zoneid": "hello" }, "params": [ { "key": "a", "value": "b" }, { "key": "c", "value": "d" } ] } ] } }, { "input": { "arguments": [ "http://example.com/", "--get", "port: {port}, default:port: {default:port}" ] }, "expected": { "returncode": 0, "stdout": "port: , default:port: 80\n" } }, { "input": { "arguments": [ "http://example.com:8080/", "--get", "port: {port}, default:port: {default:port}" ] }, "expected": { "returncode": 0, "stdout": "port: 8080, default:port: 8080\n" } }, { "input": { "arguments": [ "localhost", "-s", "host=foo", "--iterate", "host=bar baz" ] }, "expected": { "stdout": "", "returncode": 11, "stderr": "trurl error: duplicate --iterate and --set for component host\ntrurl error: Try trurl -h for help\n" } }, { "input": { "arguments": [ "emanuele6://curl.se/trurl", "", "https://example.org" ] }, "expected": { "stdout": "emanuele6://curl.se/trurl\nhttps://example.org/\n", "returncode": 0, "stderr": true } }, { "input": { "arguments": [ "--verify", "--no-guess-scheme", "hello" ] }, "expected": { "stdout": "", "returncode": 9, "stderr": "trurl error: Bad scheme [hello]\ntrurl error: Try trurl -h for help\n" } }, { "input": { "arguments": [ "--verify", "-f", "testfiles/test0000.txt" ] }, "expected": { "stdout": "http://example.org/\n", "returncode": 0, "stderr": "trurl note: skipping long line\n" } }, { "input": { "arguments": [ "-f", "testfiles/test0001.txt" ] }, "expected": { "stdout": "https://curl.se/\nhttps://docs.python.org/\ngit://github.com/curl/curl.git\nhttp://example.org/\nxyz://hello/?hi\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--no-guess-scheme", "foo", "hi", "https://example.org", "hey", "git://curl.se" ] }, "expected": { "stdout": "https://example.org/\ngit://curl.se/\n", "returncode": 0, "stderr": "trurl note: Bad scheme [foo]\ntrurl note: Bad scheme [hi]\ntrurl note: Bad scheme [hey]\n" } }, { "input": { "arguments": [ "-f", "testfiles/test0002.txt", "--json" ] }, "expected": { "stdout": "[]\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--accept-space", "-s", "query:=x=10&x=2 3", "localhost" ] }, "required": ["white-space"], "expected": { "stdout": "http://localhost/?x=10&x=2+3\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "-s", "path=\\\\", "-g", "{path}\\n{:path}", "localhost" ] }, "expected": { "stdout": "/\\\\\n/%5c%5c\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "-s", "path=\\\\", "--json", "localhost" ] }, "expected": { "stdout": [ { "url": "http://localhost/%5c%5c", "parts": { "scheme": "http", "host": "localhost", "path": "/\\\\" } } ], "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "-s", "path=\\\\", "-g", "{path}\\n{:path}", "--urlencode", "localhost" ] }, "expected": { "stdout": "/%5c%5c\n/%5c%5c\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "-s", "path=abc\\\\", "-s", "query:=a&b&a%26b", "--urlencode", "--json", "localhost" ] }, "expected": { "stdout": [ { "url": "http://localhost/abc%5c%5c?a&b&a%26b", "parts": { "scheme": "http", "host": "localhost", "path": "/abc%5c%5c", "query": "a&b&a%26b" }, "params": [ { "key": "a", "value": "" }, { "key": "b", "value": "" }, { "key": "a&b", "value": "" } ] } ], "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "-s", "scheme:=http", "-s", "host:=localhost", "-s", "path:=/ABC%5C%5C", "-s", "query:=a&b&a%26b" ] }, "expected": { "stdout": "http://localhost/ABC%5c%5c?a&b&a%26b\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "-g", "{query:b}\\t{query-all:a}\\n{:query:b}\\t{:query-all:a}", "https://example.org/foo?a=1&b=%23&a=%26#hello" ] }, "expected": { "stdout": "#\t1 &\n%23\t1 %26\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--urlencode", "-g", "{query:b}\\t{query-all:a}\\n{:query:b}\\t{:query-all:a}", "https://example.org/foo?a=1&b=%23&a=%26#hello" ] }, "expected": { "stdout": "%23\t1 %26\n%23\t1 %26\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "-a", "query=c=moo", "--sort-query", "https://example.org/foo?x=hi#rye" ] }, "expected": { "stdout": "https://example.org/foo?c=moo&x=hi#rye\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--trim", "query=a", "-a", "query=a=ciao", "-a", "query=b=salve", "https://example.org/foo?a=hi&b=hello&x=y" ] }, "expected": { "stdout": "https://example.org/foo?b=hello&x=y&a=ciao&b=salve\n", "returncode": 0, "stderr": "" } }, { "input" : { "arguments": [ "http://example.com/?q=mr%00smith", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "http://example.com/?q=mr%00smith", "parts": { "scheme": "http", "host": "example.com", "path": "/" }, "params": [ { "key": "q", "value": "mr\u0000smith" } ] } ] } }, { "input" : { "arguments": [ "http://example.com/?q=mr%00sm%00ith", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "http://example.com/?q=mr%00sm%00ith", "parts": { "scheme": "http", "host": "example.com", "path": "/" }, "params": [ { "key": "q", "value": "mr\u0000sm\u0000ith" } ] } ] } }, { "input" : { "arguments": [ "http://example.com/?q=mr%00%00%00smith", "--json" ] }, "expected": { "stderr": "", "returncode": 0, "stdout": [ { "url": "http://example.com/?q=mr%00%00%00smith", "parts": { "scheme": "http", "host": "example.com", "path": "/" }, "params": [ { "key": "q", "value": "mr\u0000\u0000\u0000smith" } ] } ] } }, { "input": { "arguments": [ "--url", "https://curl.se/we/are.html?*=moo&user=many#more", "--trim", "query=\\*" ] }, "expected": { "stdout": "https://curl.se/we/are.html?user=many#more\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "http://xn--rksmrgs-5wao1o/", "--as-idn" ] }, "required": ["punycode2idn"], "expected": { "stdout": "http://räksmörgås/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "http://xn--rksmrgs-5wao1o/", "-g", "{idn:host}" ] }, "required": ["punycode2idn"], "expected": { "stdout": "räksmörgås\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "http://xn-----/", "--as-idn", "--quiet" ] }, "required": ["punycode2idn"], "expected": { "stdout": "http://xn-----/\n", "stderr": "", "returncode": 0 } }, { "input": { "arguments": [ "--url", "http://xn-----/", "--as-idn" ] }, "required": ["punycode2idn"], "expected": { "stdout": "http://xn-----/\n", "stderr": "trurl note: Error converting url to IDN [Bad hostname]\n", "returncode": 0 } }, { "input": { "arguments": [ "--verify", "-f", "testfiles/test0000.txt", "--quiet" ] }, "expected": { "stdout": "http://example.org/\n", "returncode": 0, "stderr": "" } }, { "input": { "arguments": [ "--curl", "--verify", "foo://bar" ] }, "expected": { "stdout": "", "stderr": true, "returncode": 9 } } ] curl-trurl-4f06332/trurl.1000066400000000000000000000367211452017613700153470ustar00rootroot00000000000000.\" ************************************************************************** .\" * _ _ ____ _ .\" * Project ___| | | | _ \| | .\" * / __| | | | |_) | | .\" * | (__| |_| | _ <| |___ .\" * \___|\___/|_| \_\_____| .\" * .\" * Copyright (C) Daniel Stenberg, , et al. .\" * .\" * This software is licensed as described in the file COPYING, which .\" * you should have received as part of this distribution. The terms .\" * are also available at https://curl.se/docs/copyright.html. .\" * .\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell .\" * copies of the Software, and permit persons to whom the Software is .\" * furnished to do so, under the terms of the COPYING file. .\" * .\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY .\" * KIND, either express or implied. .\" * .\" * SPDX-License-Identifier: curl .\" * .\" ************************************************************************** .\" You can view this file with: .\" man -l trurl.1 .\" .TH trurl 1 "April 27, 2023" "trurl" "trurl Manual" .SH NAME trurl \- transpose URLs .SH SYNOPSIS .B trurl [options / URLs] .SH DESCRIPTION .B trurl parses, manipulates and outputs URLs and parts of URLs. It uses the RFC 3986 definition of URLs and it uses libcurl's URL parser to do so, which includes a few "extensions". The URL support is limited to "hierarchical" URLs, the ones that use "://" separators after the scheme. Typically you pass in one or more URLs and decide what of that you want output. Possibly modifying the URL as well. trurl knows URLs and every URL consists of up to ten separate and independent "components". These components can be extracted, removed and updated with trurl and they are referred to by their respective names: scheme, user, password, options, host, port, path, query, fragment and zoneid. .SH "OPTIONS" Options start with one or two dashes. Many of the options require an additional value next to them. Any other argument is interpreted as a URL argument, and is treated as if it was following a \fI--url\fP option. The first argument that is exactly two dashes ("--"), marks the end of options; any argument after the end of options is interpreted as a URL argument even if it starts with a dash. .IP "-a, --append [component]=[data]" Append data to a component. This can only append data to the path and the query components. For path, this URL encodes and appends the new segment to the path, separated with a slash. For query, this URL encodes and appends the new segment to the query, separated with an ampersand (&). If the appended segment contains an equal sign ('=') that one will be kept verbatim and both sides of the first occurrence will be URL encoded separately. .IP "--accept-space" When set, trurl will try to accept spaces as part of the URL and instead URL encode such occurrences accordingly. According to RFC 3986, a space cannot legally be part of a URL. This option provides a best-effort to convert the provided string into a valid URL. .IP "--default-port" When set, trurl will use the scheme's default port number for URLs with a known scheme, and without an explicit port number. Note that trurl only knows default port numbers for URL schemes that are supported by libcurl. Since, by default, trurl removes default port numbers from URLs with a known scheme, this option is pretty much ignored unless one of \fI--get\fP, \fI--json\fP, and \fI--keep-port\fP is not also specified. .IP "-f, --url-file [file name]" Read URLs to work on from the given file. Use the file name "-" (a single minus) to tell trurl to read the URLs from stdin. Each line needs to be a single valid URL. trurl will remove one carriage return character at the end of the line if present, trim off all the trailing space and tab characters, and skip all empty (after trimming) lines. The maximum line length supported in a file like this is 4094 bytes. Lines that exceed that length are skipped, and a warning is printed to stderr when they are encountered. .IP "-g, --get [format]" Output text and URL data according to the provided format string. Components from the URL can be output when specified as \fB{component}\fP or \fB[component]\fP, with the name of the part show within curly braces or brackets. You can not mix braces and brackets for this purpose in the same command line. The following component names are available (case sensitive): url, scheme, user, password, options, host, port, path, query, fragment and zoneid. \fB{component}\fP will expand to nothing if the given component does not have a value. Components are shown URL decoded by default. If you instead write the component prefixed with a colon like "{:path}", it gets output URL encoded. You may also prefix components with \fBdefault:\fP and/or \fBpuny:\fP or \fBidn:\fP, in any order. If \fBdefault:\fP is specified, like "{default:url}" or "{default:port}", and the port is not explicitly specified in the URL, the scheme's default port will be output if it is known. If \fBpuny:\fP is specified, like "{puny:url}" or "{puny:host}", the "punycoded" version of the host name will be used in the output. This option is mutually exclusive with \fBidn:\fP. If \fBidn:\fP is specified like "{idn:url}" or "{idn:host}", the International Domain Name version of the host name will be used in the output if it is provided as a correctly encoded punycode version. This option is mutually exclusive with \fBpuny:\fP. If \fI--default-port\fP is specified, all formats are expanded as if they used \fIdefault:\fP; and if \fI--punycode\fP is used, all formats are expanded as if they used \fIpuny:\fP. Also note that "{url}" is affected by the \fI--keep-port\fP option. Hosts provided as IPv6 numerical addresses will be provided within square brackets. Like "[fe80::20c:29ff:fe9c:409b]". Hosts provided as IPv4 numerical addresses will be "normalized" and provided as four dot-separated decimal numbers when output. You can access specific keys in the query string using the format \fB{query:key}\fP. Then the value of the first matching key will be output using a case sensitive match. When extracting a URL decoded query key that contains %00, such octet will be replaced with a single period '.' in the output. You can access specific keys in the query string and out all values using the format \fB{query-all:key}\fP. This looks for 'key' case sensitively and will output all values for that key space-separated. The "format" string supports the following backslash sequences: \&\\\\ - backslash \&\\t - tab \&\\n - newline \&\\r - carriage return \&\\{ - an open curly brace that does not start a variable \&\\[ - an open bracket that does not start a variable All other text in the format string will be shown as-is. .IP "-h, --help" Show the help output. .IP "--iterate [component]=[item1 item2 ...]" Set the component to multiple values and output the result once for each iteration. Several combined iterations are allowed to generate combinations, but only one \fI--iterate\fP option per component. The listed items to iterate over should be separated by single spaces. .IP "--json" Outputs all set components of the URLs as JSON objects. All components of the URL that have data will get populated in the parts object using their component names. See below for details on the format. .IP "--keep-port" By default, trurl removes default port numbers from URLs with a known scheme even if they are explicitly specified in the input URL. This options, makes trurl not remove them. .IP "--no-guess-scheme" Disables libcurl's scheme guessing feature. URLs that do not contain a scheme will be treated as invalid URLs. .IP "--punycode" Uses the "punycoded" version of the host name, which is how International Domain Names are converted into plain ASCII. If the host name is not using IDN, the regular ASCII name is used. .IP "--as-idn" Converts a "punycoded" ASCII host name to its original International Domain Name in Unicode. If the host name is not using punycode then the original host name is used. .IP "--query-separator [what]" Specify the single letter used for separating query pairs. The default is "&" but at least in the past sometimes semicolons ";" or even colons ":" have been used for this purpose. If your URL uses something other than the default letter, setting the right one makes sure trurl can do its query operations properly. .IP "--redirect [URL]" Redirect the URL to this new location. The redirection is performed on the base URL, so, if no base URL is specified, no redirection will be performed. .IP "-s, --set [component][:]=[data]" Set this URL component. Setting blank string ("") will clear the component from the URL. The following components can be set: url, scheme, user, password, options, host, port, path, query, fragment and zoneid. If a simple "="-assignment is used, the data is URL encoded when applied. If ":=" is used, the data is assumed to already be URL encoded and will be stored as-is. If no URL or \fI--url-file\fP argument is provided, trurl will try to create a URL using the components provided by the \fI--set\fP options. If not enough components are specified, this will fail. .IP "--sort-query" The "variable=content" tuplets in the query component are sorted in a case insensitive alphabetical order. This helps making URLs identical that otherwise only had their query pairs in different orders. .IP "--url [URL]" Set the input URL to work with. The URL may be provided without a scheme, which then typically is not actually a legal URL but trurl will try to figure out what is meant and guess what scheme to use (unless \fI--no-guess-scheme\fP is used). Providing multiple URLs will make trurl act on all URLs in a serial fashion. If the URL cannot be parsed for whatever reason, trurl will simply move on to the next provided URL - unless \fI--verify\fP is used. .IP "--urlencode" Outputs URL encoded version of components by default when using \fI--get\fP or \fI--json\fP. .IP "--trim [component]=[what]" Trims data off a component. Currently this can only trim a query component. "what" is specified as a full word or as a word prefix (using a single trailing asterisk ('*')) which makes trurl remove the tuples from the query string that match the instruction. To match a literal trailing asterisk instead of using a wildcard, escape it with a backslash in front of it. Like "\\*". .IP "-v, --version" Show version information and exit. .IP "--verify" When a URL is provided, return error immediately if it does not parse as a valid URL. In normal cases, trurl can forgive a bad URL input. .IP "--quiet" Suppress (some) notes and warnings. .SH "JSON output format" The \fI--json\fP option outputs a JSON array with one or more objects. One for each URL. Each URL JSON object contains a number of properties, a series of key/value pairs. The exact set depends on the given URL. .IP "url" This key exists in every object. It is the complete URL. Affected by \fI--default-port\fP, \fI--keep-port\fP, and \fI--punycode\fP. .IP "parts" This key exists in every object, and contains an object with a key for each of the settable URL components. If a component is missing, it means it is not present in the URL. The parts are URL decoded unless \fI--urlencode\fP is used. .RS .TP .B "scheme" The URL scheme. .TP .B "user" The user name. .TP .B "password" The password. .TP .B "options" The options. Note that only a few URL schemes support the "options" component. .TP .B "host" The and normalized host name. It might be a UTF-8 name if an IDN name was used. It can also be a normalized IPv4 or IPv6 address. An IPv6 address always starts with a bracket (\fB[\fP) - and no other host names can contain such a symbol. If \fI--punycode\fP is used, the punycode version of the host is outputted instead. .TP .B "port" The provided port number as a string. If the port number was not provided in the URL, but the scheme is a known one, and \fI--default-port\fP is in use, the default port for that scheme will be provided here. .TP .B "path" The path. Including the leading slash. .TP .B "query" The full query, excluding the question mark separator. .TP .B "fragment" The fragment, excluding the pound sign separator. .TP .B "zoneid" The zone id, which can only be present in an IPv6 address. When this key is present, then \fBhost\fP is an IPv6 numerical address. .RE .IP "params" This key contains an array of query key/value objects. Each such pair is listed with "key" and "value" and their respective contents in the output. The key/values are extracted from the query where they are separated by ampersands (\fB&\fP) - or the user sets with \fB--query-separator\fP. The query pairs are listed in the order of appearance in a left-to-right order, but can be made alpha-sorted with \fB--sort-query\fP. It is only present if the URL has a query. .SH EXAMPLES .IP "Replace the host name of a URL" .nf $ trurl --url https://curl.se --set host=example.com https://example.com/ .fi .IP "Create a URL by setting components" .nf $ trurl --set host=example.com --set scheme=ftp ftp://example.com/ .fi .IP "Redirect a URL" .nf $ trurl --url https://curl.se/we/are.html --redirect here.html https://curl.se/we/here.html .fi .IP "Change port number" This also shows how trurl will remove dot-dot sequences .nf $ trurl --url https://curl.se/we/../are.html --set port=8080 https://curl.se:8080/are.html .fi .IP "Extract the path from a URL" .nf $ trurl --url https://curl.se/we/are.html --get '{path}' /we/are.html .fi .IP "Extract the port from a URL" This gets the default port based on the scheme if the port is not set in the URL. .nf $ trurl --url https://curl.se/we/are.html --get '{default:port}' 443 .fi .IP "Append a path segment to a URL" .nf $ trurl --url https://curl.se/hello --append path=you https://curl.se/hello/you .fi .IP "Append a query segment to a URL" .nf $ trurl --url "https://curl.se?name=hello" --append query=search=string https://curl.se/?name=hello&search=string .fi .IP "Read URLs from stdin" .nf $ cat urllist.txt | trurl --url-file - \&... .fi .IP "Output JSON" .nf $ trurl "https://fake.host/search?q=answers&user=me#frag" --json [ { "url": "https://fake.host/search?q=answers&user=me#frag", "parts": [ "scheme": "https", "host": "fake.host", "path": "/search", "query": "q=answers&user=me" "fragment": "frag", ], "params": [ { "key": "q", "value": "answers" }, { "key": "user", "value": "me" } ] } ] .fi .IP "Remove tracking tuples from query" .nf $ trurl "https://curl.se?search=hey&utm_source=tracker" --trim query="utm_*" https://curl.se/?search=hey .fi .IP "Show a specific query key value" .nf $ trurl "https://example.com?a=home&here=now&thisthen" -g '{query:a}' home .fi .IP "Sort the key/value pairs in the query component" .nf $ trurl "https://example.com?b=a&c=b&a=c" --sort-query https://example.com?a=c&b=a&c=b .fi .IP "Work with a query that uses a semicolon separator" .nf $ trurl "https://curl.se?search=fool;page=5" --trim query="search" --query-separator ";" https://curl.se?page=5 .fi .IP "Accept spaces in the URL path" .nf $ trurl "https://curl.se/this has space/index.html" --accept-space https://curl.se/this%20has%20space/index.html .fi .IP "Create multiple variations of a URL with different schemes" .nf $ trurl "https://curl.se/path/index.html" --iterate "scheme=http ftp sftp" http://curl.se/path/index.html ftp://curl.se/path/index.html sftp://curl.se/path/index.html .fi .SH WWW https://curl.se/trurl .SH "SEE ALSO" .BR curl_url_set (3), .BR curl_url_get (3) curl-trurl-4f06332/trurl.c000066400000000000000000001173401452017613700154260ustar00rootroot00000000000000/*************************************************************************** * _ _ ____ _ * Project ___| | | | _ \| | * / __| | | | |_) | | * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * * Copyright (C) Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms * are also available at https://curl.se/docs/copyright.html. * * You may opt to use, copy, modify, merge, publish, distribute and/or sell * copies of the Software, and permit persons to whom the Software is * furnished to do so, under the terms of the COPYING file. * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY * KIND, either express or implied. * * SPDX-License-Identifier: curl * ***************************************************************************/ #include #include #include #include #include #include #include #include #include /* for setlocale() */ #include "version.h" #ifdef _MSC_VER #define strncasecmp _strnicmp #define strcasecmp _stricmp #define strdup _strdup #endif #if CURL_AT_LEAST_VERSION(7,77,0) #define SUPPORTS_NORM_IPV4 #endif #if CURL_AT_LEAST_VERSION(7,81,0) #define SUPPORTS_ZONEID #endif #if CURL_AT_LEAST_VERSION(7,80,0) #define SUPPORTS_URL_STRERROR #endif #if CURL_AT_LEAST_VERSION(7,78,0) #define SUPPORTS_ALLOW_SPACE #else #define CURLU_ALLOW_SPACE 0 #endif #if CURL_AT_LEAST_VERSION(7,88,0) #define SUPPORTS_PUNYCODE #endif #if CURL_AT_LEAST_VERSION(8,3,0) #define SUPPORTS_PUNY2IDN #endif #define OUTPUT_URL 0 /* default */ #define OUTPUT_SCHEME 1 #define OUTPUT_USER 2 #define OUTPUT_PASSWORD 3 #define OUTPUT_OPTIONS 4 #define OUTPUT_HOST 5 #define OUTPUT_PORT 6 #define OUTPUT_PATH 7 #define OUTPUT_QUERY 8 #define OUTPUT_FRAGMENT 9 #define OUTPUT_ZONEID 10 #define NUM_COMPONENTS 11 /* including "url" */ #define PROGNAME "trurl" #define REPLACE_NULL_BYTE '.' /* for query:key extractions */ enum { VARMODIFIER_URLENCODED = 1 << 1, VARMODIFIER_DEFAULT = 1 << 2, VARMODIFIER_PUNY = 1 << 3, VARMODIFIER_PUNY2IDN = 1 << 4, }; struct var { const char *name; CURLUPart part; }; struct string { char *str; size_t len; }; static const struct var variables[] = { {"scheme", CURLUPART_SCHEME}, {"user", CURLUPART_USER}, {"password", CURLUPART_PASSWORD}, {"options", CURLUPART_OPTIONS}, {"host", CURLUPART_HOST}, {"port", CURLUPART_PORT}, {"path", CURLUPART_PATH}, {"query", CURLUPART_QUERY}, {"fragment", CURLUPART_FRAGMENT}, {"zoneid", CURLUPART_ZONEID}, {NULL, 0} }; #define ERROR_PREFIX PROGNAME " error: " #define WARN_PREFIX PROGNAME " note: " /* error codes */ #define ERROR_FILE 1 #define ERROR_APPEND 2 /* --append mistake */ #define ERROR_ARG 3 /* a command line option misses its argument */ #define ERROR_FLAG 4 /* a command line flag mistake */ #define ERROR_SET 5 /* a --set problem */ #define ERROR_MEM 6 /* out of memory */ #define ERROR_URL 7 /* could not get a URL out of the set components */ #define ERROR_TRIM 8 /* a --trim problem */ #define ERROR_BADURL 9 /* if --verify is set and the URL cannot parse */ #define ERROR_GET 10 /* bad --get syntax */ #define ERROR_ITER 11 /* bad --iterate syntax */ #ifndef SUPPORTS_URL_STRERROR /* provide a fake local mockup */ static char *curl_url_strerror(CURLUcode error) { static char buffer[128]; curl_msnprintf(buffer, sizeof(buffer), "URL error %u", (int)error); return buffer; } #endif static void warnf(char *fmt, ...) { va_list ap; va_start(ap, fmt); fputs(WARN_PREFIX, stderr); vfprintf(stderr, fmt, ap); fputs("\n", stderr); va_end(ap); } #define VERIFY(o, exit_code, ...) \ do { \ if(!o->verify) \ warnf(__VA_ARGS__); \ else { \ /* make sure to terminate the JSON array */ \ if(o->jsonout) \ printf("%s]\n", o->urls ? "\n" : ""); \ errorf(o, exit_code, __VA_ARGS__); \ } \ } while(0) static void help(void) { int i; fputs( "Usage: " PROGNAME " [options] [URL]\n" " -a, --append [component]=[data] - append data to component\n" " --accept-space - give in to this URL abuse\n" " --curl - only schemes supported by libcurl\n" " --default-port - add known default ports\n" " -f, --url-file [file/-] - read URLs from file or stdin\n" " -g, --get [{component}s] - output component(s)\n" " -h, --help - this help\n" " --iterate [component]=[list] - create multiple URL outputs\n" " --json - output URL as JSON\n" " --keep-port - keep known default ports\n" " --no-guess-scheme - require scheme in URLs\n" " --punycode - encode hostnames in punycode\n" " --as-idn - encode hostnames in idn\n" " --query-separator [letter] - if something else than '&'\n" " --redirect [URL] - redirect to this\n" " -s, --set [component]=[data] - set component content\n" " --sort-query - alpha-sort the query pairs\n" " --trim [component]=[what] - trim component\n" " --url [URL] - URL to work with\n" " --urlencode - URL encode components by default\n" " -v, --version - show version\n" " --verify - return error on (first) bad URL\n" " --quiet - Suppress (some) notes and comments\n" " URL COMPONENTS:\n" " ", stdout); for(i = 0; i< NUM_COMPONENTS; i++) { printf("%s%s", i?", ":"", variables[i].name); } fputs("\n", stdout); exit(0); } static void show_version(void) { curl_version_info_data *data = curl_version_info(CURLVERSION_NOW); fprintf(stdout, "%s version %s libcurl/%s [built-with %s]\n", PROGNAME, TRURL_VERSION_TXT, data->version, LIBCURL_VERSION); /* puny code isn't guaranteed based on the version, so it must be polled * from libcurl */ bool supports_puny = false; #ifdef SUPPORTS_PUNYCODE const char *const *feature_name = data->feature_names; while(*feature_name && !supports_puny) { supports_puny = !strncmp(*feature_name, "IDN", 3); feature_name++; } #endif fprintf(stdout, "features: %s", supports_puny?"punycode ":""); #ifdef SUPPORTS_ALLOW_SPACE fprintf(stdout, "white-space "); #endif #ifdef SUPPORTS_ZONEID fprintf(stdout, "zone-id "); #endif #ifdef SUPPORTS_URL_STRERROR fprintf(stdout, "url-strerror "); #endif #ifdef SUPPORTS_NORM_IPV4 fprintf(stdout, "normalize-ipv4 "); #endif #ifdef SUPPORTS_PUNY2IDN fprintf(stdout, "punycode2idn"); #endif fprintf(stdout, "\n"); exit(0); } struct iterinfo { CURLU *uh; const char *part; size_t plen; char *ptr; unsigned int varmask; /* sets 1 << [component] */ }; struct option { struct curl_slist *url_list; struct curl_slist *append_path; struct curl_slist *append_query; struct curl_slist *set_list; struct curl_slist *trim_list; struct curl_slist *iter_list; const char *redirect; const char *qsep; const char *format; FILE *url; bool urlopen; bool jsonout; bool verify; bool accept_space; bool curl; bool default_port; bool keep_port; bool punycode; bool puny2idn; bool sort_query; bool no_guess_scheme; bool urlencode; bool end_of_options; bool quiet_warnings; /* -- stats -- */ unsigned int urls; }; void trurl_warnf(struct option *o, char *fmt, ...) { if(!o->quiet_warnings) { va_list ap; va_start(ap, fmt); fputs(WARN_PREFIX, stderr); vfprintf(stderr, fmt, ap); fputs("\n", stderr); va_end(ap); } } #define MAX_QPAIRS 1000 struct string qpairs[MAX_QPAIRS]; /* encoded */ struct string qpairsdec[MAX_QPAIRS]; /* decoded */ int nqpairs; /* how many is stored */ static void trurl_cleanup_options(struct option *o) { if(!o) return; curl_slist_free_all(o->url_list); curl_slist_free_all(o->set_list); curl_slist_free_all(o->iter_list); curl_slist_free_all(o->append_query); curl_slist_free_all(o->trim_list); curl_slist_free_all(o->append_path); } static void errorf(struct option *o, int exit_code, char *fmt, ...) { va_list ap; va_start(ap, fmt); fputs(ERROR_PREFIX, stderr); vfprintf(stderr, fmt, ap); fputs("\n" ERROR_PREFIX "Try " PROGNAME " -h for help\n", stderr); va_end(ap); trurl_cleanup_options(o); curl_global_cleanup(); exit(exit_code); } static char *strurldecode(const char *url, int inlength, int *outlength) { return curl_easy_unescape(NULL, inlength ? url : "", inlength, outlength); } static void urladd(struct option *o, const char *url) { struct curl_slist *n; n = curl_slist_append(o->url_list, url); if(n) o->url_list = n; } /* read URLs from this file/stdin */ static void urlfile(struct option *o, const char *file) { FILE *f; if(o->url) errorf(o, ERROR_FLAG, "only one --url-file is supported"); if(strcmp("-", file)) { f = fopen(file, "rt"); if(!f) errorf(o, ERROR_FILE, "--url-file %s not found", file); o->urlopen = true; } else f = stdin; o->url = f; } static void pathadd(struct option *o, const char *path) { struct curl_slist *n; char *urle = curl_easy_escape(NULL, path, 0); if(urle) { n = curl_slist_append(o->append_path, urle); if(n) { o->append_path = n; } curl_free(urle); } } static void queryadd(struct option *o, const char *query) { struct curl_slist *n; char *p = strchr(query, '='); char *urle; if(p) { /* URL encode the left and the right side of the '=' separately */ char *f1 = curl_easy_escape(NULL, query, p - query); char *f2 = curl_easy_escape(NULL, p + 1, 0); urle = curl_maprintf("%s=%s", f1, f2); curl_free(f1); curl_free(f2); } else urle = curl_easy_escape(NULL, query, 0); if(urle) { n = curl_slist_append(o->append_query, urle); if(n) { o->append_query = n; } curl_free(urle); } } static void appendadd(struct option *o, const char *arg) { if(!strncmp("path=", arg, 5)) pathadd(o, arg + 5); else if(!strncmp("query=", arg, 6)) queryadd(o, arg + 6); else errorf(o, ERROR_APPEND, "--append unsupported component: %s", arg); } static void setadd(struct option *o, const char *set) /* [component]=[data] */ { struct curl_slist *n; n = curl_slist_append(o->set_list, set); if(n) o->set_list = n; } static void iteradd(struct option *o, const char *iter) /* [component]=[data] */ { struct curl_slist *n; n = curl_slist_append(o->iter_list, iter); if(n) o->iter_list = n; } static void trimadd(struct option *o, const char *trim) /* [component]=[data] */ { struct curl_slist *n; n = curl_slist_append(o->trim_list, trim); if(n) o->trim_list = n; } static bool checkoptarg(struct option *o, const char *str, const char *given, const char *arg) { if(!strcmp(str, given)) { if(!arg) errorf(o, ERROR_ARG, "Missing argument for %s", str); return true; } return false; } static int getarg(struct option *o, const char *flag, const char *arg, bool *usedarg) { *usedarg = false; if(!strcmp("--", flag)) o->end_of_options = true; else if(!strcmp("-v", flag) || !strcmp("--version", flag)) show_version(); else if(!strcmp("-h", flag) || !strcmp("--help", flag)) help(); else if(checkoptarg(o, "--url", flag, arg)) { urladd(o, arg); *usedarg = true; } else if(checkoptarg(o, "-f", flag, arg) || checkoptarg(o, "--url-file", flag, arg)) { urlfile(o, arg); *usedarg = true; } else if(checkoptarg(o, "-a", flag, arg) || checkoptarg(o, "--append", flag, arg)) { appendadd(o, arg); *usedarg = true; } else if(checkoptarg(o, "-s", flag, arg) || checkoptarg(o, "--set", flag, arg)) { setadd(o, arg); *usedarg = true; } else if(checkoptarg(o, "--iterate", flag, arg)) { iteradd(o, arg); *usedarg = true; } else if(checkoptarg(o, "--redirect", flag, arg)) { if(o->redirect) errorf(o, ERROR_FLAG, "only one --redirect is supported"); o->redirect = arg; *usedarg = true; } else if(checkoptarg(o, "--query-separator", flag, arg)) { if(o->qsep) errorf(o, ERROR_FLAG, "only one --query-separator is supported"); if(strlen(arg) != 1) errorf(o, ERROR_FLAG, "only single-letter query separators are supported"); o->qsep = arg; *usedarg = true; } else if(checkoptarg(o, "--trim", flag, arg)) { trimadd(o, arg); *usedarg = true; } else if(checkoptarg(o, "-g", flag, arg) || checkoptarg(o, "--get", flag, arg)) { if(o->format) errorf(o, ERROR_FLAG, "only one --get is supported"); if(o->jsonout) errorf(o, ERROR_FLAG, "--get is mututally exclusive with --json"); o->format = arg; *usedarg = true; } else if(!strcmp("--json", flag)) { if(o->format) errorf(o, ERROR_FLAG, "--json is mututally exclusive with --get"); o->jsonout = true; } else if(!strcmp("--verify", flag)) o->verify = true; else if(!strcmp("--accept-space", flag)) { #ifdef SUPPORTS_ALLOW_SPACE o->accept_space = true; #else trurl_warnf(o, "built with too old libcurl version, --accept-space does not work"); #endif } else if(!strcmp("--curl", flag)) o->curl = true; else if(!strcmp("--default-port", flag)) o->default_port = true; else if(!strcmp("--keep-port", flag)) o->keep_port = true; else if(!strcmp("--punycode", flag)) { if(o->puny2idn) errorf(o, ERROR_FLAG, "--punycode is mutually exclusive with --as-idn"); o->punycode = true; } else if(!strcmp("--as-idn", flag)) { if(o->punycode) errorf(o, ERROR_FLAG, "--as-idn is mutually exclusive with --punycode"); o->puny2idn = true; } else if(!strcmp("--no-guess-scheme", flag)) o->no_guess_scheme = true; else if(!strcmp("--sort-query", flag)) o->sort_query = true; else if(!strcmp("--urlencode", flag)) o->urlencode = true; else if(!strcmp("--quiet", flag)) o->quiet_warnings = true; else return 1; /* unrecognized option */ return 0; } static void showqkey(FILE *stream, const char *key, size_t klen, bool urldecode, bool showall) { int i; bool shown = false; struct string *qp = urldecode ? qpairsdec : qpairs; for(i = 0; i< nqpairs; i++) { if(!strncmp(key, qp[i].str, klen) && (qp[i].str[klen] == '=')) { if(shown) fputc(' ', stream); fputs(&qp[i].str[klen + 1], stream); if(!showall) break; shown = true; } } } /* component to variable pointer */ static const struct var *comp2var(const char *name, size_t vlen) { int i; for(i = 0; variables[i].name; i++) if((strlen(variables[i].name) == vlen) && !strncmp(name, variables[i].name, vlen)) return &variables[i]; return NULL; } static CURLUcode geturlpart(struct option *o, int modifiers, CURLU *uh, CURLUPart part, char **out) { CURLUcode rc = curl_url_get(uh, part, out, (((modifiers & VARMODIFIER_DEFAULT) || o->default_port) ? CURLU_DEFAULT_PORT : ((part != CURLUPART_URL || o->keep_port) ? 0 : CURLU_NO_DEFAULT_PORT))| #ifdef SUPPORTS_PUNYCODE (((modifiers & VARMODIFIER_PUNY) || o->punycode) ? CURLU_PUNYCODE : 0)| #endif #ifdef SUPPORTS_PUNY2IDN (((modifiers & VARMODIFIER_PUNY2IDN) || o->puny2idn) ? CURLU_PUNY2IDN : 0) | #endif (o->curl ? 0 : CURLU_NON_SUPPORT_SCHEME)| (((modifiers & VARMODIFIER_URLENCODED) || o->urlencode) ? 0 :CURLU_URLDECODE)); #ifdef SUPPORTS_PUNY2IDN /* retry get w/ out puny2idn to handle invalid punycode conversions */ if(rc == CURLUE_BAD_HOSTNAME && (o->puny2idn || (modifiers & VARMODIFIER_PUNY2IDN))) { curl_free(*out); modifiers &= ~VARMODIFIER_PUNY2IDN; o->puny2idn = false; trurl_warnf(o, "Error converting url to IDN [%s]", curl_url_strerror(rc)); return geturlpart(o, modifiers, uh, part, out); } #endif return rc; } static void showurl(FILE *stream, struct option *o, int modifiers, CURLU *uh) { char *url; CURLUcode rc = geturlpart(o, modifiers, uh, CURLUPART_URL, &url); if(rc) { trurl_cleanup_options(o); VERIFY(o, ERROR_BADURL, "invalid url [%s]", curl_url_strerror(rc)); return; } fputs(url, stream); curl_free(url); } static void get(struct option *o, CURLU *uh) { FILE *stream = stdout; const char *ptr = o->format; bool done = false; char startbyte = 0; char endbyte = 0; while(ptr && *ptr && !done) { if(!startbyte && (('{' == *ptr) || ('[' == *ptr))) { startbyte = *ptr; if('{' == *ptr) endbyte = '}'; else endbyte = ']'; } if(startbyte == *ptr) { if(startbyte == ptr[1]) { /* an escaped {-letter */ fputc(startbyte, stream); ptr += 2; } else { /* this is meant as a variable to output */ const char *start = ptr; char *end; char *cl; size_t vlen; bool isquery = false; bool queryall = false; int mods = 0; end = strchr(ptr, endbyte); ptr++; /* pass the { */ if(!end) { /* syntax error */ fputc(startbyte, stream); continue; } /* {path} {:path} */ if(*ptr == ':') { mods |= VARMODIFIER_URLENCODED; ptr++; } vlen = end - ptr; do { cl = memchr(ptr, ':', vlen); if(!cl) break; /* modifiers! */ if(!strncmp(ptr, "default:", cl - ptr + 1)) mods |= VARMODIFIER_DEFAULT; else if(!strncmp(ptr, "puny:", cl - ptr + 1)) { if(mods & VARMODIFIER_PUNY2IDN) errorf(o, ERROR_GET, "puny modifier is mutually exclusive with idn"); mods |= VARMODIFIER_PUNY; } else if(!strncmp(ptr, "idn:", cl - ptr + 1)) { if(mods & VARMODIFIER_PUNY) errorf(o, ERROR_GET, "idn modifier is mutually exclusive with puny"); mods |= VARMODIFIER_PUNY2IDN; } else { /* {query: or {query-all: */ if(!strncmp(ptr, "query-all:", cl - ptr + 1)) { isquery = true; queryall = true; } else if(!strncmp(ptr, "query:", cl - ptr + 1)) isquery = true; else { /* syntax error */ vlen = 0; end[1] = '\0'; } break; } ptr = cl + 1; vlen = end - ptr; } while(true); if(isquery) { showqkey(stream, cl + 1, end - cl - 1, !o->urlencode && !(mods & VARMODIFIER_URLENCODED), queryall); } else if(!vlen) errorf(o, ERROR_GET, "Bad --get syntax: %s", start); else if(!strncmp(ptr, "url", vlen)) showurl(stream, o, mods, uh); else { const struct var *v = comp2var(ptr, vlen); if(v) { char *nurl; CURLUcode rc = geturlpart(o, mods, uh, v->part, &nurl); switch(rc) { case CURLUE_OK: fputs(nurl, stream); curl_free(nurl); case CURLUE_NO_SCHEME: case CURLUE_NO_USER: case CURLUE_NO_PASSWORD: case CURLUE_NO_OPTIONS: case CURLUE_NO_HOST: case CURLUE_NO_PORT: case CURLUE_NO_QUERY: case CURLUE_NO_FRAGMENT: #ifdef SUPPORTS_ZONEID case CURLUE_NO_ZONEID: #endif /* silently ignore */ break; default: trurl_warnf(o, "%s (%s)\n", curl_url_strerror(rc), v->name); break; } } } ptr = end + 1; /* pass the end */ } } else if('\\' == *ptr && ptr[1]) { switch(ptr[1]) { case 'r': fputc('\r', stream); break; case 'n': fputc('\n', stream); break; case 't': fputc('\t', stream); break; case '\\': fputc('\\', stream); break; case '{': fputc('{', stream); break; case '[': fputc('[', stream); break; default: /* unknown, just output this */ fputc(*ptr, stream); fputc(ptr[1], stream); break; } ptr += 2; } else { fputc(*ptr, stream); ptr++; } } fputc('\n', stream); } static const struct var *setone(CURLU *uh, const char *setline, struct option *o) { char *ptr = strchr(setline, '='); const struct var *v = NULL; if(ptr && (ptr > setline)) { size_t vlen = ptr - setline; bool urlencode = true; bool found = false; if(ptr[-1] == ':') { urlencode = false; vlen--; } v = comp2var(setline, vlen); if(v) { CURLUcode rc; rc = curl_url_set(uh, v->part, ptr[1] ? &ptr[1] : NULL, (o->curl ? 0 : CURLU_NON_SUPPORT_SCHEME)| (urlencode ? CURLU_URLENCODE : 0) ); if(rc) warnf("Error setting %s: %s", v->name, curl_url_strerror(rc)); found = true; } if(!found) errorf(o, ERROR_SET, "unknown component: %.*s", (int)vlen, setline); } else errorf(o, ERROR_SET, "invalid --set syntax: %s", setline); return v; } static unsigned int set(CURLU *uh, struct option *o) { struct curl_slist *node; unsigned int mask = 0; for(node = o->set_list; node; node = node->next) { const struct var *v; char *setline = node->data; v = setone(uh, setline, o); if(v) { if(mask & (1 << v->part)) errorf(o, ERROR_SET, "duplicate --set for component %s", v->name); mask |= (1 << v->part); } } return mask; /* the set components */ } static void jsonString(FILE *stream, const char *in, size_t len, bool lowercase) { const unsigned char *i = (unsigned char *)in; const char *in_end = &in[len]; fputc('\"', stream); for(; i < (unsigned char *)in_end; i++) { switch(*i) { case '\\': fputs("\\\\", stream); break; case '\"': fputs("\\\"", stream); break; case '\b': fputs("\\b", stream); break; case '\f': fputs("\\f", stream); break; case '\n': fputs("\\n", stream); break; case '\r': fputs("\\r", stream); break; case '\t': fputs("\\t", stream); break; default: if(*i < 32) fprintf(stream, "\\u%04x", *i); else { char out = *i; if(lowercase && (out >= 'A' && out <= 'Z')) /* do not use tolower() since that's locale specific */ out |= ('a' - 'A'); fputc(out, stream); } break; } } fputc('\"', stream); } static void json(struct option *o, CURLU *uh) { int i; bool first = true; char *url; CURLUcode rc = geturlpart(o, 0, uh, CURLUPART_URL, &url); if(rc) { trurl_cleanup_options(o); VERIFY(o, ERROR_BADURL, "invalid url [%s]", curl_url_strerror(rc)); return; } printf("%s\n {\n \"url\": ", o->urls ? "," : ""); jsonString(stdout, url, strlen(url), false); curl_free(url); fputs(",\n \"parts\": {\n", stdout); for(i = 0; variables[i].name; i++) { char *part; rc = geturlpart(o, 0, uh, variables[i].part, &part); if(!rc) { if(!first) fputs(",\n", stdout); first = false; printf(" \"%s\": ", variables[i].name); jsonString(stdout, part, strlen(part), false); curl_free(part); } } fputs("\n }", stdout); first = true; if(nqpairs) { int j; fputs(",\n \"params\": [\n", stdout); for(j = 0 ; j < nqpairs; j++) { const char *sep = strchr(qpairsdec[j].str, '='); const char *value = sep ? sep + 1 : ""; /* don't print out empty/trimmed values */ if(!qpairsdec[j].str[0]) continue; if(!first) fputs(",\n", stdout); first = false; fputs(" {\n \"key\": ", stdout); jsonString(stdout, qpairsdec[j].str, sep ? (size_t)(sep - qpairsdec[j].str) : qpairsdec[j].len, false); fputs(",\n \"value\": ", stdout); jsonString(stdout, sep?value:"", sep?qpairsdec[j].len:0, false); fputs("\n }", stdout); } fputs("\n ]", stdout); } fputs("\n }", stdout); } /* --trim query="utm_*" */ static void trim(struct option *o) { struct curl_slist *node; for(node = o->trim_list; node; node = node->next) { char *instr = node->data; if(strncmp(instr, "query", 5)) /* for now we can only trim query components */ errorf(o, ERROR_TRIM, "Unsupported trim component: %s", instr); char *ptr = strchr(instr, '='); if(ptr && (ptr > instr)) { /* 'ptr' should be a fixed string or a pattern ending with an asterisk */ size_t inslen; bool pattern = false; int i; char *temp = NULL; ptr++; /* pass the = */ inslen = strlen(ptr); if(inslen) { pattern = ptr[inslen - 1] == '*'; if(pattern && (inslen > 1)) { pattern ^= ptr[inslen - 2] == '\\'; if(!pattern) { /* the two final letters are \*, but the backslash needs to be removed. Get a copy and edit that accordingly. */ temp = strdup(ptr); if(!temp) return; /* out of memory, bail out */ temp[inslen - 2] = '*'; temp[inslen - 1] = '\0'; ptr = temp; inslen--; /* one byte shorter now */ } } if(pattern) inslen--; } for(i = 0 ; i < nqpairs; i++) { char *q = qpairs[i].str; char *sep = strchr(q, '='); size_t qlen; if(sep) qlen = sep - q; else qlen = strlen(q); if((pattern && (inslen <= qlen) && !strncasecmp(q, ptr, inslen)) || (!pattern && (inslen == qlen) && !strncasecmp(q, ptr, inslen))) { /* this qpair should be stripped out */ free(qpairs[i].str); curl_free(qpairsdec[i].str); qpairs[i].str = strdup(""); /* marked as deleted */ qpairs[i].len = 0; qpairsdec[i].str = strdup(""); /* marked as deleted */ qpairsdec[i].len = 0; } } free(temp); } } } /* memdup the amount and add a trailing zero */ struct string *memdupzero(char *source, size_t len) { struct string *ret = malloc(sizeof(struct string)); if(!ret) return NULL; ret->str = malloc(len + 1); if(!ret->str) { free(ret); return NULL; } memcpy(ret->str, source, len); ret->str[len] = 0; ret->len = len; return ret; } /* URL decode the pair and return it in an allocated chunk */ struct string *memdupdec(char *source, size_t len, bool json) { char *sep = memchr(source, '=', len); char *left = NULL; char *right = NULL; int right_len = 0; int left_len = 0; char *str; left = strurldecode(source, sep ? (size_t)(sep - source) : len, &left_len); if(sep) { char *p; int plen; right = strurldecode(sep + 1, len - (sep - source) - 1, (int *)&right_len); /* convert null bytes to periods */ for(plen = right_len, p = right; plen; plen--, p++) { if(!*p && !json) { *p = REPLACE_NULL_BYTE; } } } str = curl_maprintf("%.*s%s%.*s", left_len, left, right ? "=":"", right_len, right?right:""); /* handle strings with null characters */ if(sep && right) { memcpy(str + left_len + 1, right, right_len); } curl_free(right); curl_free(left); struct string *ret = malloc(sizeof(struct string)); if(!ret) { return NULL; } ret->str = str; if(right) ret->len = right_len; else { ret->len = left_len; } return ret; } static void freeqpairs(void) { int i; for(i = 0; istr; qpairs[nqpairs].len = p->len; qpairsdec[nqpairs].str = pdec->str; qpairsdec[nqpairs].len = pdec->len; nqpairs++; } } else warnf("too many query pairs"); if(p) ret = p->str; if(pdec) free(pdec); if(p) free(p); return ret; } /* convert the query string into an array of name=data pair */ static void extractqpairs(CURLU *uh, struct option *o) { char *q = NULL; memset(qpairs, 0, sizeof(qpairs)); nqpairs = 0; /* extract the query */ if(!curl_url_get(uh, CURLUPART_QUERY, &q, 0)) { char *p = q; char *amp; while(*p) { size_t len; amp = strchr(p, o->qsep[0]); if(!amp) len = strlen(p); else len = amp - p; addqpair(p, len, o->jsonout); if(amp) p = amp + 1; else break; } } curl_free(q); } static void qpair2query(CURLU *uh, struct option *o) { int i; char *nq = NULL; for(i = 0; iqsep: "", qpairs[i].str); curl_free(oldnq); } if(nq) { int rc = curl_url_set(uh, CURLUPART_QUERY, nq, 0); if(rc) trurl_warnf(o, "internal problem"); } curl_free(nq); } /* sort case insensitively */ static int cmpfunc(const void *p1, const void *p2) { int len = (((struct string *)p1)->len) < (((struct string *)p2)->len)? (((struct string *)p1)->len):(((struct string *)p2)->len); for(int i = 0; i < len; i++) { char c1 = ((struct string *)p1)->str[i] | ('a' - 'A'); char c2 = ((struct string *)p2)->str[i] | ('a' - 'A'); if(c1 != c2) return c1 - c2; } return 0; } static void sortquery(struct option *o) { if(o->sort_query) { /* not these two lists may no longer be the same order after the sort */ qsort(&qpairs[0], nqpairs, sizeof(struct string), cmpfunc); qsort(&qpairsdec[0], nqpairs, sizeof(struct string), cmpfunc); } } static CURLUcode seturl(struct option *o, CURLU *uh, const char *url) { return curl_url_set(uh, CURLUPART_URL, url, (o->no_guess_scheme ? 0 : CURLU_GUESS_SCHEME)| (o->curl ? 0 : CURLU_NON_SUPPORT_SCHEME)| (o->accept_space ? CURLU_ALLOW_SPACE : 0)| CURLU_URLENCODE); } static void singleurl(struct option *o, const char *url, /* might be NULL */ struct iterinfo *iinfo, struct curl_slist *iter) { CURLU *uh = iinfo->uh; if(!uh) { uh = curl_url(); if(!uh) errorf(o, ERROR_MEM, "out of memory"); if(url) { CURLUcode rc = seturl(o, uh, url); if(rc) { curl_url_cleanup(uh); VERIFY(o, ERROR_BADURL, "%s [%s]", curl_url_strerror(rc), url); return; } if(o->redirect) { rc = seturl(o, uh, o->redirect); if(rc) { curl_url_cleanup(uh); VERIFY(o, ERROR_BADURL, "invalid redirection: %s [%s]", curl_url_strerror(rc), o->redirect); return; } } } } do { struct curl_slist *p; bool url_is_invalid = false; unsigned setmask = 0; /* set everything */ setmask = set(uh, o); if(iter) { char iterbuf[1024]; /* "part=item1 item2 item2" */ const char *part; size_t plen; const char *w; size_t wlen; char *sep; bool urlencode = true; const struct var *v; if(!iinfo->ptr) { part = iter->data; sep = strchr(part, '='); if(!sep) errorf(o, ERROR_ITER, "wrong iterate syntax"); plen = sep - part; if(sep[-1] == ':') { urlencode = false; plen--; } w = sep + 1; /* store for next lap */ iinfo->part = part; iinfo->plen = plen; v = comp2var(part, plen); if(!v) { curl_url_cleanup(uh); errorf(o, ERROR_ITER, "bad component for iterate"); } if(iinfo->varmask & (1<part)) { curl_url_cleanup(uh); errorf(o, ERROR_ITER, "duplicate component for iterate: %s", v->name); } if(setmask & (1 << v->part)) { curl_url_cleanup(uh); errorf(o, ERROR_ITER, "duplicate --iterate and --set for component %s", v->name); } } else { part = iinfo->part; plen = iinfo->plen; v = comp2var(part, plen); w = iinfo->ptr; } sep = strchr(w, ' '); if(sep) { wlen = sep - w; iinfo->ptr = sep + 1; /* next word is here */ } else { /* last word */ wlen = strlen(w); iinfo->ptr = NULL; } curl_msnprintf(iterbuf, sizeof(iterbuf), "%.*s%s=%.*s", (int)plen, part, urlencode ? "" : ":", (int)wlen, w); setone(uh, iterbuf, o); if(iter->next) { struct iterinfo info; memset(&info, 0, sizeof(info)); info.uh = uh; info.varmask = iinfo->varmask | (1 << v->part); singleurl(o, url, &info, iter->next); } } /* append path segments */ for(p = o->append_path; p; p = p->next) { char *apath = p->data; char *opath; char *npath; size_t olen; /* extract the current path */ curl_url_get(uh, CURLUPART_PATH, &opath, 0); /* does the existing path end with a slash, then don't add one in between */ olen = strlen(opath); /* append the new segment */ npath = curl_maprintf("%s%s%s", opath, opath[olen-1] == '/' ? "" : "/", apath); if(npath) { /* set the new path */ curl_url_set(uh, CURLUPART_PATH, npath, 0); } curl_free(npath); curl_free(opath); } extractqpairs(uh, o); /* trim parts */ trim(o); /* append query segments */ for(p = o->append_query; p; p = p->next) { addqpair(p->data, strlen(p->data), o->jsonout); } sortquery(o); /* put the query back */ qpair2query(uh, o); /* make sure the URL is still valid */ if(!url || o->redirect || o->set_list || o->append_path) { char *ourl = NULL; CURLUcode rc = curl_url_get(uh, CURLUPART_URL, &ourl, 0); if(rc) { if(o->verify) /* only clean up if we're exiting */ curl_url_cleanup(uh); VERIFY(o, ERROR_URL, "not enough input for a URL"); url_is_invalid = true; } else { rc = seturl(o, uh, ourl); if(rc) { if(o->verify) /* only clean up if we're exiting */ curl_url_cleanup(uh); VERIFY(o, ERROR_BADURL, "%s [%s]", curl_url_strerror(rc), ourl); url_is_invalid = true; } else { char *nurl = NULL; rc = curl_url_get(uh, CURLUPART_URL, &nurl, 0); if(!rc) curl_free(nurl); else { if(o->verify) /* only clean up if we're exiting */ curl_url_cleanup(uh); VERIFY(o, ERROR_BADURL, "url became invalid"); url_is_invalid = true; } } curl_free(ourl); } } if(iter && iter->next) ; else if(url_is_invalid) ; else if(o->jsonout) json(o, uh); else if(o->format) { /* custom output format */ get(o, uh); } else { /* default output is full URL */ char *nurl = NULL; int rc = geturlpart(o, 0, uh, CURLUPART_URL, &nurl); if(!rc) { printf("%s\n", nurl); curl_free(nurl); } } fflush(stdout); freeqpairs(); o->urls++; } while(iinfo->ptr); if(!iinfo->uh) curl_url_cleanup(uh); } int main(int argc, const char **argv) { int exit_status = 0; struct option o; struct curl_slist *node; memset(&o, 0, sizeof(o)); setlocale(LC_ALL, ""); curl_global_init(CURL_GLOBAL_ALL); for(argc--, argv++; argc > 0; argc--, argv++) { bool usedarg = false; if(!o.end_of_options && argv[0][0] == '-') { /* dash-dash prefixed */ if(getarg(&o, argv[0], argv[1], &usedarg)) errorf(&o, ERROR_FLAG, "unknown option: %s", argv[0]); } else { /* this is a URL */ urladd(&o, argv[0]); } if(usedarg) { /* skip the parsed argument */ argc--; argv++; } } if(!o.qsep) o.qsep = "&"; if(o.jsonout) putchar('['); if(o.url) { /* this is a file to read URLs from */ char buffer[4096]; /* arbitrary max */ bool end_of_file = false; while(!end_of_file && fgets(buffer, sizeof(buffer), o.url)) { char *eol = strchr(buffer, '\n'); if(eol && (eol > buffer)) { if(eol[-1] == '\r') /* CRLF detected */ eol--; } else if(eol == buffer) { /* empty line */ continue; } else if(feof(o.url)) { /* end of file */ eol = strlen(buffer) + buffer; end_of_file = true; } else { /* line too long */ int ch; trurl_warnf(&o, "skipping long line"); do { ch = getc(o.url); } while(ch != EOF && ch != '\n'); if(ch == EOF) { if(ferror(o.url)) trurl_warnf(&o, "getc: %s", strerror(errno)); end_of_file = true; } continue; } /* trim trailing spaces and tabs */ while((eol > buffer) && ((eol[-1] == ' ') || eol[-1] == '\t')) eol--; if(eol > buffer) { /* if there is actual content left to deal with */ struct iterinfo iinfo; memset(&iinfo, 0, sizeof(iinfo)); *eol = 0; /* end of URL */ singleurl(&o, buffer, &iinfo, o.iter_list); } } if(!end_of_file && ferror(o.url)) trurl_warnf(&o, "fgets: %s", strerror(errno)); if(o.urlopen) fclose(o.url); } else { /* not reading URLs from a file */ node = o.url_list; do { if(node) { const char *url = node->data; struct iterinfo iinfo; memset(&iinfo, 0, sizeof(iinfo)); singleurl(&o, url, &iinfo, o.iter_list); node = node->next; } else { struct iterinfo iinfo; memset(&iinfo, 0, sizeof(iinfo)); o.verify = true; singleurl(&o, NULL, &iinfo, o.iter_list); } } while(node); } if(o.jsonout) printf("%s]\n", o.urls ? "\n" : ""); /* we're done with libcurl, so clean it up */ trurl_cleanup_options(&o); curl_global_cleanup(); return exit_status; } curl-trurl-4f06332/version.h000066400000000000000000000021551452017613700157450ustar00rootroot00000000000000#ifndef TRURL_VERSION_H #define TRURL_VERSION_H /*************************************************************************** * _ _ ____ _ * Project ___| | | | _ \| | * / __| | | | |_) | | * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * * Copyright (C) Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms * are also available at https://curl.se/docs/copyright.html. * * You may opt to use, copy, modify, merge, publish, distribute and/or sell * copies of the Software, and permit persons to whom the Software is * furnished to do so, under the terms of the COPYING file. * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY * KIND, either express or implied. * * SPDX-License-Identifier: curl * ***************************************************************************/ #define TRURL_VERSION_TXT "0.8" #endif curl-trurl-4f06332/winbuild/000077500000000000000000000000001452017613700157215ustar00rootroot00000000000000curl-trurl-4f06332/winbuild/.vcpkg000066400000000000000000000003251452017613700170340ustar00rootroot00000000000000# git clone https://github.com/microsoft/vcpkg.git # .\vcpkg\bootstrap-vcpkg.bat .\vcpkg\vcpkg install curl:x86-windows-static-md .\vcpkg\vcpkg install curl:x64-windows-static-md .\vcpkg\vcpkg integrate install curl-trurl-4f06332/winbuild/README.md000066400000000000000000000055331452017613700172060ustar00rootroot00000000000000 # Building trurl with Microsoft C++ Build Tools Download and install [Microsoft C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) When installing, choose the `Desktop development with C++` option. ## Open a command prompt Open the **x64 Native Tools Command Prompt for VS 2022**, or if you are on an x86 platform **x86 Native Tools Command Prompt for VS 2022** ## Set up the vcpkg repository Note: The location of the vcpkg repository does not necessarily need to correspond to the trurl directory, it can be set up anywhere. But it is recommended to use a short path such as `C:\src\vcpkg` or `C:\dev\vcpkg`, since otherwise you may run into path issues for some port build systems. Once you are in the console, run the below commands to clone the vcpkg repository and set up the curl dependencies: ~~~ git clone https://github.com/microsoft/vcpkg.git .\vcpkg\bootstrap-vcpkg.bat .\vcpkg\vcpkg install curl:x86-windows-static-md .\vcpkg\vcpkg install curl:x64-windows-static-md .\vcpkg\vcpkg integrate install ~~~ Once the vcpkg repository is set up you do not need to run these commands again. If a newer version of curl is released, you may need to run `git pull` in the vcpkg repository and then `vcpkg upgrade` to fetch the new version. ## Build in the console Once the vcpkg repository and dependencies are set up, go to the winbuild directory in the trurl sources: cd trurl\winbuild Then you can call the build command with the desired parameters. The builds will be placed in an output directory as described below. ## Parameters - The `Configuration` parameter can be set to either `Debug` or `Release` - The `Platform` parameter can be set to either `x86` or `x64` ## Build commands - x64 Debug: `msbuild /m /t:Clean,Build /p:Configuration=Debug /p:Platform=x64 trurl.sln` - x64 Release: `msbuild /m /t:Clean,Build /p:Configuration=Release /p:Platform=x64 trurl.sln` - x86 Debug: `msbuild /m /t:Clean,Build /p:Configuration=Debug /p:Platform=x86 trurl.sln` - x86 Release: `msbuild /m /t:Clean,Build /p:Configuration=Release /p:Platform=x86 trurl.sln` Note: If you are using the x64 Native Tools Command Prompt you can also run the x86 build commands. ## Output directories The output files will be placed in: `winbuild\bin\\\` PDB files will be generated in the same directory as the executable for Debug builds, but they will not be generated for release builds. Intermediate files will be placed in: `winbuild\obj\\\` These include build logs and obj files. ## Tests Tests can be run by going to the directory of the output files in the console and running `perl .\..\..\..\..\test.pl` You will need perl installed to run the tests, such as [Strawberry Perl](https://strawberryperl.com/) curl-trurl-4f06332/winbuild/trurl.sln000066400000000000000000000026261452017613700176150ustar00rootroot00000000000000 Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.33516.290 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "trurl", "trurl.vcxproj", "{575657CF-843F-491C-B15B-881C28DF36CA}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 Debug|x86 = Debug|x86 Release|x64 = Release|x64 Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {575657CF-843F-491C-B15B-881C28DF36CA}.Debug|x64.ActiveCfg = Debug|x64 {575657CF-843F-491C-B15B-881C28DF36CA}.Debug|x64.Build.0 = Debug|x64 {575657CF-843F-491C-B15B-881C28DF36CA}.Debug|x86.ActiveCfg = Debug|Win32 {575657CF-843F-491C-B15B-881C28DF36CA}.Debug|x86.Build.0 = Debug|Win32 {575657CF-843F-491C-B15B-881C28DF36CA}.Release|x64.ActiveCfg = Release|x64 {575657CF-843F-491C-B15B-881C28DF36CA}.Release|x64.Build.0 = Release|x64 {575657CF-843F-491C-B15B-881C28DF36CA}.Release|x86.ActiveCfg = Release|Win32 {575657CF-843F-491C-B15B-881C28DF36CA}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {14A4D782-313F-4F61-A2C5-EF2CD877D3F3} EndGlobalSection EndGlobal curl-trurl-4f06332/winbuild/trurl.vcxproj000066400000000000000000000221111452017613700205030ustar00rootroot00000000000000 Debug Win32 Release Win32 Debug x64 Release x64 16.0 Win32Proj {575657cf-843f-491c-b15b-881c28df36ca} trurl 10.0 Application true v143 Unicode Application false v143 true Unicode Application true v143 Unicode Application false v143 true Unicode $(SolutionDir)bin\$(Platform)\$(Configuration)\ $(SolutionDir)obj\$(Platform)\$(Configuration)\ $(SolutionDir)bin\$(Platform)\$(Configuration)\ $(SolutionDir)obj\$(Platform)\$(Configuration)\ $(SolutionDir)bin\$(PlatformShortName)\$(Configuration)\ $(SolutionDir)obj\$(PlatformShortName)\$(Configuration)\ $(SolutionDir)bin\$(PlatformShortName)\$(Configuration)\ $(SolutionDir)obj\$(PlatformShortName)\$(Configuration)\ true true true x64-windows true true x64-windows true true x86-windows true true x86-windows Level4 true WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true Console true ws2_32.lib;wldap32.lib;advapi32.lib;crypt32.lib;Normaliz.lib Level4 true true true WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true Console true true false ws2_32.lib;wldap32.lib;advapi32.lib;crypt32.lib;Normaliz.lib Level4 true _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true Console true ws2_32.lib;wldap32.lib;advapi32.lib;crypt32.lib;Normaliz.lib Level4 true true true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true Console true true false ws2_32.lib;wldap32.lib;advapi32.lib;crypt32.lib;Normaliz.lib curl-trurl-4f06332/winbuild/vcpkg-configuration.json000066400000000000000000000004621452017613700225750ustar00rootroot00000000000000{ "default-registry": { "kind": "git", "baseline": "43401f5835f97f48180724bdeb49a8e4a994b848", "repository": "https://github.com/microsoft/vcpkg" }, "registries": [ { "kind": "artifact", "location": "https://aka.ms/vcpkg-ce-default", "name": "microsoft" } ] } curl-trurl-4f06332/winbuild/vcpkg.json000066400000000000000000000001161452017613700177240ustar00rootroot00000000000000{ "name": "trurl", "version": "0.x", "dependencies": [ "curl" ] }