pax_global_header00006660000000000000000000000064152353055130014514gustar00rootroot0000000000000052 comment=9061cc3e06a00c04cd92965dae46850f2785a91b videolabs-mirrorbits-441567e/000077500000000000000000000000001523530551300161375ustar00rootroot00000000000000videolabs-mirrorbits-441567e/.github/000077500000000000000000000000001523530551300174775ustar00rootroot00000000000000videolabs-mirrorbits-441567e/.github/workflows/000077500000000000000000000000001523530551300215345ustar00rootroot00000000000000videolabs-mirrorbits-441567e/.github/workflows/ci.yml000066400000000000000000000014001523530551300226450ustar00rootroot00000000000000name: test on: push: pull_request: jobs: build: name: Build and Test runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: go-version: [1.18, 1.24, 'stable'] steps: - name: Check out code uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Build run: | make - name: Run tests run: | make test - name: Create release tarball if: startsWith(github.ref, 'refs/tags/') run: | make release - uses: actions/upload-artifact@v4 if: startsWith(github.ref, 'refs/tags/') with: name: release-${{ matrix.go-version }} path: dist/* videolabs-mirrorbits-441567e/.gitignore000066400000000000000000000000301523530551300201200ustar00rootroot00000000000000bin dist tmp *~ .vscode videolabs-mirrorbits-441567e/.travis.yml000066400000000000000000000012501523530551300202460ustar00rootroot00000000000000language: go sudo: required # https://docs.travis-ci.com/user/languages/go/#go-import-path go_import_path: github.com/etix/mirrorbits go: - "1.11.x" - "1.12.x" - "1.13.x" - master os: - linux matrix: allow_failures: - go: master before_install: - sudo apt-get -qq update - sudo apt-get install -y libgeoip-dev - curl -L https://github.com/google/protobuf/releases/download/v3.9.1/protoc-3.9.1-linux-x86_64.zip -o /tmp/protoc.zip - unzip /tmp/protoc.zip -d "$HOME"/protoc install: - go version - export GOBIN="$GOPATH/bin" - export PROTOCBIN="$HOME/protoc/bin" - export PATH="$PATH:$PROTOCBIN:$GOBIN" - go env script: - make && make test videolabs-mirrorbits-441567e/CHANGELOG.md000066400000000000000000000224271523530551300177570ustar00rootroot00000000000000## v0.6.2 ### FEATURES - New option `GeographicalSort` to disable the sorting of mirrors by distance (#225) - Support for rsync over TLS, using `rsyncs://` URLs (#203) - Show why a mirror is down in `mirrorbits list` (#221) ### ENHANCEMENTS - Use IEC prefixes (KiB, MiB...) for file sizes on the web pages (#205) - Use a longer timeout when removing a mirror (#206) - Improve the scan logs (#201) - Update the OpenStreetMap tile server URL (#204) - Improve spacing in the CLI table outputs (#224) ### BUGFIXES - Fix mirror name matching on the CLI when a name is a substring of another (#134) - Fix a bogus error forwarding in the RPC layer (#220) ### Changes - Go 1.18+ is now required, and the vendor tree has been removed (#214) - Redis 4.0+ is now required (#223) - `mirrorbits list`: the header line is now uppercase (#224) - Replace deprecated dependencies (`io/ioutil`, `pkg/errors`, `gopass`) with the standard library (#210, #216, #217, #219) ## v0.6.1 ### BUGFIXES - Regression: mirrorbits returned "500 Internal Server Error" when the Redis database was not ready, instead of redirecting users to the fallback mirror(s) (#195) - Fix malformed redirections when the fallback URL(s) (in the configuration file) lacks a trailing slash (c6abff6) ## v0.6 ### FEATURES - New command `mirrorbits logs ` to make per-mirror logs available on the CLI (#5) - New command `mirrorbits geoupdate ` to update the geolocation of a mirror (#96) - New option `FixTimezoneOffsets` to detect and automatically fix timezone shifts on mirrors (mostly for those using FTP) (2d9d467) - New option `SameDownloadInterval` to avoid counting very close range downloads from a same source (#128) - New option `AllowHTTPToHTTPSRedirects` to allow (default) or disallow redirections of HTTP requests to HTTPS mirrors (d108400) - New option `AllowOutdatedFiles` to allow redirections to outdated files on the mirrors, under certain conditions (#85, #188) - Add support for `If-Modified-Since` aka. RFC-7232 (#169) - Support for HTTP+HTTPS mirrors: - so far, a mirror was defined as either HTTP or HTTPS, via the `HttpURL` field - now, it's possible to set a URL without a scheme (eg. `HttpURL: mirror.example.org/some/path/`), in that case mirrorbits performs two health checks (HTTP and HTTPS), and can redirect to either HTTP or HTTPS depending on the context - this "scheme-less" URL also works for the fallback mirrors defined in `mirrorbits.conf` - see Changes below for more changes related to this feature ### ENHANCEMENTS - Enforce checks on modtime based on FTP and rsync capabilities - Use `type=notify` in the systemd service file to indicate readiness of the http server (#90) - Make unauthorized redirect errors more visible - Require HTTPS based on `X-Forwarded-Proto` header (this can still be overridden by the `?https` parameter) (#97) - Do not list disabled mirrors as down (#132) - Add Bash completion ### BUGFIXES - Fix a race condition in automatic mirror scan - Restore case-insensitive mirror name matching on the CLI - Fix outdated entries in the LRU cache under certain conditions (#114) ### Changes to support HTTP+HTTPS mirrors - This new version includes a **DATABASE UPGRADE**. You won't be able to roll back to a previous mirrorbits version after upgrading. The db upgrade should be fast. - Daemon logs: the protocol used for the health-check is now logged: - before: ` mirror.example.org Up! (509ms)` - after : ` mirror.example.org HTTPS Up! (509ms)` - Command `mirrorbits list`: for the STATE column, new values are possible for HTTP+HTTPS mirrors: - `up/down` if HTTP health-check succeeded but HTTPS health-check failed - `down/up` for the other way round - HTML templates have been updated to support HTTP+HTTPS mirrors. Make sure to use the latest templates `mirrorlist.html` and `mirrorstats.html`. ### Other Changes - Use Go modules (Go 1.11+) - Downloads logs: the method of the request is now logged just before the path: - before: ` REDIRECT 302 "/README" [...]` - after : ` REDIRECT 302 GET "/README" [...]` ## v0.5.1 ### ENHANCEMENTS - Sort the mirrors by the last state date in the list command ### BUGFIXES - Regression: mirrors were not able to transition between up and down states ## v0.5 ### FEATURES - Allow renaming a mirror directly from `mirrorbits edit` - Option to exclude a country from being served by a mirror ### ENHANCEMENTS - Use of GeoIP2 mmdb databases - RPC between the CLI and the server - Use SHA256 as new default hash - General improvements on the web templates - Google Maps replaced by OpenStreetMap (#74) - Google Charts replaced by Flot (#76) - Possibility to fetch and serve Javascript locally without relying on CDNs (#76) - Dockerfile improvements - Systemd service file with process isolation ### BUGFIXES - Add the Redis database index in pubsub announcements (#75) - Exclude partial directories from rsync (#64) ### Changes - JSON API: - Name contains the name of the mirror (previously known as ID) - ID now contains the unique ID of the mirror ## v0.4 ### FEATURES - Allow negative scores to reduce the weight of a mirror - Follow symbolic links within a repository - Allow/Disallow per-mirror redirects configuration - Display the sync offset between each mirror and the source on the mirrorstats page (requires a trace file on the repository) - New cli option to force a rehashing of all files during a refresh - Added a Dockerfile ### ENHANCEMENTS - Support password protected rsync URLs - Allow https URLs when adding a mirror - Display location and score in the list output - Display mirror status in the stats output - Improvements in the selection algorithm - Load OSM tiles using https - Keep the list of mirrors sorted by score in the mirrorlist - Set cache-control to disable caching - Log unauthorized redirection from a mirror - New option to set the maximum number of backup mirrors to return in link headers - Support for Google Maps API keys - Mirrorlist and Mirrorstats UI refresh - Use UTC time on mirrorlist / mirrorstats page - Improved error reporting - Add dependency vendoring ### BUGFIXES - Fix a possible crash while Redis is loading the dataset - Fix a race condition when updating mirrors state - Fix a rare deadlock within the FTP client ## v0.3 ### FEATURES - Support for HA via Redis sentinel - Clustering support (multiple Mirrorbits instances) [#6](https://github.com/etix/mirrorbits/issues/6) - Support for Redis DB index - SHA256 and MD5 hashing support (in addition to SHA1) [#4](https://github.com/etix/mirrorbits/issues/4) - Configurable interval for sync and check - CLI: get stats by matching regular expressions - HTTP: get the checksum of any file by appending ?sha1, ?sha256 or ?md5 to any served file - Added a Makefile to support different builds ### ENHANCEMENTS - Improved systemd service file - New mirrorlist template [#15](https://github.com/etix/mirrorbits/issues/15) - Geoip databases are now updated (in memory) during a reload - Reuse all Redis connections when possible - Detect and wait until Redis has loaded the dataset into memory - Improved handling of X-Forwarded-For IP addresses [#23](https://github.com/etix/mirrorbits/issues/23) - Logging: enable the colored output only if supported by the terminal - More configuration items can be applied with a simple reload - Improved scan behavior for newly added mirror (healthcheck only after successful scan) - Limit redis verbosity in CLI operations - CLI: reduce the number of database requests required to fetch stats by time interval - CLI: differentiate down vs disabled mirrors - FTP: add a connection timeout - Don't try to open download logs when using the cli - process: ensure the file descriptor is valid before finalizing a seamless binary upgrade - Mirrors with a weight less than 1% will show <1% instead - Graceful exit is now faster - General improvements on error reporting ### BUGFIXES - Fix Redis password authentication - Fix a crash in the weight randomization algorithm - Fix a bug causing a rescan of all mirrors during startup - Fix a bug causing some disabled mirrors to be health-checked - Don't reload logs if outputting on stderr (journald is now happy) - Fix a crash if no mirrors and no fallbacks are available - CLI: fix matching of a mirror ID containing the same substring [#19](https://github.com/etix/mirrorbits/issues/19) - scan: fix an issue causing a constant rehashing of all files [#18](https://github.com/etix/mirrorbits/issues/18) - The geoip-lite-update script did not update the databases correctly ## v0.2 ### FEATURES - Request a scan using a specific protocol (rsync or ftp) - Print basic download stats (mirrorbits stats ) ### ENHANCEMENTS - Improve parse errors in the configuration - Don't log if logdir is unset ### BUGFIXES - Fix a minor corner case when the client and server are in the exact same location ## v0.1.2 ### BUGFIXES - Fix a possible division by zero during mirror selection ## v0.1.1 ### FEATURES - CLI: a parse error in the mirror configuration can now be retried - CLI: add support for taking notes / comments on a mirror - CLI: add a command-line flag to auto-enable a mirror after a successful scan - CLI: add a flag to scan all mirrors at once ### ENHANCEMENTS - Improved mirror selection algorithm ### BUGFIXES - Fix a few corner cases in weight distribution ## v0.1.0 Initial release videolabs-mirrorbits-441567e/Dockerfile000066400000000000000000000016261523530551300201360ustar00rootroot00000000000000FROM golang:latest LABEL maintainer="etix@l0cal.com" ADD . /go/mirrorbits RUN apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y pkg-config zlib1g-dev protobuf-compiler libprotoc-dev rsync && \ apt-get clean RUN go get -u github.com/maxmind/geoipupdate2/cmd/geoipupdate && \ go install -ldflags "-X main.defaultConfigFile=/etc/GeoIP.conf -X main.defaultDatabaseDirectory=/usr/share/GeoIP" github.com/maxmind/geoipupdate2/cmd/geoipupdate && \ echo "AccountID 0\nLicenseKey 000000000000\nEditionIDs GeoLite2-City GeoLite2-Country GeoLite2-ASN" > /etc/GeoIP.conf && \ mkdir /usr/share/GeoIP && \ /go/bin/geoipupdate RUN mkdir /srv/repo /var/log/mirrorbits && \ cd /go/mirrorbits && \ make install PREFIX=/usr RUN cp /go/mirrorbits/contrib/docker/mirrorbits.conf /etc/mirrorbits.conf ENTRYPOINT /usr/bin/mirrorbits daemon -config /etc/mirrorbits.conf EXPOSE 8080 videolabs-mirrorbits-441567e/LICENSE.txt000066400000000000000000000020751523530551300177660ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2014-2019 Ludovic Fauvet Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.videolabs-mirrorbits-441567e/Makefile000066400000000000000000000054141523530551300176030ustar00rootroot00000000000000.PHONY: all build dev clean release test installdirs install uninstall install-service uninstall-service service-systemd regen-proto VERSION := $(shell git describe --always --dirty --tags) SHA := $(shell git rev-parse --short HEAD) BRANCH := $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) BUILD := $(SHA)-$(BRANCH) BINARY_NAME := mirrorbits BINARY := bin/$(BINARY_NAME) TARBALL := dist/mirrorbits-$(VERSION).tar.gz TEMPLATES := templates/ ifneq (${DESTDIR}$(PREFIX),) TEMPLATES = ${DESTDIR}$(PREFIX)/share/mirrorbits endif PREFIX ?= /usr/local PACKAGE = github.com/etix/mirrorbits LDFLAGS := -X $(PACKAGE)/core.VERSION=$(VERSION) -X $(PACKAGE)/core.BUILD=$(BUILD) -X $(PACKAGE)/config.TEMPLATES_PATH=${TEMPLATES} GOFLAGS := -ldflags "$(LDFLAGS)" GOFLAGSDEV := -race -ldflags "$(LDFLAGS) -X $(PACKAGE)/core.DEV=-dev" GOPATH ?= $(HOME)/go export PATH := $(GOPATH)/bin:$(PATH) PKG_CONFIG ?= /usr/bin/pkg-config SERVICEDIR_SYSTEMD ?= $(shell $(PKG_CONFIG) systemd --variable=systemdsystemunitdir) all: build regen-proto: rpc/rpc.proto @ if ! which protoc > /dev/null; then \ echo "error: protoc not installed" >&2; \ exit 1; \ fi go install github.com/golang/protobuf/protoc-gen-go@v1.3.5 && \ rm -f rpc/rpc.pb.go && \ protoc -I rpc rpc/rpc.proto --go_out=plugins=grpc:rpc build: GO111MODULE=on go build $(GOFLAGS) -o $(BINARY) . dev: GO111MODULE=on go build $(GOFLAGSDEV) -o $(BINARY) . clean: @echo Cleaning workspace... @rm -f $(BINARY) @rm -f contrib/init/systemd/mirrorbits.service @rm -dRf dist release: $(TARBALL) test: GO111MODULE=on go test $(GOFLAGS) ./... installdirs: mkdir -p ${DESTDIR}${PREFIX}/{bin,share} ${DESTDIR}$(PREFIX)/share/mirrorbits install: build installdirs install-service # For the 'make install' to work with sudo it might be necessary to add # the Go binary path to the 'secure_path' and add 'GOPATH' to 'env_keep'. @cp -vf $(BINARY) ${DESTDIR}${PREFIX}/bin/ @cp -vf templates/* ${DESTDIR}$(PREFIX)/share/mirrorbits uninstall: uninstall-service @rm -vf ${DESTDIR}${PREFIX}/bin/$(BINARY_NAME) @rm -vfr ${DESTDIR}$(PREFIX)/share/mirrorbits ifeq (,${SERVICEDIR_SYSTEMD}) install-service: uninstall-service: else install-service: service-systemd install -Dm644 contrib/init/systemd/mirrorbits.service ${DESTDIR}${SERVICEDIR_SYSTEMD}/mirrorbits.service uninstall-service: @rm -vf ${DESTDIR}${SERVICEDIR_SYSTEMD}/mirrorbits.service service-systemd: @sed "s|##PREFIX##|$(PREFIX)|" contrib/init/systemd/mirrorbits.service.in > contrib/init/systemd/mirrorbits.service endif $(TARBALL): build @echo Packaging release... @mkdir -p tmp/mirrorbits @cp -f $(BINARY) tmp/mirrorbits/ @cp -r templates tmp/mirrorbits/ @cp mirrorbits.conf tmp/mirrorbits/ @mkdir -p dist/ @tar -czf $@ -C tmp mirrorbits && echo release tarball has been created: $@ @rm -rf tmp videolabs-mirrorbits-441567e/README.md000066400000000000000000000200121523530551300174110ustar00rootroot00000000000000[![Build Status](https://travis-ci.org/etix/mirrorbits.svg?branch=master)](https://travis-ci.org/etix/mirrorbits) [![Go Report Card](https://goreportcard.com/badge/github.com/etix/mirrorbits)](https://goreportcard.com/report/github.com/etix/mirrorbits) Mirrorbits =========== Mirrorbits is a geographical download redirector written in [Go](https://golang.org) for distributing files efficiently across a set of mirrors. It offers a simple and economic way to create a Content Delivery Network layer using a pure software stack. It is primarily designed for the distribution of large-scale Open-Source projects with a lot of traffic. ![mirrorbits_screenshot](https://cloud.githubusercontent.com/assets/38853/3636687/ab6bba38-0fd8-11e4-9d69-01543ed2531a.png) ## Main Features * Blazing fast, can reach 8K QPS on a single laptop * Easy to deploy and maintain, everything is packed in a single binary * Automatic synchronization with the mirrors over **rsync** or **FTP** * Response can be either JSON or HTTP redirect * Support partial repositories * Complete checksum / size control * Realtime monitoring and reports * Disable misbehaving mirrors without human intervention * Realtime decision making based on location, AS number and defined rules * Smart load-balancing over multiple mirrors in the same area to avoid hotspots * Ability to adjust the weight of each mirror * Limit access to a country, region or ASN for any mirror * Clustering (multiple mirrorbits instances) * High-availability using redis-sentinel * Automatically fix timezone offsets for broken mirrors * Realtime statistics per file / mirror / date * Realtime reconfiguration * Seamless binary upgrade (aka zero downtime upgrade) * [Mirmon](http://www.staff.science.uu.nl/~penni101/mirmon/) support * Full **IPv6** support * If-Modified-Since (RFC-7232) support * more... ## Is it production ready? **Yes!** Mirrorbits has served **billions** of files already and is known to be running in production at: * [CarbonROM](https://carbonrom.org/) * [Chaos Computer Club](https://media.ccc.de/) to distribute media * [Jellyfin](https://jellyfin.org/) since [April 2021](https://jellyfin.org/posts/mirrorbits-cdn/) * [Jenkins](https://www.jenkins.io/) to distribute Jenkins releases since [February 2020](https://github.com/jenkins-infra/docker-mirrorbits) * [Kali Linux](https://www.kali.org/) to distribute packages and images since [December 2023](https://www.kali.org/blog/kali-linux-2023-4-release/#enters-mirrorbits) * [Kodi](http://kodi.tv/) (previously XBMC) since [July 2015](https://forum.kodi.tv/showthread.php?tid=233824) * [LineageOS](http://lineageos.org/) (previously CyanogenMod) since January 2017 * [MariaDB](https://mariadb.org/) to distribute packages (deb/rpm) for Linux distributions since [December 2021](https://mariadb.org/mirrorbits/) * [OSMC](https://osmc.tv) * [VideoLAN](http://www.videolan.org/) to distribute [VLC media player](http://www.videolan.org/vlc/) since [April 2014](https://blog.l0cal.com/2014/07/11/mirrorbits-is-now-on-github/) * [Endless OS](https://endlessos.org/os) * [MSYS2](https://www.msys2.org/) to distribute packages since [June 2021](https://github.com/msys2/msys2-main-server/commit/6a212b9ac76913f96) Yet some things might change before the 1.0 release. If you intend to deploy Mirrorbits in a production system it is advised to notify the author first so we can help you to make any transition as seamless as possible! _Previous projects which have used Mirrorbits:_ * [Parrot OS](https://www.parrotsec.org) * [Popcorn Time](https://popcorntime.io) * [SuperRepo](https://superrepo.org) # Quick start ## Prerequisites * Go 1.18 or later * Protobuf (protoc) * Redis 3.2 or later (with [persistence](https://redis.io/topics/persistence) enabled) * GeoIP2 databases from [Maxmind](https://dev.maxmind.com/geoip/geoip2/geolite2/) (preferably updated regularly) :warning: **GeoIP-legacy is not supported anymore, please use the new GeoIP2 mmdb databases!** **Optional:** * redis-sentinel (for high-availability support) ## Upgrading Before upgrading to the latest version, please check [this guide](https://github.com/etix/mirrorbits/wiki/Upgrade-Guide). ## Installation You can either get a [prebuilt version](https://github.com/etix/mirrorbits/releases) or choose to build it yourself. ### Docker A docker "quick start" can be found [on the wiki](https://github.com/etix/mirrorbits/wiki/Running-within-Docker). ### Manual build ``` $ git clone https://github.com/etix/mirrorbits.git $ cd mirrorbits $ sudo make install ``` The resulting executable should now live in your */usr/local/bin* directory. You can also specify a `PREFIX` or `DESTDIR` if necessary: ``` sudo make install PREFIX=/usr ``` ## Configuration A sample configuration file can be found [here](mirrorbits.conf). ## Running Mirrorbits is a self-contained application and can act, at the same time, as the server and the cli. To run the server: ``` mirrorbits daemon ``` Additional options can be found with ```mirrorbits -help```. To run the cli: ``` mirrorbits help ``` Add a mirror: ``` mirrorbits add -ftp="ftp://ftp.mirrors.example/myproject/" -http="http://ftp.mirrors.example/myproject/" mirrors.example ``` Enable the mirror: ``` mirrorbits enable mirrors.example ``` ### Realtime file availability By appending `?mirrorlist` to any file served by mirrorbits, you'll be able to get some useful realtime informations about the given file. You can see a [live example here](https://get.videolan.org/vlc/2.2.4/win32/vlc-2.2.4-win32.exe?mirrorlist). ### Realtime mirrors statistics Mirror statistics are available by querying mirrorbits with the `?mirrorstats` argument. You can see a [live example here](https://get.videolan.org/?mirrorstats). ## Clustering / High availability Multiple instances of mirrorbits can be started simultaneously on different servers, discovery of other nodes should be automatic as long as all the instances are connected to the same redis server. In addition to the clustering it is advised to use redis-sentinel to monitor the database and gracefully handle failover. ## Upgrading Mirrorbits has a mode called *seamless binary upgrade* to upgrade the server executable at runtime without service disruption. Once the binary has been replaced on the filesystem just issue the following command in the cli: ``` mirrorbits upgrade ``` ## Considerations * When configured in redirect mode, Mirrorbits can easily serve client requests directly but it is usually recommended to set it behind a reverse proxy like nginx. In this case take care to pass the IP address of the client within a X-Forwarded-For header: ``` proxy_set_header X-Forwarded-For $remote_addr; ``` * It is advised to never cache requests intended for Mirrorbits since each request is supposed to be unique, caching the result might have unexpected consequences. # We're social! The best place to discuss about mirrorbits is to join the [#VideoLAN IRC channel on Libera.chat](https://www.videolan.org/webirc/). For the latest news, you can follow [@mirrorbits](http://twitter.com/mirrorbits) on Twitter. # License MIT > Permission is hereby granted, free of charge, to any person obtaining a copy > of this software and associated documentation files (the "Software"), to deal > in the Software without restriction, including without limitation the rights > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell > copies of the Software, and to permit persons to whom the Software is > furnished to do so, subject to the following conditions: > > The above copyright notice and this permission notice shall be included in > all copies or substantial portions of the Software. > > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN > THE SOFTWARE. videolabs-mirrorbits-441567e/cli/000077500000000000000000000000001523530551300167065ustar00rootroot00000000000000videolabs-mirrorbits-441567e/cli/commands.go000066400000000000000000000722361523530551300210500ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package cli import ( "bufio" "bytes" "context" "errors" "flag" "fmt" "net/url" "os" "os/exec" "reflect" "sort" "strings" "sync" "text/tabwriter" "time" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/rpc" "github.com/etix/mirrorbits/utils" "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/empty" "github.com/op/go-logging" "golang.org/x/term" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "gopkg.in/yaml.v3" ) const ( commentSeparator = "##### Comments go below this line #####" defaultRPCTimeout = time.Second * 10 ) var ( log = logging.MustGetLogger("main") ) type cli struct { sync.Mutex rpcconn *grpc.ClientConn creds *loginCreds } // ParseCommands parses the command line and call the appropriate functions func ParseCommands(args ...string) error { c := &cli{ creds: &loginCreds{ Password: core.RPCPassword, }, } if len(args) > 0 && args[0] != "help" { method, exists := c.getMethod(args[0]) if !exists { fmt.Println("Error: Command not found:", args[0]) return c.CmdHelp() } if len(c.creds.Password) == 0 && core.RPCAskPass { fmt.Print("Password: ") passwd, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println("") if err != nil { return err } c.creds.Password = string(passwd) } ret := method.Func.CallSlice([]reflect.Value{ reflect.ValueOf(c), reflect.ValueOf(args[1:]), })[0].Interface() if c.rpcconn != nil { c.rpcconn.Close() } if ret == nil { return nil } return ret.(error) } return c.CmdHelp() } func (c *cli) getMethod(name string) (reflect.Method, bool) { methodName := "Cmd" + strings.ToUpper(name[:1]) + strings.ToLower(name[1:]) return reflect.TypeOf(c).MethodByName(methodName) } func (c *cli) CmdHelp() error { help := fmt.Sprintf("Usage: mirrorbits [OPTIONS] COMMAND [arg...]\n\nA smart download redirector.\n\n") help += fmt.Sprintf("Server commands:\n %-10.10s%s\n\n", "daemon", "Start the server") help += fmt.Sprintf("CLI commands:\n") for _, command := range [][]string{ {"add", "Add a new mirror"}, {"disable", "Disable a mirror"}, {"edit", "Edit a mirror"}, {"enable", "Enable a mirror"}, {"export", "Export the mirror database"}, {"geoupdate", "Update geolocation of a mirror"}, {"list", "List all mirrors"}, {"logs", "Print logs of a mirror"}, {"refresh", "Refresh the local repository"}, {"reload", "Reload configuration"}, {"remove", "Remove a mirror"}, {"scan", "(Re-)Scan a mirror"}, {"show", "Print a mirror configuration"}, {"stats", "Show download stats"}, {"upgrade", "Seamless binary upgrade"}, {"version", "Print version information"}, } { help += fmt.Sprintf(" %-10.10s%s\n", command[0], command[1]) } fmt.Fprintf(os.Stderr, "%s\n", help) return nil } // SubCmd prints the usage of a subcommand func SubCmd(name, signature, description string) *flag.FlagSet { flags := flag.NewFlagSet(name, flag.ContinueOnError) flags.Usage = func() { fmt.Fprintf(os.Stderr, "\nUsage: mirrorbits %s %s\n\n%s\n\n", name, signature, description) flags.PrintDefaults() } return flags } type ByDate []*rpc.Mirror func (d ByDate) Len() int { return len(d) } func (d ByDate) Swap(i, j int) { d[i], d[j] = d[j], d[i] } func (d ByDate) Less(i, j int) bool { return d[i].StateSince.Seconds > d[j].StateSince.Seconds } func (c *cli) CmdList(args ...string) error { cmd := SubCmd("list", "", "Get the list of mirrors") http := cmd.Bool("http", false, "Print HTTP addresses") rsync := cmd.Bool("rsync", false, "Print rsync addresses") ftp := cmd.Bool("ftp", false, "Print FTP addresses") location := cmd.Bool("location", false, "Print the country and continent code") state := cmd.Bool("state", true, "Print the state of the mirror") score := cmd.Bool("score", false, "Print the score of the mirror") disabled := cmd.Bool("disabled", false, "List disabled mirrors only") enabled := cmd.Bool("enabled", false, "List enabled mirrors only") down := cmd.Bool("down", false, "List only mirrors currently down") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 0 { cmd.Usage() return nil } client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() list, err := client.List(ctx, &empty.Empty{}) if err != nil { log.Fatal("list error:", err) } sort.Sort(ByDate(list.Mirrors)) w := new(tabwriter.Writer) w.Init(os.Stdout, 0, 8, 1, '\t', 0) fmt.Fprint(w, "IDENTIFIER") if *score == true { fmt.Fprint(w, "\tSCORE") } if *http == true { fmt.Fprint(w, "\tHTTP") } if *rsync == true { fmt.Fprint(w, "\tRSYNC") } if *ftp == true { fmt.Fprint(w, "\tFTP") } if *location == true { fmt.Fprint(w, "\tLOCATION") } if *state == true { fmt.Fprint(w, "\tSTATE\tSINCE\tREASON") } fmt.Fprint(w, "\n") for _, mirror := range list.Mirrors { if *disabled == true { if mirror.Enabled == true { continue } } if *enabled == true { if mirror.Enabled == false { continue } } if *down == true { if IsUp(mirror) || mirror.Enabled == false { continue } } stateSince, err := ptypes.Timestamp(mirror.StateSince) if err != nil { log.Fatal("list error:", err) } fmt.Fprintf(w, "%s", mirror.Name) if *score == true { fmt.Fprintf(w, "\t%d", mirror.Score) } if *http == true { fmt.Fprintf(w, "\t%s", mirror.HttpURL) } if *rsync == true { fmt.Fprintf(w, "\t%s", mirror.RsyncURL) } if *ftp == true { fmt.Fprintf(w, "\t%s", mirror.FtpURL) } if *location == true { countries := strings.Split(mirror.CountryCodes, " ") countryCode := "/" if len(countries) >= 1 { countryCode = countries[0] } fmt.Fprintf(w, "\t%s (%s)", countryCode, mirror.ContinentCode) } if *state == true { status := "disabled" reason := "" if mirror.Enabled == true { status = StatusString(mirror) reason = ReasonString(mirror) } since := stateSince.Format(time.RFC1123) fmt.Fprintf(w, "\t%s\t(%s)\t%s", status, since, reason) } fmt.Fprint(w, "\n") } w.Flush() return nil } func IsHTTPOnly(m *rpc.Mirror) bool { return strings.HasPrefix(m.HttpURL, "http://") } func IsHTTPSOnly(m *rpc.Mirror) bool { return strings.HasPrefix(m.HttpURL, "https://") } func IsUp(m *rpc.Mirror) bool { if m.HttpUp == m.HttpsUp { return m.HttpUp } if IsHTTPOnly(m) { return m.HttpUp } if IsHTTPSOnly(m) { return m.HttpsUp } return false } func StatusString(m *rpc.Mirror) string { var http string var https string if m.HttpUp { http = "up" } else { http = "down" } if m.HttpsUp { https = "up" } else { https = "down" } if m.HttpUp == m.HttpsUp { return http } if IsHTTPOnly(m) { return http } if IsHTTPSOnly(m) { return https } return fmt.Sprintf("%s/%s", http, https) } func ReasonString(m *rpc.Mirror) string { var http string var https string http = m.HttpDownReason https = m.HttpsDownReason if http == https { return http } if IsHTTPOnly(m) { return http } if IsHTTPSOnly(m) { return https } if http == "" { return https } if https == "" { return http } return fmt.Sprintf("%s / %s", http, https) } func (c *cli) CmdAdd(args ...string) error { cmd := SubCmd("add", "[OPTIONS] IDENTIFIER", "Add a new mirror") http := cmd.String("http", "", "HTTP base URL") rsync := cmd.String("rsync", "", "RSYNC base URL (for scanning only)") ftp := cmd.String("ftp", "", "FTP base URL (for scanning only)") sponsorName := cmd.String("sponsor-name", "", "Name of the sponsor") sponsorURL := cmd.String("sponsor-url", "", "URL of the sponsor") sponsorLogo := cmd.String("sponsor-logo", "", "URL of a logo to display for this mirror") adminName := cmd.String("admin-name", "", "Admin's name") adminEmail := cmd.String("admin-email", "", "Admin's email") customData := cmd.String("custom-data", "", "Associated data to return when the mirror is selected (i.e. json document)") continentOnly := cmd.Bool("continent-only", false, "The mirror should only handle its continent") countryOnly := cmd.Bool("country-only", false, "The mirror should only handle its country") asOnly := cmd.Bool("as-only", false, "The mirror should only handle clients in the same AS number") score := cmd.Int("score", 0, "Weight to give to the mirror during selection") comment := cmd.String("comment", "", "Comment") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() < 1 { cmd.Usage() return nil } if strings.Contains(cmd.Arg(0), " ") { fmt.Fprintf(os.Stderr, "The identifier cannot contain a space\n") os.Exit(-1) } if *http == "" { fmt.Fprintf(os.Stderr, "You *must* pass at least an HTTP URL\n") os.Exit(-1) } if utils.HasAnyPrefix(*http, "http://", "https://") { _, err := url.Parse(*http) if err != nil { fmt.Fprintf(os.Stderr, "Can't parse HTTP URL\n") os.Exit(-1) } } else if strings.Contains(*http, "://") { fmt.Fprintf(os.Stderr, "The HTTP URL has an invalid scheme\n") os.Exit(-1) } else { // No scheme, yes we do accept it. // Note that the documentation of net/url mentions that parsing // such URL is "invalid but may not necessarily return an error", // so let's add a scheme before we parse it. _, err := url.Parse("http://" + *http) if err != nil { fmt.Fprintf(os.Stderr, "Can't parse HTTP URL\n") os.Exit(-1) } } mirror := &mirrors.Mirror{ Name: cmd.Arg(0), HttpURL: *http, RsyncURL: *rsync, FtpURL: *ftp, SponsorName: *sponsorName, SponsorURL: *sponsorURL, SponsorLogoURL: *sponsorLogo, AdminName: *adminName, AdminEmail: *adminEmail, CustomData: *customData, ContinentOnly: *continentOnly, CountryOnly: *countryOnly, ASOnly: *asOnly, Score: *score, Comment: *comment, } client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() m, err := rpc.MirrorToRPC(mirror) if err != nil { log.Fatal("edit error:", err) } reply, err := client.AddMirror(ctx, m) if err != nil { if err.Error() == rpc.ErrNameAlreadyTaken.Error() { log.Fatalf("Mirror %s already exists!\n", mirror.Name) } log.Fatal("edit error:", err) } for i := 0; i < len(reply.Warnings); i++ { fmt.Println(reply.Warnings[i]) if i == len(reply.Warnings)-1 { fmt.Println("") } } if reply.Country != "" { fmt.Println("Mirror location:") fmt.Printf("Latitude: %.4f\n", reply.Latitude) fmt.Printf("Longitude: %.4f\n", reply.Longitude) fmt.Printf("Continent: %s\n", reply.Continent) fmt.Printf("Country: %s\n", reply.Country) fmt.Printf("ASN: %s\n", reply.ASN) fmt.Println("") } fmt.Printf("Mirror '%s' added successfully\n", mirror.Name) fmt.Printf("Enable this mirror using\n $ mirrorbits enable %s\n", mirror.Name) return nil } func (c *cli) CmdRemove(args ...string) error { cmd := SubCmd("remove", "IDENTIFIER", "Remove an existing mirror") force := cmd.Bool("f", false, "Never prompt for confirmation") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } id, name := c.matchMirror(cmd.Arg(0)) if *force == false { fmt.Printf("Removing %s, are you sure? [y/N]", name) reader := bufio.NewReader(os.Stdin) s, _ := reader.ReadString('\n') switch s[0] { case 'y', 'Y': break default: return nil } } client := c.GetRPC() // Use a timeout longer than the default, removing a mirror can take time ctx, cancel := context.WithTimeout(context.Background(), time.Second * 60) defer cancel() _, err := client.RemoveMirror(ctx, &rpc.MirrorIDRequest{ ID: int32(id), }) if err != nil { log.Fatal("remove error:", err) } fmt.Printf("Mirror '%s' removed successfully\n", name) return nil } func (c *cli) CmdScan(args ...string) error { cmd := SubCmd("scan", "[IDENTIFIER]", "(Re-)Scan a mirror") enable := cmd.Bool("enable", false, "Enable the mirror automatically if the scan is successful") all := cmd.Bool("all", false, "Scan all mirrors at once") ftp := cmd.Bool("ftp", false, "Force a scan using FTP") rsync := cmd.Bool("rsync", false, "Force a scan using rsync") timeout := cmd.Uint("timeout", 0, "Timeout in seconds") if err := cmd.Parse(args); err != nil { return nil } if !*all && cmd.NArg() != 1 || *all && cmd.NArg() != 0 { cmd.Usage() return nil } client := c.GetRPC() ctx, cancel := context.WithCancel(context.Background()) defer cancel() list := make(map[int]string) // Get the list of mirrors to scan if *all == true { reply, err := client.MatchMirror(ctx, &rpc.MatchRequest{ Pattern: "", // Match all of them }) if err != nil { return errors.New("Cannot fetch the list of mirrors") } for _, m := range reply.Mirrors { list[int(m.ID)] = m.Name } } else { // Single mirror id, name := c.matchMirror(cmd.Arg(0)) list[id] = name } // Set the method of the scan (if not default) var method rpc.ScanMirrorRequest_Method if *ftp == false && *rsync == false { method = rpc.ScanMirrorRequest_ALL } else if *rsync == true { method = rpc.ScanMirrorRequest_RSYNC } else if *ftp == true { method = rpc.ScanMirrorRequest_FTP } for id, name := range list { if *timeout > 0 { ctx, cancel = context.WithTimeout(context.Background(), time.Duration(*timeout)*time.Second) defer cancel() } fmt.Printf("Scanning %s... ", name) reply, err := client.ScanMirror(ctx, &rpc.ScanMirrorRequest{ ID: int32(id), AutoEnable: *enable, Protocol: method, }) if err != nil { s := status.Convert(err) if s.Code() == codes.FailedPrecondition || len(list) == 1 { return errors.New("\nscan error: " + grpc.ErrorDesc(err)) } fmt.Println("scan error:", grpc.ErrorDesc(err)) continue } else { fmt.Printf("%d files indexed, %d known and %d removed\n", reply.FilesIndexed, reply.KnownIndexed, reply.Removed) if reply.GetTZOffsetMs() != 0 { fmt.Printf(" ∟ Timezone offset detected and corrected: %d milliseconds\n", reply.TZOffsetMs) } if reply.Enabled { fmt.Println(" ∟ Enabled") } } } return nil } func (c *cli) CmdRefresh(args ...string) error { cmd := SubCmd("refresh", "", "Scan the local repository") rehash := cmd.Bool("rehash", false, "Force a rehash of the files") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 0 { cmd.Usage() return nil } fmt.Print("Refreshing the local repository... ") client := c.GetRPC() ctx, cancel := context.WithCancel(context.Background()) defer cancel() _, err := client.RefreshRepository(ctx, &rpc.RefreshRepositoryRequest{ Rehash: *rehash, }) if err != nil { fmt.Println("") log.Fatal(err) } fmt.Println("done") return nil } func (c *cli) matchMirror(pattern string) (id int, name string) { if len(pattern) == 0 { return -1, "" } client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() reply, err := client.MatchMirror(ctx, &rpc.MatchRequest{ Pattern: pattern, }) if err != nil { fmt.Fprintf(os.Stderr, "mirror matching: %s\n", err) os.Exit(1) } switch len(reply.Mirrors) { case 0: fmt.Fprintf(os.Stderr, "No match for '%s'\n", pattern) os.Exit(1) case 1: id, name, err := GetSingle(reply.Mirrors) if err != nil { log.Fatal("unexpected error:", err) } return id, name default: fmt.Fprintln(os.Stderr, "Multiple match:") for _, mirror := range reply.Mirrors { fmt.Fprintf(os.Stderr, " %s\n", mirror.Name) } os.Exit(1) } return } func GetSingle(list []*rpc.MirrorID) (int, string, error) { if len(list) == 0 { return -1, "", errors.New("list is empty") } else if len(list) > 1 { return -1, "", errors.New("too many results") } return int(list[0].ID), list[0].Name, nil } func (c *cli) CmdEdit(args ...string) error { cmd := SubCmd("edit", "[IDENTIFIER]", "Edit a mirror") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } // Find the editor to use editor := os.Getenv("EDITOR") if editor == "" { for _, p := range []string{"editor", "vi", "emacs", "nano"} { _, err := exec.LookPath(p) if err == nil { editor = p break } } if editor == "" { log.Fatal("No text editor found, please set the EDITOR environment variable") } } id, _ := c.matchMirror(cmd.Arg(0)) client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() rpcm, err := client.MirrorInfo(ctx, &rpc.MirrorIDRequest{ ID: int32(id), }) if err != nil { log.Fatal("edit error:", err) } mirror, err := rpc.MirrorFromRPC(rpcm) if err != nil { log.Fatal("edit error:", err) } // Generate a yaml configuration string from the struct out, err := yaml.Marshal(mirror) // Open a temporary file f, err := os.CreateTemp("", "edit") if err != nil { log.Fatal("Cannot create temporary file:", err) } defer os.Remove(f.Name()) f.WriteString("# You can now edit this mirror configuration.\n" + "# Just save and quit when you're done.\n\n") f.WriteString(string(out)) f.WriteString(fmt.Sprintf("\n%s\n\n%s\n", commentSeparator, mirror.Comment)) f.Close() // Checksum the original file chk, _ := filesystem.Sha256sum(f.Name()) reopen: // Launch the editor with the filename as first parameter exe := exec.Command(editor, f.Name()) exe.Stdin = os.Stdin exe.Stdout = os.Stdout exe.Stderr = os.Stderr err = exe.Run() if err != nil { log.Fatal(err) } // Read the file back out, err = os.ReadFile(f.Name()) if err != nil { log.Fatal("Cannot read file", f.Name()) } // Checksum the file back and compare chk2, _ := filesystem.Sha256sum(f.Name()) if bytes.Compare(chk, chk2) == 0 { fmt.Println("Aborted - settings are unmodified, so there is nothing to change.") return nil } var comment string yamlstr := string(out) commentIndex := strings.Index(yamlstr, commentSeparator) if commentIndex > 0 { comment = strings.TrimSpace(yamlstr[commentIndex+len(commentSeparator):]) yamlstr = yamlstr[:commentIndex] } reopen := func(err error) bool { eagain: fmt.Printf("%s\nRetry? [Y/n]", err.Error()) reader := bufio.NewReader(os.Stdin) s, _ := reader.ReadString('\n') switch s[0] { case 'y', 'Y', 10: return true case 'n', 'N': fmt.Println("Aborted") return false default: goto eagain } } // Fill the struct from the yaml err = yaml.Unmarshal([]byte(yamlstr), &mirror) if err != nil { switch reopen(err) { case true: goto reopen case false: return nil } } mirror.Comment = comment ctx, cancel = context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() m, err := rpc.MirrorToRPC(mirror) if err != nil { log.Fatal("edit error:", err) } reply, err := client.UpdateMirror(ctx, m) if err != nil { if err.Error() == rpc.ErrNameAlreadyTaken.Error() { switch reopen(errors.New("Name already taken")) { case true: goto reopen case false: return nil } } log.Fatal("edit error:", err) } if len(reply.Diff) > 0 { fmt.Println(reply.Diff) } fmt.Printf("Mirror '%s' edited successfully\n", mirror.Name) return nil } func (c *cli) CmdGeoupdate(args ...string) error { cmd := SubCmd("geoupdate", "[IDENTIFIER]", "Update geolocation of a mirror") force := cmd.Bool("f", false, "Never prompt for confirmation") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } id, name := c.matchMirror(cmd.Arg(0)) // Get mirror with geolocation updated client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() reply, err := client.GeoUpdateMirror(ctx, &rpc.MirrorIDRequest{ ID: int32(id), }) if err != nil { log.Fatal("edit error:", err) } // Print warnings if any for i := 0; i < len(reply.Warnings); i++ { fmt.Println(reply.Warnings[i]) if i == len(reply.Warnings)-1 { fmt.Println("") } } // Print diff if any if len(reply.Diff) > 0 { fmt.Println(reply.Diff) } else { fmt.Println("Geolocation is up to date, there is nothing to change.") return nil } // Ask for confirmation if *force == false { fmt.Printf("Update mirror %s? [y/N]", name) reader := bufio.NewReader(os.Stdin) s, _ := reader.ReadString('\n') switch s[0] { case 'y', 'Y': break default: return nil } } // Update the mirror ctx, cancel = context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() reply2, err := client.UpdateMirror(ctx, reply.Mirror) if err != nil { log.Fatal("edit error:", err) } // The diff shouldn't have changed, but let's check if reply2.Diff != reply.Diff { fmt.Println("Unexpected diff, see below:") fmt.Println(reply2.Diff) } fmt.Printf("Mirror '%s' updated successfully\n", name) return nil } func (c *cli) CmdShow(args ...string) error { cmd := SubCmd("show", "[IDENTIFIER]", "Print a mirror configuration") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } id, _ := c.matchMirror(cmd.Arg(0)) client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() rpcm, err := client.MirrorInfo(ctx, &rpc.MirrorIDRequest{ ID: int32(id), }) if err != nil { log.Fatal("edit error:", err) } mirror, err := rpc.MirrorFromRPC(rpcm) if err != nil { log.Fatal("edit error:", err) } // Generate a yaml configuration string from the struct out, err := yaml.Marshal(mirror) if err != nil { log.Fatal("show error:", err) } fmt.Printf("%s\nComment:\n%s\n", out, mirror.Comment) return nil } func (c *cli) CmdExport(args ...string) error { cmd := SubCmd("export", "[format]", "Export the mirror database.\n\nAvailable formats: mirmon") rsync := cmd.Bool("rsync", true, "Export rsync URLs") http := cmd.Bool("http", true, "Export http URLs") ftp := cmd.Bool("ftp", true, "Export ftp URLs") disabled := cmd.Bool("disabled", true, "Export disabled mirrors") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } if cmd.Arg(0) != "mirmon" { fmt.Fprintf(os.Stderr, "Unsupported format\n") cmd.Usage() return nil } client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() list, err := client.List(ctx, &empty.Empty{}) if err != nil { log.Fatal("export error:", err) } w := new(tabwriter.Writer) w.Init(os.Stdout, 0, 8, 1, '\t', 0) for _, m := range list.Mirrors { if *disabled == false { if m.Enabled == false { continue } } ccodes := strings.Fields(m.CountryCodes) urls := make([]string, 0, 3) if *rsync == true && m.RsyncURL != "" { urls = append(urls, m.RsyncURL) } if *http == true && m.HttpURL != "" { if utils.HasAnyPrefix(m.HttpURL, "http://", "https://") { urls = append(urls, m.HttpURL) } else { urls = append(urls, "http://" + m.HttpURL) urls = append(urls, "https://" + m.HttpURL) } } if *ftp == true && m.FtpURL != "" { urls = append(urls, m.FtpURL) } for _, u := range urls { fmt.Fprintf(w, "%s\t%s\t%s\n", ccodes[0], u, m.AdminEmail) } } w.Flush() return nil } func (c *cli) CmdEnable(args ...string) error { cmd := SubCmd("enable", "[IDENTIFIER]", "Enable a mirror") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } c.changeStatus(cmd.Arg(0), true) return nil } func (c *cli) CmdDisable(args ...string) error { cmd := SubCmd("disable", "[IDENTIFIER]", "Disable a mirror") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } c.changeStatus(cmd.Arg(0), false) return nil } func (c *cli) changeStatus(pattern string, enabled bool) { id, name := c.matchMirror(pattern) client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() _, err := client.ChangeStatus(ctx, &rpc.ChangeStatusRequest{ ID: int32(id), Enabled: enabled, }) if err != nil { if enabled { log.Fatalf("Couldn't enable mirror '%s': %s\n", name, err) } else { log.Fatalf("Couldn't disable mirror '%s': %s\n", name, err) } } if enabled { fmt.Printf("Mirror '%s' enabled successfully\n", name) } else { fmt.Printf("Mirror '%s' disabled successfully\n", name) } return } func (c *cli) CmdStats(args ...string) error { cmd := SubCmd("stats", "[OPTIONS] [mirror|file] [IDENTIFIER|PATTERN]", "Show download stats for a particular mirror or a file pattern") dateStart := cmd.String("start-date", "", "Starting date (format YYYY-MM-DD)") dateEnd := cmd.String("end-date", "", "Ending date (format YYYY-MM-DD)") human := cmd.Bool("h", true, "Human readable version") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 2 || (cmd.Arg(0) != "mirror" && cmd.Arg(0) != "file") { cmd.Usage() return nil } start, err := time.Parse("2006-1-2", *dateStart) if err != nil { start = time.Now() } startproto, _ := ptypes.TimestampProto(start) end, err := time.Parse("2006-1-2", *dateEnd) if err != nil { end = time.Now() } endproto, _ := ptypes.TimestampProto(end) client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() if cmd.Arg(0) == "file" { // File stats reply, err := client.StatsFile(ctx, &rpc.StatsFileRequest{ Pattern: cmd.Arg(1), DateStart: startproto, DateEnd: endproto, }) if err != nil { log.Fatal("file stats error:", err) } // Format the results w := new(tabwriter.Writer) w.Init(os.Stdout, 0, 8, 1, '\t', 0) // Sort keys and count requests var keys []string var requests int64 for k, req := range reply.Files { requests += req keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { fmt.Fprintf(w, "%s:\t%d\n", k, reply.Files[k]) } if len(keys) > 0 { // Add a line separator fmt.Fprintf(w, "\t\n") } fmt.Fprintf(w, "Total download requests:\t%d\n", requests) w.Flush() } else if cmd.Arg(0) == "mirror" { // Mirror stats id, name := c.matchMirror(cmd.Arg(1)) reply, err := client.StatsMirror(ctx, &rpc.StatsMirrorRequest{ ID: int32(id), DateStart: startproto, DateEnd: endproto, }) if err != nil { log.Fatal("mirror stats error:", err) } // Format the results w := new(tabwriter.Writer) w.Init(os.Stdout, 0, 8, 1, '\t', 0) fmt.Fprintf(w, "Identifier:\t%s\n", name) if !reply.Mirror.Enabled { fmt.Fprintf(w, "Status:\tdisabled\n") } else { fmt.Fprintf(w, "Status:\t%s\n", StatusString(reply.Mirror)) } fmt.Fprintf(w, "Download requests:\t%d\n", reply.Requests) fmt.Fprint(w, "Bytes transferred:\t") if *human { fmt.Fprintln(w, utils.ReadableSize(reply.Bytes)) } else { fmt.Fprintln(w, reply.Bytes) } w.Flush() } return nil } func (c *cli) CmdLogs(args ...string) error { cmd := SubCmd("logs", "[IDENTIFIER]", "Print logs of a mirror") maxResults := cmd.Uint("l", 500, "Maximum number of logs to return") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 1 { cmd.Usage() return nil } id, name := c.matchMirror(cmd.Arg(0)) client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() resp, err := client.GetMirrorLogs(ctx, &rpc.GetMirrorLogsRequest{ ID: int32(id), MaxResults: int32(*maxResults), }) if err != nil { log.Fatal("logs error:", err) } if len(resp.Line) == 0 { fmt.Printf("No logs for %s\n", name) return nil } fmt.Printf("Printing logs for %s:\n", name) for _, l := range resp.Line { fmt.Println(l) } return nil } func (c *cli) CmdReload(args ...string) error { cmd := SubCmd("reload", "", "Reload configuration") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 0 { cmd.Usage() return nil } client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() _, err := client.Reload(ctx, &empty.Empty{}) if err != nil { log.Fatal("reload error:", err) } return nil } func (c *cli) CmdUpgrade(args ...string) error { cmd := SubCmd("upgrade", "", "Seamless binary upgrade") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 0 { cmd.Usage() return nil } client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() _, err := client.Upgrade(ctx, &empty.Empty{}) if err != nil { log.Fatal("upgrade error:", err) } return nil } func (c *cli) CmdVersion(args ...string) error { cmd := SubCmd("version", "", "Print version information") if err := cmd.Parse(args); err != nil { return nil } if cmd.NArg() != 0 { cmd.Usage() return nil } fmt.Printf("Client:\n") core.PrintVersion(core.GetVersionInfo()) fmt.Println() client := c.GetRPC() ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout) defer cancel() reply, err := client.GetVersion(ctx, &empty.Empty{}) if err != nil { s := status.Convert(err) return fmt.Errorf("version error: %w", s.Err()) } if reply.Version != "" { fmt.Printf("Server:\n") core.PrintVersion(core.VersionInfo{ Version: reply.Version, Build: reply.Build, GoVersion: reply.GoVersion, OS: reply.OS, Arch: reply.Arch, GoMaxProcs: int(reply.GoMaxProcs), }) } return nil } videolabs-mirrorbits-441567e/cli/rpc.go000066400000000000000000000026311523530551300200230ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package cli import ( "context" "fmt" "os" "strconv" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/rpc" "github.com/golang/protobuf/ptypes/empty" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) func (c *cli) GetRPC() rpc.CLIClient { c.Lock() defer c.Unlock() if c.rpcconn == nil { conn, err := grpc.Dial(core.RPCHost+":"+strconv.FormatUint(uint64(core.RPCPort), 10), grpc.WithInsecure(), grpc.WithBlock(), grpc.FailOnNonTempDialError(true), grpc.WithPerRPCCredentials(c.creds)) if err != nil { fmt.Fprintf(os.Stderr, "rpc: %s\n", err) os.Exit(1) } c.rpcconn = conn client := rpc.NewCLIClient(c.rpcconn) _, err = client.Ping(context.Background(), &empty.Empty{}) s := status.Convert(err) if s.Code() == codes.Unauthenticated { if len(c.creds.Password) == 0 { fmt.Fprintf(os.Stderr, "Please set the server password with the -P option.\n") } else { fmt.Fprintf(os.Stderr, "Password refused\n") } os.Exit(1) } } return rpc.NewCLIClient(c.rpcconn) } type loginCreds struct { Password string } func (c *loginCreds) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { return map[string]string{ "password": c.Password, }, nil } func (c *loginCreds) RequireTransportSecurity() bool { return false } videolabs-mirrorbits-441567e/config/000077500000000000000000000000001523530551300174045ustar00rootroot00000000000000videolabs-mirrorbits-441567e/config/config.go000066400000000000000000000162661523530551300212130ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package config import ( "fmt" "os" "path/filepath" "sync" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/utils" "github.com/op/go-logging" "gopkg.in/yaml.v3" ) var ( // TEMPLATES_PATH is set at compile time TEMPLATES_PATH = "" ) var ( log = logging.MustGetLogger("main") config *Configuration configMutex sync.RWMutex subscribers []chan bool subscribersLock sync.RWMutex ) func defaultConfig() Configuration { return Configuration{ Repository: "", Templates: TEMPLATES_PATH, LocalJSPath: "", OutputMode: "auto", ListenAddress: ":8080", Gzip: false, AllowHTTPToHTTPSRedirects: true, SameDownloadInterval: 600, RedisAddress: "127.0.0.1:6379", RedisPassword: "", RedisDB: 0, LogDir: "", TraceFileLocation: "", GeoipDatabasePath: "/usr/share/GeoIP/", GeographicalSort: true, ConcurrentSync: 5, ScanInterval: 30, CheckInterval: 1, RepositoryScanInterval: 5, MaxLinkHeaders: 10, FixTimezoneOffsets: false, Hashes: hashing{ SHA1: false, SHA256: true, MD5: false, }, DisallowRedirects: false, WeightDistributionRange: 1.5, DisableOnMissingFile: false, RPCListenAddress: "localhost:3390", RPCPassword: "", } } // Configuration contains all the option available in the yaml file type Configuration struct { Repository string `yaml:"Repository"` Templates string `yaml:"Templates"` LocalJSPath string `yaml:"LocalJSPath"` OutputMode string `yaml:"OutputMode"` ListenAddress string `yaml:"ListenAddress"` Gzip bool `yaml:"Gzip"` AllowHTTPToHTTPSRedirects bool `yaml:"AllowHTTPToHTTPSRedirects"` SameDownloadInterval int `yaml:"SameDownloadInterval"` RedisAddress string `yaml:"RedisAddress"` RedisPassword string `yaml:"RedisPassword"` RedisDB int `yaml:"RedisDB"` LogDir string `yaml:"LogDir"` TraceFileLocation string `yaml:"TraceFileLocation"` GeoipDatabasePath string `yaml:"GeoipDatabasePath"` GeographicalSort bool `yaml:"GeographicalSort"` ConcurrentSync int `yaml:"ConcurrentSync"` ScanInterval int `yaml:"ScanInterval"` CheckInterval int `yaml:"CheckInterval"` RepositoryScanInterval int `yaml:"RepositoryScanInterval"` MaxLinkHeaders int `yaml:"MaxLinkHeaders"` FixTimezoneOffsets bool `yaml:"FixTimezoneOffsets"` Hashes hashing `yaml:"Hashes"` DisallowRedirects bool `yaml:"DisallowRedirects"` WeightDistributionRange float32 `yaml:"WeightDistributionRange"` DisableOnMissingFile bool `yaml:"DisableOnMissingFile"` AllowOutdatedFiles []OutdatedFilesConfig `yaml:"AllowOutdatedFiles"` Fallbacks []Fallback `yaml:"Fallbacks"` RedisSentinelMasterName string `yaml:"RedisSentinelMasterName"` RedisSentinels []sentinels `yaml:"RedisSentinels"` RPCListenAddress string `yaml:"RPCListenAddress"` RPCPassword string `yaml:"RPCPassword"` } type Fallback struct { URL string `yaml:"URL"` CountryCode string `yaml:"CountryCode"` ContinentCode string `yaml:"ContinentCode"` } type sentinels struct { Host string `yaml:"Host"` } type hashing struct { SHA1 bool `yaml:"SHA1"` SHA256 bool `yaml:"SHA256"` MD5 bool `yaml:"MD5"` } type OutdatedFilesConfig struct { Prefix string `yaml:"Prefix"` Minutes int `yaml:"Minutes"` } // LoadConfig loads the configuration file if it has not yet been loaded func LoadConfig() { if config != nil { return } err := ReloadConfig() if err != nil { log.Fatal(err) } } // ReloadConfig reloads the configuration file and update it globally func ReloadConfig() error { if core.ConfigFile == "" { if fileExists("/etc/mirrorbits.conf") { core.ConfigFile = "/etc/mirrorbits.conf" } } content, err := os.ReadFile(core.ConfigFile) if err != nil { fmt.Println("Configuration could not be found.\n\tUse -config ") os.Exit(1) } if os.Getenv("DEBUG") != "" { fmt.Println("Reading configuration from", core.ConfigFile) } c := defaultConfig() // Overload the default configuration with the user's one err = yaml.Unmarshal(content, &c) if err != nil { return fmt.Errorf("%s in %s", err, core.ConfigFile) } // Sanitize if c.WeightDistributionRange <= 0 { return fmt.Errorf("WeightDistributionRange must be > 0") } if !utils.IsInSlice(c.OutputMode, []string{"auto", "json", "redirect"}) { return fmt.Errorf("Config: outputMode can only be set to 'auto', 'json' or 'redirect'") } if c.Repository == "" { return fmt.Errorf("Path to local repository not configured (see mirrorbits.conf)") } c.Repository, err = filepath.Abs(c.Repository) if err != nil { return fmt.Errorf("Invalid local repository path: %s", err) } if c.RepositoryScanInterval < 0 { c.RepositoryScanInterval = 0 } for i := range c.Fallbacks { c.Fallbacks[i].URL = utils.NormalizeURL(c.Fallbacks[i].URL) } for _, rule := range c.AllowOutdatedFiles { if len(rule.Prefix) > 0 && rule.Prefix[0] != '/' { return fmt.Errorf("AllowOutdatedFiles.Prefix must start with '/'") } if rule.Minutes < 0 { return fmt.Errorf("AllowOutdatedFiles.Minutes must be >= 0") } } if config != nil && (c.RedisAddress != config.RedisAddress || c.RedisPassword != config.RedisPassword || !testSentinelsEq(c.RedisSentinels, config.RedisSentinels)) { // TODO reload redis connections // Currently established connections will be updated only in case of disconnection } // Lock the pointer during the swap configMutex.Lock() config = &c configMutex.Unlock() // Notify all subscribers that the configuration has been reloaded notifySubscribers() return nil } // GetConfig returns a pointer to a configuration object // FIXME reading from the pointer could cause a race! func GetConfig() *Configuration { configMutex.RLock() defer configMutex.RUnlock() if config == nil { panic("Configuration not loaded") } return config } // SetConfiguration is only used for testing purpose func SetConfiguration(c *Configuration) { config = c } // SubscribeConfig allows subscribers to get notified when // the configuration is updated. func SubscribeConfig(subscriber chan bool) { subscribersLock.Lock() defer subscribersLock.Unlock() subscribers = append(subscribers, subscriber) } func notifySubscribers() { subscribersLock.RLock() defer subscribersLock.RUnlock() for _, subscriber := range subscribers { select { case subscriber <- true: default: // Don't block if the subscriber is unavailable // and discard the message. } } } func fileExists(filename string) bool { _, err := os.Stat(filename) return err == nil } func testSentinelsEq(a, b []sentinels) bool { if len(a) != len(b) { return false } for i := range a { if a[i].Host != b[i].Host { return false } } return true } videolabs-mirrorbits-441567e/contrib/000077500000000000000000000000001523530551300175775ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/completions/000077500000000000000000000000001523530551300221335ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/completions/mirrorbits.bash000066400000000000000000000140731523530551300251730ustar00rootroot00000000000000# mirrorbits(1) completion -*- shell-script -*- # ex: ts=4 sw=4 et filetype=sh # # Copyright (c) 2024 Arnaud Rebillout # Distributed under the same license as mirrorbits. _in_array() { local i for i in "${@:2}"; do [[ $1 = "$i" ]] && return done } _mirrorbits_list() { local port=$1 mirrorbits -p $port list -state=false | tail -n +2 || : } _mirrorbits() { # Variables assigned by _init_completion: # cur Current argument. # prev Previous argument. # words Argument array. # cword Argument array size. local cur prev words cword _init_completion || return # Mirrorbits cli options, must come before the command local CLI_OPTIONS=( "-P" "-a" "-cpuprofile" "-debug" "-h" "-p") # Mirrorbits commands local COMMANDS=( "add" "daemon" "disable" "edit" "enable" "export" "geoupdate" "list" "logs" "refresh" "reload" "remove" "scan" "show" "stats" "upgrade" "version") # Check if a command was already given local command i for (( i = 1; i < cword; i++ )); do if _in_array "${words[i]}" "${COMMANDS[@]}"; then command=${words[i]} break fi done # Check if a port was already given, tricky as we must # support -p X, -p=X, --p X and --p=X local port for (( i = 1; i < cword; i++ )); do if [[ ! ${words[i]} =~ ^[0-9]+$ ]]; then continue fi if [[ ${words[i - 1]} =~ ^-?-p$ ]]; then port=${words[i]} break fi if [[ ${words[i - 1]} == = ]] && (( i > 1 )) && [[ ${words[i - 2]} =~ ^-?-p$ ]]; then port=${words[i]} break fi done # No port? Default port is 3390 if [[ -z $port ]]; then port=3390 fi # Completion per command if [[ -n $command ]]; then case $command in daemon) COMPREPLY=( $( compgen -W '-help -config -cpuprofile -debug -log -monitor -p' -- "$cur" ) ) ;; add) COMPREPLY=( $( compgen -W '-help -admin-email -admin-name -as-only -comment -continent-only -country-only -custom-data -ftp -http -rsync -score -sponsor-logo -sponsor-name -sponsor-url ' -- "$cur" ) ) ;; disable|edit|enable|show) case $cur in -*) COMPREPLY=( $( compgen -W '-help' -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "$( _mirrorbits_list $port )" -- "$cur" ) ) ;; esac ;; export) COMPREPLY=( $( compgen -W '-help -disabled -ftp -http -rsync' -- "$cur" ) ) ;; geoupdate) case $cur in -*) COMPREPLY=( $( compgen -W '-help -f' -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "$( _mirrorbits_list $port )" -- "$cur" ) ) ;; esac ;; list) COMPREPLY=( $( compgen -W '-help -disabled -down -enabled -ftp -http -location -rsync -score -state ' -- "$cur" ) ) ;; logs) case $cur in -*) COMPREPLY=( $( compgen -W '-help -l' -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "$( _mirrorbits_list $port )" -- "$cur" ) ) ;; esac ;; refresh) COMPREPLY=( $( compgen -W '-help -rehash' -- "$cur" ) ) ;; reload|upgrade|version) COMPREPLY=( $( compgen -W '-help' -- "$cur" ) ) ;; remove) case $cur in -*) COMPREPLY=( $( compgen -W '-help -f' -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "$( _mirrorbits_list $port )" -- "$cur" ) ) ;; esac ;; scan) case $cur in -*) COMPREPLY=( $( compgen -W '-help -all -enable -ftp -rsync -timeout' -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "$( _mirrorbits_list $port )" -- "$cur" ) ) ;; esac ;; stats) case $cur in -*) COMPREPLY=( $( compgen -W '-help -end-date -h -start-date ' -- "$cur" ) ) ;; *) if _in_array mirror "${words[@]:2}"; then COMPREPLY=( $( compgen -W "$( _mirrorbits_list $port )" -- "$cur" ) ) elif _in_array file "${words[@]:2}"; then COMPREPLY=() else COMPREPLY=( $( compgen -W 'file mirror' -- "$cur" ) ) fi ;; esac ;; *) COMPREPLY=() ;; esac else # no command yet case "$cur" in -*) COMPREPLY=( $( compgen -W '${CLI_OPTIONS[@]}' -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W '${COMMANDS[@]}' -- "$cur" ) ) ;; esac fi return 0 } && complete -F _mirrorbits mirrorbits videolabs-mirrorbits-441567e/contrib/docker/000077500000000000000000000000001523530551300210465ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/docker/mirrorbits.conf000066400000000000000000000001301523530551300241030ustar00rootroot00000000000000# vim: set ft=yaml: Repository: /srv/repo ListenAddress: :8080 RedisAddress: redis:6379videolabs-mirrorbits-441567e/contrib/geoip/000077500000000000000000000000001523530551300207025ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/geoip/geoip-lite-update000066400000000000000000000046441523530551300241530ustar00rootroot00000000000000#!/bin/bash # geoip-lite-update -- update geoip lite database(s). # (c) 2008,2009,2010,2011,2012,2013,2014 poeml@cmdline.net # Distribute under GPLv2 if it proves worthy. # With added support for: # - GeoLiteCityv6 # - GeoIPASNum # - GeoIPASNumv6 # by Ludovic Fauvet for i in curl wget ftp; do if which $i &>/dev/null; then prg=$i break fi done if [ -z "$prg" ]; then echo cannot find a tool to download, like curl or wget >&2 exit 1 fi case $prg in curl) prg="curl -s -O" ;; wget) prg="wget --quiet" ;; esac set -e # GeoIP data used to be in /usr/share/GeoIP in the openSUSE package, and was moved later. # try the old location first - if it's present, it means that the user had his own # updated database there cd /usr/share/GeoIP/ 2>/dev/null || cd /var/lib/GeoIP rm -f GeoIP.dat.gz $prg https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz gunzip -c GeoIP.dat.gz > GeoIP.dat.updated.new mv GeoIP.dat.updated.new GeoIP.dat.updated rm -f GeoLiteCity.dat.gz $prg https://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz gunzip -c GeoLiteCity.dat.gz > GeoLiteCity.dat.updated.new mv GeoLiteCity.dat.updated.new GeoLiteCity.dat.updated rm -f GeoLiteCityv6.dat.gz $prg https://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz gunzip -c GeoLiteCityv6.dat.gz > GeoLiteCityv6.dat.updated.new mv GeoLiteCityv6.dat.updated.new GeoLiteCityv6.dat.updated rm -f GeoIPv6.dat.gz $prg https://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz gunzip -c GeoIPv6.dat.gz > GeoIPv6.dat.updated.new mv GeoIPv6.dat.updated.new GeoIPv6.dat.updated rm -f GeoIPASNum.dat.gz $prg https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz gunzip -c GeoIPASNum.dat.gz > GeoIPASNum.dat.updated.new mv GeoIPASNum.dat.updated.new GeoIPASNum.dat.updated rm -f GeoIPASNumv6.dat.gz $prg https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNumv6.dat.gz gunzip -c GeoIPASNumv6.dat.gz > GeoIPASNumv6.dat.updated.new mv GeoIPASNumv6.dat.updated.new GeoIPASNumv6.dat.updated set +e if [ "$1" = "--no-reload" ]; then exit 0 fi if [ -x /etc/init.d/apache2 ]; then /etc/init.d/apache2 reload elif [ -x /etc/init.d/httpd ]; then /etc/init.d/httpd reload elif [ -x /usr/bin/systemctl ]; then /usr/bin/systemctl reload httpd >/dev/null 2>&1 || : elif [ -x /bin/systemctl ]; then /bin/systemctl reload httpd >/dev/null 2>&1 || : fi videolabs-mirrorbits-441567e/contrib/init/000077500000000000000000000000001523530551300205425ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/init/systemd/000077500000000000000000000000001523530551300222325ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/init/systemd/.gitignore000066400000000000000000000000231523530551300242150ustar00rootroot00000000000000mirrorbits.service videolabs-mirrorbits-441567e/contrib/init/systemd/mirrorbits.service.in000066400000000000000000000007361523530551300264230ustar00rootroot00000000000000[Unit] Description=Mirrorbits redirector Documentation=https://github.com/etix/mirrorbits After=network.target [Service] Type=notify DynamicUser=yes LogsDirectory=mirrorbits RuntimeDirectory=mirrorbits PIDFile=/run/mirrorbits/mirrorbits.pid ExecStart=##PREFIX##/bin/mirrorbits daemon -p /run/mirrorbits/mirrorbits.pid ExecReload=/bin/kill -HUP $MAINPID ExecStop=-/bin/kill -QUIT $MAINPID TimeoutStopSec=5 KillMode=mixed Restart=on-failure [Install] WantedBy=multi-user.target videolabs-mirrorbits-441567e/contrib/init/sysvinit-debian/000077500000000000000000000000001523530551300236525ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/init/sysvinit-debian/mirrorbits000066400000000000000000000111661523530551300257760ustar00rootroot00000000000000#! /bin/sh ### BEGIN INIT INFO # Provides: mirrorbits # Required-Start: redis-server $remote_fs $syslog # Required-Stop: redis-server $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Mirrorbits initscript # Description: Simple HTTP mirror redirector written in Go. ### END INIT INFO # Author: Ludovic Fauvet # Do NOT "set -e" # PATH should only include /usr/* if it runs after the mountnfs.sh script PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC="Mirrorbits is a geographic load-balancer for mirrors" NAME=mirrorbits CONFFILE=/etc/mirrorbits.conf RUNLOG=/var/log/mirrorbits/mirrorbits.log PIDFILE=/var/run/$NAME.pid DAEMON=/usr/bin/$NAME DAEMON_ARGS="daemon -p $PIDFILE -log $RUNLOG" SCRIPTNAME=/etc/init.d/$NAME # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 # Read configuration variable file if it is present [ -r /etc/default/$NAME ] && . /etc/default/$NAME # Load the VERBOSE setting and other rcS variables . /lib/init/vars.sh # Define LSB log_* functions. # Depend on lsb-base (>= 3.2-14) to ensure that this file is present # and status_of_proc is working. . /lib/lsb/init-functions # # Function that starts the daemon/service # do_start() { # Return # 0 if daemon has been started # 1 if daemon was already running # 2 if daemon could not be started start-stop-daemon --start --quiet --pidfile $PIDFILE -b --exec $DAEMON --test > /dev/null \ || return 1 start-stop-daemon --start --quiet --pidfile $PIDFILE -b --exec $DAEMON -- \ $DAEMON_ARGS \ || return 2 # Add code here, if necessary, that waits for the process to be ready # to handle requests from services started subsequently which depend # on this one. As a last resort, sleep for some time. } # # Function that stops the daemon/service # do_stop() { # Return # 0 if daemon has been stopped # 1 if daemon was already stopped # 2 if daemon could not be stopped # other if a failure occurred start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME RETVAL="$?" [ "$RETVAL" = 2 ] && return 2 # Wait for children to finish too if this is a daemon that forks # and if the daemon is only ever run from this initscript. # If the above conditions are not satisfied then add some other code # that waits for the process to drop all resources that could be # needed by services started subsequently. A last resort is to # sleep for some time. start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $DAEMON [ "$?" = 2 ] && return 2 # Many daemons don't delete their pidfiles when they exit. rm -f $PIDFILE return "$RETVAL" } # # Function that sends a SIGHUP to the daemon/service # do_reload() { # # If the daemon can reload its configuration without # restarting (for example, when it is sent a SIGHUP), # then implement that here. # start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME return 0 } do_configtest() { return 0 # not supported yet if [ "$#" -ne 0 ]; then case "$1" in -q) FLAG=$1 ;; *) ;; esac shift fi $DAEMON -t $FLAG -c $CONFFILE RETVAL="$?" return $RETVAL } do_upgrade() { do_configtest -q || return 6 PID=$(cat $PIDFILE) if [ ! -x /proc/${PID} ]; then echo "$NAME is not running" exit 0 fi start-stop-daemon --stop --signal USR2 --quiet --pidfile $PIDFILE --name $NAME RETVAL="$?" echo "Upgrading..." return $RETVAL } case "$1" in start) [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" do_start case "$?" in 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; esac ;; stop) [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" do_stop case "$?" in 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; esac ;; status) status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? ;; configtest) do_configtest ;; upgrade) do_upgrade ;; reload|force-reload) log_daemon_msg "Reloading $DESC" "$NAME" do_reload log_end_msg $? ;; restart|force-reload) log_daemon_msg "Restarting $DESC" "$NAME" do_configtest -q || exit $RETVAL do_stop case "$?" in 0|1) do_start case "$?" in 0) log_end_msg 0 ;; 1) log_end_msg 1 ;; # Old process is still running *) log_end_msg 1 ;; # Failed to start esac ;; *) # Failed to stop log_end_msg 1 ;; esac ;; *) echo "Usage: $SCRIPTNAME {start|stop|status|restart|reload|force-reload|upgrade|configtest}" >&2 exit 3 ;; esac : videolabs-mirrorbits-441567e/contrib/localjs/000077500000000000000000000000001523530551300212265ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/localjs/fetchfiles.sh000077500000000000000000000104171523530551300237040ustar00rootroot00000000000000#!/bin/bash # List of scripts to fetch and store locally whattofetch=( "https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/excanvas.js" "https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/excanvas.min.js" "https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.js" "https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js" "https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.js" "https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.min.js" "https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.pie.js" "https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.pie.min.js" "https://cdnjs.cloudflare.com/ajax/libs/flot.tooltip/0.9.0/jquery.flot.tooltip.js" "https://cdnjs.cloudflare.com/ajax/libs/flot.tooltip/0.9.0/jquery.flot.tooltip.min.js" "https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.3.4/leaflet.css" "https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.3.4/leaflet.js" "https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.3.4/images/marker-icon.png" "https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.3.4/images/marker-shadow.png" "https://cdnjs.cloudflare.com/ajax/libs/leaflet.markercluster/1.4.1/MarkerCluster.css" "https://cdnjs.cloudflare.com/ajax/libs/leaflet.markercluster/1.4.1/leaflet.markercluster.js" ) showhelp() { echo "Syntax: $0 directory" echo "where directory is the directory in which you want to store the downloaded files." echo "" echo "This will download the Javascript- and Font-files used by the default" echo "templates in mirrorbits. You can then self-host that directory tree on" echo "your webserver instead of using external CDNs." echo "You then need to set the LocalJSPath option in your mirrorbits config to" echo "point at the web-accessible path to that directory." } getlocalfilename () { local sfn="$1" lfn=${sfn#https://cdnjs.cloudflare.com/ajax/libs} } downloadfile () { curl=`which curl` if [ ${#curl} -ge 4 ] ; then $curl -L --output "$2" "$1" return fi wget=`which wget` if [ ${#wget} -ge 4 ] ; then $wget --output-document="$2" "$1" return fi echo "ERROR: Neither curl nor wget were found in path. Please install either curl or wget." exit 1 } if [ "$#" -ne 1 ] ; then showhelp exit 1 fi if [ "$1" == "--help" -o "$1" == "-h" ] ; then showhelp exit 0 fi localdir="$1" if [ ! -d "$localdir" ] ; then echo "Target directory ${localdir} does not exist or is not a directory." showhelp exit 1 fi unzip=`which unzip` if [ ${#unzip} -lt 5 ] ; then echo "ERROR: unzip was not found in path. Please install unzip." exit 1 fi for sf in ${whattofetch[@]}; do lfn="/void/void/void/void/" getlocalfilename "$sf" # lfn is now filled. tf="${localdir}${lfn}" if [ -e "$tf" ] ; then echo "No need to fetch $sf to $tf, it already exists." else tdn=`dirname "${tf}"` if [ ! -e "$tdn" ] ; then mkdir -p "$tdn" fi downloadfile "$sf" "$tf" fi done # The font stuff is a bit more messy. # For the Lato font, we rely on a service that's packing the mess into # a ZIP file for us. if [ ! -e "${localdir}/fonts" ] ; then mkdir -p "${localdir}/fonts" fi downloadfile "https://google-webfonts-helper.herokuapp.com/api/fonts/lato?download=zip&subsets=latin&variants=900,regular" "${localdir}/fonts/lato-font-900and400.zip" unzip "${localdir}/fonts/lato-font-900and400.zip" -d "${localdir}/fonts/" rm -f "${localdir}/fonts/lato-font-900and400.zip" # For fontawesome, we download and unpack the whole ZIP as well. # Downloading individual files will not work here, as depending on the # browser random other files will be included. if [ ! -e "${localdir}/font-awesome" ] ; then mkdir -p "${localdir}/font-awesome" fi if [ -e "${localdir}/font-awesome/4.7.0" ] ; then # Only download and extract if that directory isn't there yet, as we # have to use a 'mv' command here that would fail if the target already # existed. echo "Note: Skipping font-awesome download and extraction, it seems to be in place already." echo "To force redownload and extraction, remove '${localdir}/font-awesome/4.7.0'" else downloadfile "https://fontawesome.com/v4.7.0/assets/font-awesome-4.7.0.zip" "${localdir}/font-awesome-4.7.0.zip" unzip "${localdir}/font-awesome-4.7.0.zip" -d "${localdir}/font-awesome" rm -f "${localdir}/font-awesome-4.7.0.zip" mv "${localdir}/font-awesome/font-awesome-4.7.0" "${localdir}/font-awesome/4.7.0" fi videolabs-mirrorbits-441567e/contrib/logrotate/000077500000000000000000000000001523530551300215775ustar00rootroot00000000000000videolabs-mirrorbits-441567e/contrib/logrotate/mirrorbits000066400000000000000000000004521523530551300237170ustar00rootroot00000000000000/var/log/mirrorbits/*.log { daily rotate 14 missingok notifempty compress delaycompress postrotate if [ -e /run/systemd/system ]; then if systemctl -q is-active mirrorbits; then systemctl kill -s USR1 mirrorbits fi else killall -s USR1 mirrorbits || true fi endscript } videolabs-mirrorbits-441567e/contrib/mirrorbits-del-stats000077500000000000000000000070051523530551300236210ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2023 Arnaud Rebillout # Distributed under the same license as mirrorbits. set -euo pipefail USAGE="Usage: $(basename $0) [-f] :... [--] [REDIS_CLI_ARGS...] Delete old Mirrorbits stats. This script deletes old Mirrorbits stats, by using the Redis client, and directly deleting the keys in the Redis database. It asks for confirmation before acting, unless you provide -f in argument. Mirrorbits maintains daily, monthly and yearly stats. The pairs : define how many stats to keep for each period. For example, the command: $(basename $0) daily:30 monthly:12 -- -p 6380 will keep only the last 30 daily stats and the last 12 monthly stats. Older stats will be deleted. Yearly stats are left untouched, as we didn't prodive any yearly: argument. Finally, trailing arguments are handed over to the redis-cli command, hence '-p 6380' makes redis-cli use the port 6380. " ARGS= FORCE=0 while [ $# -gt 0 ]; do case $1 in --) shift; break ;; -f|--force) FORCE=1 ;; -h|--help) echo "$USAGE"; exit 0 ;; -*) break ;; *) ARGS="$ARGS $1" ;; esac shift done # Find which redis cli to use REDIS_CLI=redis-cli for x in valkey redict keydb; do if command -v $x-cli 2>&1 >/dev/null; then REDIS_CLI=$x-cli break fi done REDIS="$REDIS_CLI $@" # Get all the STATS_* keys echo "Scanning the ${REDIS_CLI/-cli/} database for STATS_* keys, this might take a while ..." if $REDIS_CLI --help 2>&1 | grep -q -- " --count "; then KEYS=$($REDIS --scan --count 1000 --pattern "STATS_*") else KEYS=$($REDIS --scan --pattern "STATS_*") fi # Keep only the daily/monthly/yearly stats FILE_KEYS=$(echo "$KEYS" | grep "^STATS_FILE_[0-9]") MIRROR_KEYS=$(echo "$KEYS" | grep "^STATS_MIRROR_[0-9]") MIRROR_BYTES_KEYS=$(echo "$KEYS" | grep "^STATS_MIRROR_BYTES_[0-9]") # Iterate over arguments NO_KEY_TO_DELETE=1 for arg in $ARGS; do period=$(echo $arg | cut -d: -f1) retention=$(echo $arg | cut -d: -f2) case $period in daily) pattern="[A-Z]_[0-9]{4}_[01][0-9]_[0-3][0-9]$" ;; monthly) pattern="[A-Z]_[0-9]{4}_[01][0-9]$" ;; yearly) pattern="[A-Z]_[0-9]{4}$" ;; *) echo "Invalid period '$period', skipping." >&2 continue esac if ! echo "$retention" | grep -qx "[0-9]\+"; then echo "Invalid retention '$retention', skipping." >&2 continue fi for v in FILE_KEYS MIRROR_KEYS MIRROR_BYTES_KEYS; do keys=$(echo "${!v}" | grep -E "$pattern" | LC_ALL=C sort -u) toremove=$(echo "$keys" | head -n -$retention) if [ -z "$toremove" ]; then continue fi NO_KEY_TO_DELETE=0 echo "The following keys will be removed:" echo $toremove | fold -s -w 80 if [ $FORCE = 0 ]; then echo read -r -p "Proceed? [Y/n] " if [ -z "$REPLY" ]; then REPLY=Y; fi if [ "${REPLY,,}" != y ]; then echo "Skipped" continue fi fi # Remove by blocks of 100 keys while [ -n "$toremove" ]; do keys=$(echo "$toremove" | head -n 100) first=$(echo "$keys" | head -n 1) last=$(echo "$keys" | tail -n 1) echo "Removing keys from $first to $last ..." $REDIS DEL $keys toremove=$(echo "$toremove" | tail -n +101) done done done if [ $NO_KEY_TO_DELETE = 1 ]; then echo "No key to delete, nothing was done." fi videolabs-mirrorbits-441567e/contrib/mirrorbits-geoupdate-all000077500000000000000000000066731523530551300244560ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2023 Arnaud Rebillout # Distributed under the same license as mirrorbits. set -euo pipefail USAGE="$(basename $0) [-f [-f]] [--] [MIRRORBITS_ARGS...] Update geolocation for all mirrors. The argument '-f' controls the exact behavior of the script: * no -f: ask for confirmation for each mirror. * one -f: if there are only latitude and longitude changes, don't ask for confirmation. Otherwise (eg. ASN, country or continent changed), either 1) ask for confirmation if running interactively, or 2) print the changes and skip this mirror - in the end the script returns the special exit code 33 to signal that some mirrors were not updated. * two -f: update all mirrors without asking for confirmation. Trailing arguments are handed over to the mirrorbits command. For example, the command: mirrorbits-geoupdate-all -f -- -p 3391 updates all the mirrors for the mirrorbits instance listening on port 3391. BUGS: This script assumes that calling 'mirrorbits geoupdate \$mirror' twice in a row returns the same result, but this is not always true. For some mirrors, the hostname returns more than one address, and each address might have a different geolocation, maybe even different a ASN. " FORCE=0 while [ $# -gt 0 ]; do case $1 in -f|--force) FORCE=$((FORCE + 1)) ;; -h|--help) echo "$USAGE"; exit 0 ;; --) shift; break ;; *) break ;; esac shift done MIRRORBITS="mirrorbits $@" # Get all the mirrors MIRRORS=$($MIRRORBITS list -state=false | grep -iv "^identifier\b" || :) MIRRORS=$(echo "$MIRRORS" | LC_ALL=C sort -u) # Helper to print like ansible, bash-fu from: # https://stackoverflow.com/a/54505990/776208 echo_updating() { COLUMNS=$(tput cols 2>/dev/null || echo 80) printf "%-${COLUMNS}s\n" "[^Updating^mirror:^$1^]^" | tr "^ " " =" } # Iterate over the mirrors EXIT_STATUS=0 MIRRORS_WITH_CHANGES=0 for mirror in $MIRRORS; do # Ask to update mirror, but bail out. We only want to see the changes. output=$(echo n | $MIRRORBITS geoupdate $mirror) # Keep only the lines that list the changes. changes=$(echo "$output" | grep "^[+-] " || :) # No change? Keep going then. if [ -z "$changes" ]; then continue fi MIRRORS_WITH_CHANGES=$((MIRRORS_WITH_CHANGES + 1)) echo echo_updating $mirror # FORCE == 0 aka. interactive. Always ask for confirmation. if [ $FORCE -eq 0 ]; then $MIRRORBITS geoupdate $mirror continue fi # FORCE == 2 aka. just do it. Never ask for confirmation. if [ $FORCE -ge 2 ]; then $MIRRORBITS geoupdate -f $mirror continue fi # FORCE == 1 aka. automatic but conservative. # Filter out latitude and longitude changes. If there are no other # changes, act without waiting for confirmation. If there are other # changes, and we're running interactively, ask for confirmation. # Otherwise, print a warning, and keep going. We'll return a special # exit code in the end. changes=$(echo "$changes" | grep -Ev "L(at|ong)itude:" || :) if [ -z "$changes" ]; then $MIRRORBITS geoupdate -f $mirror elif [ -t 0 ]; then $MIRRORBITS geoupdate $mirror else echo "NOT UPDATING! Changes need review, see below:" echo "$output" | grep -iv "y/n" || : EXIT_STATUS=33 fi done if [ $MIRRORS_WITH_CHANGES = 0 ]; then echo "No mirror to update, nothing was done." fi exit $EXIT_STATUS videolabs-mirrorbits-441567e/core/000077500000000000000000000000001523530551300170675ustar00rootroot00000000000000videolabs-mirrorbits-441567e/core/banner.go000066400000000000000000000006021523530551300206610ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package core // Banner is a piece of (ascii) art shown during startup var Banner = ` _______ __ __ __ __ | | |__|.----.----.-----.----.| |--.|__| |_.-----. | | || _| _| _ | _|| _ || | _|__ --| |__|_|__|__||__| |__| |_____|__| |_____||__|____|_____| %s` videolabs-mirrorbits-441567e/core/context.go000066400000000000000000000006771523530551300211140ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package core // ContextKey reprensents a context key associated with a value type ContextKey int const ( // ContextAllowRedirects is the key for option: AllowRedirects ContextAllowRedirects ContextKey = iota // ContextMirrorID is the key for the variable: MirrorID ContextMirrorID // ContextMirrorName is the key for the variable: MirrorName ContextMirrorName ) videolabs-mirrorbits-441567e/core/database.go000066400000000000000000000006351523530551300211660ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package core const ( // RedisMinimumVersion contains the minimum redis version required to run the application RedisMinimumVersion = "3.2.0" // DBVersion represents the current DB format version DBVersion = 2 // DBVersionKey contains the global redis key containing the DB version format DBVersionKey = "MIRRORBITS_DB_VERSION" ) videolabs-mirrorbits-441567e/core/flags.go000066400000000000000000000025031523530551300205120ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package core import ( "flag" "os" ) var ( Daemon bool Debug bool Monitor bool ConfigFile string CpuProfile string PidFile string RunLog string RPCPort uint RPCHost string RPCPassword string RPCAskPass bool NArg int ) func Parseflags() { flag.BoolVar(&Debug, "debug", false, "Debug mode") flag.StringVar(&CpuProfile, "cpuprofile", "", "write cpu profile to file") flag.UintVar(&RPCPort, "p", 3390, "Server port") flag.StringVar(&RPCHost, "h", "localhost", "Server host") flag.StringVar(&RPCPassword, "P", "", "Server password") flag.BoolVar(&RPCAskPass, "a", false, "Ask for server password") flag.Parse() NArg = flag.NArg() daemon := flag.NewFlagSet("daemon", flag.ExitOnError) daemon.BoolVar(&Debug, "debug", false, "Debug mode") daemon.StringVar(&CpuProfile, "cpuprofile", "", "write cpu profile to file") daemon.StringVar(&ConfigFile, "config", "", "Path to the config file") daemon.BoolVar(&Monitor, "monitor", true, "Enable the background mirrors monitor") daemon.StringVar(&PidFile, "p", "", "Path to pid file") daemon.StringVar(&RunLog, "log", "", "File to output logs (default: stderr)") if len(os.Args) > 1 && os.Args[1] == "daemon" { Daemon = true daemon.Parse(os.Args[2:]) } } videolabs-mirrorbits-441567e/core/scan.go000066400000000000000000000006211523530551300203410ustar00rootroot00000000000000package core import "time" // ScannerType holds the type of scanner in use type ScannerType int8 const ( // RSYNC represents an rsync scanner RSYNC ScannerType = iota // FTP represents an ftp scanner FTP ) // Precision is used to compute the precision of the mod time (millisecond, second) type Precision time.Duration func (p Precision) Duration() time.Duration { return time.Duration(p) } videolabs-mirrorbits-441567e/core/version.go000066400000000000000000000021311523530551300211000ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package core import ( "fmt" "runtime" ) var ( VERSION = "" BUILD = "" DEV = "" ) // VersionInfo is a struct containing version related informations type VersionInfo struct { Version string Build string GoVersion string OS string Arch string GoMaxProcs int } // GetVersionInfo returns the details of the current build func GetVersionInfo() VersionInfo { return VersionInfo{ Version: VERSION, Build: BUILD + DEV, GoVersion: runtime.Version(), OS: runtime.GOOS, Arch: runtime.GOARCH, GoMaxProcs: runtime.GOMAXPROCS(0), } } // PrintVersion prints the versions contained in a VersionReply func PrintVersion(info VersionInfo) { fmt.Printf(" %-17s %s\n", "Version:", info.Version) fmt.Printf(" %-17s %s\n", "Build:", info.Build) fmt.Printf(" %-17s %s\n", "GoVersion:", info.GoVersion) fmt.Printf(" %-17s %s\n", "Operating System:", info.OS) fmt.Printf(" %-17s %s\n", "Architecture:", info.Arch) fmt.Printf(" %-17s %d\n", "Gomaxprocs:", info.GoMaxProcs) } videolabs-mirrorbits-441567e/daemon/000077500000000000000000000000001523530551300174025ustar00rootroot00000000000000videolabs-mirrorbits-441567e/daemon/cluster.go000066400000000000000000000120351523530551300214130ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package daemon import ( "fmt" "math/rand" "sort" "strconv" "strings" "sync" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/utils" ) const ( clusterAnnouncePrefix = "HELLO" ) type cluster struct { redis *database.Redis nodeID string nodes []node nodeIndex int nodeTotal int nodesLock sync.RWMutex mirrorsIndex []int stop chan bool wg sync.WaitGroup running bool StartStopLock sync.Mutex announceText string } type node struct { ID string LastAnnounce int64 } type byNodeID []node func (n byNodeID) Len() int { return len(n) } func (n byNodeID) Swap(i, j int) { n[i], n[j] = n[j], n[i] } func (n byNodeID) Less(i, j int) bool { return n[i].ID < n[j].ID } // NewCluster creates a new instance of the cluster agent func NewCluster(r *database.Redis) *cluster { c := &cluster{ redis: r, nodes: make([]node, 0), stop: make(chan bool), } hostname := utils.Hostname() if len(hostname) == 0 { hostname = "unknown" } c.nodeID = fmt.Sprintf("%s-%05d", hostname, rand.Intn(32000)) c.announceText = clusterAnnouncePrefix + strconv.Itoa(GetConfig().RedisDB) return c } func (c *cluster) Start() { c.StartStopLock.Lock() defer c.StartStopLock.Unlock() if c.running == true { return } log.Debug("Cluster starting...") c.running = true c.wg.Add(1) c.stop = make(chan bool) go c.clusterLoop() } func (c *cluster) Stop() { c.StartStopLock.Lock() defer c.StartStopLock.Unlock() select { case _, _ = <-c.stop: return default: close(c.stop) c.wg.Wait() c.running = false log.Debug("Cluster stopped") } } func (c *cluster) clusterLoop() { clusterChan := make(chan string, 10) announceTicker := time.NewTicker(1 * time.Second) c.refreshNodeList(c.nodeID, c.nodeID) c.redis.Pubsub.SubscribeEvent(database.CLUSTER, clusterChan) for { select { case <-c.stop: c.wg.Done() return case <-announceTicker.C: c.announce() case data := <-clusterChan: if !strings.HasPrefix(data, c.announceText+" ") { // Garbage continue } c.refreshNodeList(data[len(c.announceText)+1:], c.nodeID) } } } func (c *cluster) announce() { r := c.redis.Get() database.Publish(r, database.CLUSTER, fmt.Sprintf("%s %s", c.announceText, c.nodeID)) r.Close() } func (c *cluster) refreshNodeList(nodeID, self string) { found := false c.nodesLock.Lock() // Expire unreachable nodes for i := 0; i < len(c.nodes); i++ { if utils.ElapsedSec(c.nodes[i].LastAnnounce, 5) && c.nodes[i].ID != nodeID && c.nodes[i].ID != self { log.Noticef("<- Node %s left the cluster", c.nodes[i].ID) c.nodes = append(c.nodes[:i], c.nodes[i+1:]...) i-- } else if c.nodes[i].ID == nodeID { found = true c.nodes[i].LastAnnounce = time.Now().UTC().Unix() } } // Join new node if !found { if nodeID != self { log.Noticef("-> Node %s joined the cluster", nodeID) } n := node{ ID: nodeID, LastAnnounce: time.Now().UTC().Unix(), } // TODO use binary search here // See https://golang.org/pkg/sort/#Search c.nodes = append(c.nodes, n) sort.Sort(byNodeID(c.nodes)) } c.nodeTotal = len(c.nodes) // TODO use binary search here // See https://golang.org/pkg/sort/#Search for i, n := range c.nodes { if n.ID == self { c.nodeIndex = i break } } c.nodesLock.Unlock() } func (c *cluster) AddMirror(mirror *mirrors.Mirror) { c.nodesLock.Lock() c.mirrorsIndex = addMirrorIDToSlice(c.mirrorsIndex, mirror.ID) c.nodesLock.Unlock() } func (c *cluster) RemoveMirror(mirror *mirrors.Mirror) { c.nodesLock.Lock() c.mirrorsIndex = removeMirrorIDFromSlice(c.mirrorsIndex, mirror.ID) c.nodesLock.Unlock() } func (c *cluster) RemoveMirrorID(id int) { c.nodesLock.Lock() c.mirrorsIndex = removeMirrorIDFromSlice(c.mirrorsIndex, id) c.nodesLock.Unlock() } func (c *cluster) IsHandled(mirrorID int) bool { c.nodesLock.RLock() defer c.nodesLock.RUnlock() index := sort.SearchInts(c.mirrorsIndex, mirrorID) mRange := int(float32(len(c.mirrorsIndex))/float32(c.nodeTotal) + 0.5) start := mRange * c.nodeIndex // Check bounding to see if this mirror must be handled by this node. // The distribution of the nodes should be balanced except for the last node // that could contain one more node. if index >= start && (index < start+mRange || c.nodeIndex == c.nodeTotal-1) { return true } return false } func removeMirrorIDFromSlice(slice []int, mirrorID int) []int { // See https://golang.org/pkg/sort/#SearchInts idx := sort.SearchInts(slice, mirrorID) if idx < len(slice) && slice[idx] == mirrorID { slice = append(slice[:idx], slice[idx+1:]...) } return slice } func addMirrorIDToSlice(slice []int, mirrorID int) []int { // See https://golang.org/pkg/sort/#SearchInts idx := sort.SearchInts(slice, mirrorID) if idx >= len(slice) || slice[idx] != mirrorID { slice = append(slice[:idx], append([]int{mirrorID}, slice[idx:]...)...) } return slice } videolabs-mirrorbits-441567e/daemon/cluster_test.go000066400000000000000000000124761523530551300224630ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package daemon import ( "fmt" "reflect" "sort" "testing" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/mirrors" . "github.com/etix/mirrorbits/testing" ) func TestMain(m *testing.M) { SetConfiguration(&Configuration{ RedisDB: 42, }) m.Run() } func TestStart(t *testing.T) { _, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCluster(conn) c.Start() defer c.Stop() if c.running != true { t.Fatalf("Expected true, got false") } } func TestStop(t *testing.T) { _, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCluster(conn) c.Start() c.Stop() if c.running != false { t.Fatalf("Expected false, got true") } } func TestClusterLoop(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCluster(conn) cmdPublish := mock.Command("PUBLISH", string(database.CLUSTER), fmt.Sprintf("%s %s", c.announceText, c.nodeID)).Expect("1") c.Start() defer c.Stop() n := time.Now() for { if time.Since(n) > 1500*time.Millisecond { t.Fatalf("Announce not made") } if mock.Stats(cmdPublish) > 0 { // Success break } time.Sleep(50 * time.Millisecond) } } func TestRefreshNodeList(t *testing.T) { _, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCluster(conn) n := node{ ID: "test-4242", LastAnnounce: time.Now().UTC().Unix(), } c.nodes = append(c.nodes, n) sort.Sort(byNodeID(c.nodes)) n = node{ ID: "meh-4242", LastAnnounce: time.Now().UTC().Add(time.Second * -6).Unix(), } c.nodes = append(c.nodes, n) sort.Sort(byNodeID(c.nodes)) c.Start() defer c.Stop() c.refreshNodeList("test-4242", "test-4242") if len(c.nodes) != 1 { t.Fatalf("Node meh-4242 should have left (%d != 1)", len(c.nodes)) } c.refreshNodeList("meh-4242", "test-4242") if len(c.nodes) != 2 { t.Fatalf("Node meh-4242 should have joined (%d != 2)", len(c.nodes)) } } func TestAddMirror(t *testing.T) { _, conn := PrepareRedisTest() c := NewCluster(conn) r := []int{2} c.AddMirror(&mirrors.Mirror{ ID: 2, Name: "bbb", }) if !reflect.DeepEqual(r, c.mirrorsIndex) { t.Fatalf("Expected %+v, got %+v", r, c.mirrorsIndex) } r = []int{1, 2} c.AddMirror(&mirrors.Mirror{ ID: 1, Name: "aaa", }) if !reflect.DeepEqual(r, c.mirrorsIndex) { t.Fatalf("Expected %+v, got %+v", r, c.mirrorsIndex) } r = []int{1, 2, 3} c.AddMirror(&mirrors.Mirror{ ID: 3, Name: "ccc", }) if !reflect.DeepEqual(r, c.mirrorsIndex) { t.Fatalf("Expected %+v, got %+v", r, c.mirrorsIndex) } } func TestRemoveMirror(t *testing.T) { _, conn := PrepareRedisTest() c := NewCluster(conn) c.AddMirror(&mirrors.Mirror{ ID: 1, Name: "aaa", }) c.AddMirror(&mirrors.Mirror{ ID: 2, Name: "bbb", }) c.AddMirror(&mirrors.Mirror{ ID: 3, Name: "ccc", }) c.RemoveMirror(&mirrors.Mirror{ID: 4}) r := []int{1, 2, 3} if !reflect.DeepEqual(r, c.mirrorsIndex) { t.Fatalf("Expected %+v, got %+v", r, c.mirrorsIndex) } c.RemoveMirror(&mirrors.Mirror{ID: 1}) r = []int{2, 3} if !reflect.DeepEqual(r, c.mirrorsIndex) { t.Fatalf("Expected %+v, got %+v", r, c.mirrorsIndex) } c.RemoveMirror(&mirrors.Mirror{ID: 3}) r = []int{2} if !reflect.DeepEqual(r, c.mirrorsIndex) { t.Fatalf("Expected %+v, got %+v", r, c.mirrorsIndex) } } func TestIsHandled(t *testing.T) { _, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCluster(conn) c.Start() defer c.Stop() c.AddMirror(&mirrors.Mirror{ ID: 1, Name: "aaa", }) c.AddMirror(&mirrors.Mirror{ ID: 2, Name: "bbb", }) c.AddMirror(&mirrors.Mirror{ ID: 3, Name: "ccc", }) c.AddMirror(&mirrors.Mirror{ ID: 4, Name: "ddd", }) c.nodeTotal = 1 if !c.IsHandled(1) || !c.IsHandled(2) || !c.IsHandled(3) || !c.IsHandled(4) { t.Fatalf("All mirrors should be handled") } c.nodeTotal = 2 handled := 0 if c.IsHandled(1) { handled++ } if c.IsHandled(2) { handled++ } if c.IsHandled(3) { handled++ } if c.IsHandled(4) { handled++ } if handled != 2 { t.Fatalf("Expected 2, got %d", handled) } } func TestRemoveMirrorIDFromSlice(t *testing.T) { s1 := []int{1, 2, 3, 4, 5} r1 := []int{1, 2, 4, 5} r := removeMirrorIDFromSlice(s1, 3) if !reflect.DeepEqual(r1, r) { t.Fatalf("Expected %+v, got %+v", r1, r) } s2 := []int{1, 2, 3, 4, 5} r2 := []int{2, 3, 4, 5} r = removeMirrorIDFromSlice(s2, 1) if !reflect.DeepEqual(r2, r) { t.Fatalf("Expected %+v, got %+v", r2, r) } s3 := []int{1, 2, 3, 4, 5} r3 := []int{1, 2, 3, 4} r = removeMirrorIDFromSlice(s3, 5) if !reflect.DeepEqual(r3, r) { t.Fatalf("Expected %+v, got %+v", r3, r) } s4 := []int{1, 2, 3, 4, 5} r4 := []int{1, 2, 3, 4, 5} r = removeMirrorIDFromSlice(s4, 6) if !reflect.DeepEqual(r4, r) { t.Fatalf("Expected %+v, got %+v", r4, r) } } func TestAddMirrorIDToSlice(t *testing.T) { s1 := []int{1, 3} r1 := []int{1, 2, 3} r := addMirrorIDToSlice(s1, 2) if !reflect.DeepEqual(r1, r) { t.Fatalf("Expected %+v, got %+v", r1, r) } s2 := []int{2, 3, 4} r2 := []int{1, 2, 3, 4} r = addMirrorIDToSlice(s2, 1) if !reflect.DeepEqual(r2, r) { t.Fatalf("Expected %+v, got %+v", r2, r) } s3 := []int{1, 2, 3} r3 := []int{1, 2, 3, 4} r = addMirrorIDToSlice(s3, 4) if !reflect.DeepEqual(r3, r) { t.Fatalf("Expected %+v, got %+v", r3, r) } } videolabs-mirrorbits-441567e/daemon/monitor.go000066400000000000000000000372721523530551300214330ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package daemon import ( "context" "errors" "fmt" "math/rand" "net" "net/http" "strconv" "strings" "sync" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/scan" "github.com/etix/mirrorbits/utils" "github.com/gomodule/redigo/redis" "github.com/op/go-logging" ) var ( healthCheckThreads = 10 userAgent = "Mirrorbits/" + core.VERSION + " PING CHECK" clientTimeout = time.Duration(20 * time.Second) clientDeadline = time.Duration(40 * time.Second) errRedirect = errors.New("Redirect not allowed") errMirrorNotScanned = errors.New("Mirror has not yet been scanned") log = logging.MustGetLogger("main") ) type monitor struct { redis *database.Redis cache *mirrors.Cache mirrors map[int]*mirror mapLock sync.Mutex httpClient http.Client httpTransport http.Transport healthCheckChan chan int syncChan chan int stop chan struct{} configNotifier chan bool wg sync.WaitGroup formatLongestID int cluster *cluster trace *scan.Trace } type mirror struct { mirrors.Mirror checking bool scanning bool lastCheck time.Time } func (m *mirror) NeedHealthCheck() bool { return time.Since(m.lastCheck) > time.Duration(GetConfig().CheckInterval)*time.Minute } func (m *mirror) NeedSync() bool { return time.Since(m.LastSync.Time) > time.Duration(GetConfig().ScanInterval)*time.Minute } func (m *mirror) IsScanning() bool { return m.scanning } func (m *mirror) IsChecking() bool { return m.checking } // NewMonitor returns a new instance of monitor func NewMonitor(r *database.Redis, c *mirrors.Cache) *monitor { m := new(monitor) m.redis = r m.cache = c m.cluster = NewCluster(r) m.mirrors = make(map[int]*mirror) m.healthCheckChan = make(chan int, healthCheckThreads*5) m.syncChan = make(chan int) m.stop = make(chan struct{}) m.configNotifier = make(chan bool, 1) m.trace = scan.NewTraceHandler(m.redis, m.stop) SubscribeConfig(m.configNotifier) rand.Seed(time.Now().UnixNano()) m.httpTransport = http.Transport{ DisableKeepAlives: true, MaxIdleConnsPerHost: 0, Dial: func(network, addr string) (net.Conn, error) { deadline := time.Now().Add(clientDeadline) c, err := net.DialTimeout(network, addr, clientTimeout) if err != nil { return nil, err } c.SetDeadline(deadline) return c, nil }, } m.httpClient = http.Client{ CheckRedirect: checkRedirect, Transport: &m.httpTransport, } return m } func (m *monitor) Stop() { select { case _, _ = <-m.stop: return default: m.cluster.Stop() close(m.stop) } } func (m *monitor) Wait() { m.wg.Wait() } // Return an error if the endpoint is an unauthorized redirect func checkRedirect(req *http.Request, via []*http.Request) error { redirects := req.Context().Value(core.ContextAllowRedirects).(mirrors.Redirects) if redirects.Allowed() { return nil } name := req.Context().Value(core.ContextMirrorName) for _, r := range via { if r.URL != nil { log.Warningf("Unauthorized redirection for %s: %s => %s", name, r.URL.String(), req.URL.String()) } } return errRedirect } // Main monitor loop func (m *monitor) MonitorLoop() { m.wg.Add(1) defer m.wg.Done() mirrorUpdateEvent := m.cache.GetMirrorInvalidationEvent() // Wait until the database is ready to be used for { r := m.redis.Get() if r.Err() != nil { if _, ok := r.Err().(database.NetReadyError); ok { time.Sleep(100 * time.Millisecond) continue } } break } // Scan the local repository m.retry(func(i uint) error { err := m.scanRepository() if err != nil { if i == 0 { log.Errorf("%+v", fmt.Errorf("unable to scan the local repository: %w", err)) } return err } return nil }, 1*time.Second) // Synchronize the list of all known mirrors m.retry(func(i uint) error { ids, err := m.mirrorsID() if err != nil { if i == 0 { log.Errorf("%+v", fmt.Errorf("unable to retrieve the mirror list: %w", err)) } return err } err = m.syncMirrorList(ids...) if err != nil { if i == 0 { log.Errorf("%+v", fmt.Errorf("unable to sync the list of mirrors: %w", err)) } return err } return nil }, 500*time.Millisecond) if utils.IsStopped(m.stop) { return } // Start the cluster manager m.cluster.Start() // Start the health check routines for i := 0; i < healthCheckThreads; i++ { m.wg.Add(1) go m.healthCheckLoop() } // Start the mirror sync routines for i := 0; i < GetConfig().ConcurrentSync; i++ { m.wg.Add(1) go m.syncLoop() } // Setup recurrent tasks var repositoryScanTicker <-chan time.Time repositoryScanInterval := -1 mirrorCheckTicker := time.NewTicker(1 * time.Second) // Disable the mirror check while stopping to avoid spurious events go func() { select { case <-m.stop: mirrorCheckTicker.Stop() } }() // Force a first configuration reload to setup the timers select { case m.configNotifier <- true: default: } for { select { case <-m.stop: return case v := <-mirrorUpdateEvent: id, err := strconv.Atoi(v) if err == nil { m.syncMirrorList(id) } case <-m.configNotifier: if repositoryScanInterval != GetConfig().RepositoryScanInterval { repositoryScanInterval = GetConfig().RepositoryScanInterval if repositoryScanInterval == 0 { repositoryScanTicker = nil } else { repositoryScanTicker = time.Tick(time.Duration(repositoryScanInterval) * time.Minute) } } case <-repositoryScanTicker: m.scanRepository() case <-mirrorCheckTicker.C: if m.redis.Failure() { continue } m.mapLock.Lock() for id, v := range m.mirrors { if !v.Enabled { // Ignore disabled mirrors continue } if !m.cluster.IsHandled(id) { continue } if v.NeedHealthCheck() && !v.IsChecking() { select { case m.healthCheckChan <- id: m.mirrors[id].checking = true default: } } if v.NeedSync() && !v.IsScanning() { select { case m.syncChan <- id: m.mirrors[id].scanning = true default: } } } m.mapLock.Unlock() } } } // Returns a list of all mirrors ID func (m *monitor) mirrorsID() ([]int, error) { var ids []int list, err := m.redis.GetListOfMirrors() if err != nil { return nil, err } for id := range list { ids = append(ids, id) } return ids, nil } // Sync the remote mirror struct with the local dataset func (m *monitor) syncMirrorList(mirrorsIDs ...int) error { for _, id := range mirrorsIDs { mir, err := m.cache.GetMirror(id) if err != nil && err != redis.ErrNil { log.Errorf("Fetching mirror %s failed: %s", id, err.Error()) continue } else if err == redis.ErrNil { // Mirror has been deleted m.mapLock.Lock() delete(m.mirrors, id) m.mapLock.Unlock() m.cluster.RemoveMirrorID(id) continue } // Compute the space required to display the mirror names in the logs if len(mir.Name) > m.formatLongestID { m.formatLongestID = len(mir.Name) } m.cluster.AddMirror(&mir) m.mapLock.Lock() if _, ok := m.mirrors[mir.ID]; ok { // Update existing mirror tmp := m.mirrors[mir.ID] tmp.Mirror = mir m.mirrors[mir.ID] = tmp } else { // Add new mirror m.mirrors[mir.ID] = &mirror{ Mirror: mir, } } m.mapLock.Unlock() } log.Debugf("%d mirror%s updated", len(mirrorsIDs), utils.Plural(len(mirrorsIDs))) return nil } // Main health check loop // TODO merge with the monitorLoop? func (m *monitor) healthCheckLoop() { defer m.wg.Done() for { select { case <-m.stop: return case id := <-m.healthCheckChan: if utils.IsStopped(m.stop) { return } var mptr *mirror var mirror mirror var ok bool m.mapLock.Lock() if mptr, ok = m.mirrors[id]; !ok { m.mapLock.Unlock() continue } // Copy the mirror struct for read-only access mirror = *mptr m.mapLock.Unlock() err := m.healthCheck(mirror.Mirror) if err == errMirrorNotScanned { // Not removing the 'checking' lock is intended here so the mirror won't // be checked again until the rsync/ftp scan is finished. continue } m.mapLock.Lock() if mirror, ok := m.mirrors[id]; ok { if !database.RedisIsLoading(err) { mirror.lastCheck = time.Now().UTC() } mirror.checking = false } m.mapLock.Unlock() } } } // Main sync loop // TODO merge with the monitorLoop? func (m *monitor) syncLoop() { defer m.wg.Done() for { select { case <-m.stop: return case id := <-m.syncChan: var mir mirror var mirrorPtr *mirror var ok bool m.mapLock.Lock() if mirrorPtr, ok = m.mirrors[id]; !ok { m.mapLock.Unlock() continue } mir = *mirrorPtr m.mapLock.Unlock() conn := m.redis.Get() scanning, err := scan.IsScanning(conn, id) if err != nil { conn.Close() if !database.RedisIsLoading(err) { log.Warningf("syncloop: %s", err.Error()) } goto end } else if scanning { log.Debugf("[%s] scan already in progress on another node", mir.Name) conn.Close() goto end } conn.Close() log.Debugf("Scanning %s", mir.Name) // Start fetching the latest trace go func() { err := m.trace.GetLastUpdate(mir.Mirror) if err != nil && err != scan.ErrNoTrace { var numError *strconv.NumError if errors.As(err, &numError) { if numError.Err == strconv.ErrSyntax { log.Warningf("[%s] parsing trace file failed: %s is not a valid timestamp", mir.Name, strconv.Quote(numError.Num)) return } } else { log.Warningf("[%s] fetching trace file failed: %s", mir.Name, err) } } }() err = scan.ErrNoSyncMethod // First try to scan with rsync if mir.RsyncURL != "" { _, err = scan.Scan(core.RSYNC, m.redis, m.cache, mir.RsyncURL, id, m.stop) } // If it failed or rsync wasn't supported // fallback to FTP if err != nil && err != scan.ErrScanAborted && mir.FtpURL != "" { _, err = scan.Scan(core.FTP, m.redis, m.cache, mir.FtpURL, id, m.stop) } if err == scan.ErrScanInProgress { log.Warningf("%-30.30s Scan already in progress", mir.Name) goto end } if err == nil && mir.Enabled == true && mir.IsUp() == false { m.healthCheckChan <- id } end: m.mapLock.Lock() if mirrorPtr, ok = m.mirrors[id]; ok { mirrorPtr.scanning = false } m.mapLock.Unlock() } } } // Do an actual health check against a given mirror func (m *monitor) healthCheck(mirror mirrors.Mirror) error { // Get the URL to a random file available on this mirror file, size, err := m.getRandomFile(mirror.ID) if err != nil { if err == redis.ErrNil { return errMirrorNotScanned } else if !database.RedisIsLoading(err) { log.Warningf("%s: Error: Cannot obtain a random file: %s", mirror.Name, err) } return err } // Perform health check(s) if utils.HasAnyPrefix(mirror.HttpURL, "http://", "https://") { err = m.healthCheckDo(&mirror, mirror.HttpURL, file, size) } else { err = m.healthCheckDo(&mirror, "http://"+mirror.HttpURL, file, size) err2 := m.healthCheckDo(&mirror, "https://"+mirror.HttpURL, file, size) if err2 != nil { err = err2 } } return err } func (m *monitor) healthCheckDo(mirror *mirrors.Mirror, url string, file string, size int64) error { // Get protocol proto := mirrors.HTTP if strings.HasPrefix(url, "https://") { proto = mirrors.HTTPS } // Format log output format := "%-" + fmt.Sprintf("%d.%ds %-5s ", m.formatLongestID+4, m.formatLongestID+4, proto) // Prepare the HTTP request req, err := http.NewRequest("HEAD", strings.TrimRight(url, "/")+file, nil) req.Header.Set("User-Agent", userAgent) req.Close = true ctx, cancel := context.WithTimeout(req.Context(), clientDeadline) ctx = context.WithValue(ctx, core.ContextMirrorID, mirror.ID) ctx = context.WithValue(ctx, core.ContextMirrorName, mirror.Name) ctx = context.WithValue(ctx, core.ContextAllowRedirects, mirror.AllowRedirects) req = req.WithContext(ctx) defer cancel() go func() { select { case <-m.stop: log.Debugf("Aborting health-check for %s", url) cancel() case <-ctx.Done(): } }() var contentLength string var statusCode int elapsed, err := m.httpDo(ctx, req, func(resp *http.Response, err error) error { if err != nil { return err } defer resp.Body.Close() statusCode = resp.StatusCode contentLength = resp.Header.Get("Content-Length") return nil }) if utils.IsStopped(m.stop) { return nil } if err != nil { var opErr *net.OpError if errors.As(err, &opErr) { log.Debugf("Op: %s | Net: %s | Addr: %s | Err: %s | Temporary: %t", opErr.Op, opErr.Net, opErr.Addr, opErr.Error(), opErr.Temporary()) } reason := "Unreachable" if strings.Contains(err.Error(), errRedirect.Error()) { reason = "Unauthorized redirect" } markErr := mirrors.MarkMirrorDown(m.redis, mirror.ID, proto, reason) if markErr != nil { log.Errorf(format+"Unable to mark mirror as down: %s", mirror.Name, markErr) } log.Errorf(format+"Error: %s (%dms)", mirror.Name, err.Error(), elapsed/time.Millisecond) return err } switch statusCode { case 200: err = mirrors.MarkMirrorUp(m.redis, mirror.ID, proto) if err != nil { log.Errorf(format+"Unable to mark mirror as up: %s", mirror.Name, err) } rsize, err := strconv.ParseInt(contentLength, 10, 64) if err == nil && rsize != size { log.Warningf(format+"File size mismatch! [%s] (%dms)", mirror.Name, file, elapsed/time.Millisecond) } else { log.Noticef(format+"Up! (%dms)", mirror.Name, elapsed/time.Millisecond) } case 404: err = mirrors.MarkMirrorDown(m.redis, mirror.ID, proto, fmt.Sprintf("File not found %s (error 404)", file)) if err != nil { log.Errorf(format+"Unable to mark mirror as down: %s", mirror.Name, err) } if GetConfig().DisableOnMissingFile { err = mirrors.DisableMirror(m.redis, mirror.ID) if err != nil { log.Errorf(format+"Unable to disable mirror: %s", mirror.Name, err) } } log.Errorf(format+"Error: File %s not found (error 404)", mirror.Name, file) default: err = mirrors.MarkMirrorDown(m.redis, mirror.ID, proto, fmt.Sprintf("Got status code %d", statusCode)) if err != nil { log.Errorf(format+"Unable to mark mirror as down: %s", mirror.Name, err) } log.Warningf(format+"Down! Status: %d", mirror.Name, statusCode) } return nil } func (m *monitor) httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) (time.Duration, error) { var elapsed time.Duration c := make(chan error, 1) go func() { start := time.Now() err := f(m.httpClient.Do(req)) elapsed = time.Since(start) c <- err }() select { case <-ctx.Done(): m.httpTransport.CancelRequest(req) <-c // Wait for f to return. return elapsed, ctx.Err() case err := <-c: return elapsed, err } } // Get a random filename known to be served by the given mirror func (m *monitor) getRandomFile(id int) (file string, size int64, err error) { sinterKey := fmt.Sprintf("HANDLEDFILES_%d", id) rconn := m.redis.Get() defer rconn.Close() file, err = redis.String(rconn.Do("SRANDMEMBER", sinterKey)) if err != nil { return } size, err = redis.Int64(rconn.Do("HGET", fmt.Sprintf("FILE_%s", file), "size")) if err != nil { return } return } // Trigger a sync of the local repository func (m *monitor) scanRepository() error { err := scan.ScanSource(m.redis, false, m.stop) if err != nil { log.Errorf("Scanning source failed: %s", err.Error()) } return err } // Retry a function until no errors is returned while still allowing // the process to be stopped. func (m *monitor) retry(fn func(iteration uint) error, delay time.Duration) { var i uint for { err := fn(i) i++ if err == nil { break } select { case <-m.stop: return case <-time.After(delay): } } } videolabs-mirrorbits-441567e/database/000077500000000000000000000000001523530551300177035ustar00rootroot00000000000000videolabs-mirrorbits-441567e/database/interfaces/000077500000000000000000000000001523530551300220265ustar00rootroot00000000000000videolabs-mirrorbits-441567e/database/interfaces/redis.go000066400000000000000000000003221523530551300234600ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package interfaces import "github.com/gomodule/redigo/redis" type Redis interface { Get() redis.Conn UnblockedGet() redis.Conn } videolabs-mirrorbits-441567e/database/lock.go000066400000000000000000000040601523530551300211620ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package database import ( "errors" "math/rand" "strconv" "sync" "time" "github.com/gomodule/redigo/redis" ) var ( ErrInvalidLockName = errors.New("invalid lock name") ErrAlreadyLocked = errors.New("lock already acquired") ) type Lock struct { sync.RWMutex redis *Redis name string value string held bool } func init() { rand.Seed(time.Now().UnixNano()) } func (r *Redis) AcquireLock(name string) (*Lock, error) { if len(name) == 0 { return nil, ErrInvalidLockName } l := &Lock{ redis: r, name: "LOCK_" + name, value: strconv.Itoa(rand.Int()), held: true, } conn := r.UnblockedGet() defer conn.Close() _, err := redis.String(conn.Do("SET", l.name, l.value, "NX", "PX", "5000")) if err == redis.ErrNil { return nil, ErrAlreadyLocked } else if err != nil { return nil, err } // Start the lock keepalive go l.keepalive() return l, nil } func (l *Lock) keepalive() { for { l.Lock() if l.held == false { l.Unlock() return } l.Unlock() valid, err := l.isValid() if err != nil { continue } if !valid { l.Lock() l.held = false l.Unlock() return } conn := l.redis.UnblockedGet() ok, err := redis.Bool(conn.Do("PEXPIRE", l.name, "5000")) conn.Close() if err != nil { continue } if !ok { l.held = false return } time.Sleep(1 * time.Second) } } func (l *Lock) isValid() (bool, error) { conn := l.redis.UnblockedGet() defer conn.Close() value, err := redis.String(conn.Do("GET", l.name)) if err != nil && err != redis.ErrNil { return false, err } if value != l.value { return false, nil } return true, nil } func (l *Lock) Release() { l.Lock() if l.held == false { l.Unlock() return } l.held = false l.Unlock() conn := l.redis.UnblockedGet() defer conn.Close() v, _ := redis.String(conn.Do("GET", l.name)) if v == l.value { // Delete the key only if we are still the owner conn.Do("DEL", l.name) } } func (l *Lock) Held() bool { l.RLock() defer l.RUnlock() return l.held } videolabs-mirrorbits-441567e/database/pubsub.go000066400000000000000000000102651523530551300215360ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package database import ( "sync" "time" "github.com/gomodule/redigo/redis" "github.com/op/go-logging" ) var ( log = logging.MustGetLogger("main") ) type pubsubEvent string const ( CLUSTER pubsubEvent = "_mirrorbits_cluster" FILE_UPDATE pubsubEvent = "_mirrorbits_file_update" MIRROR_UPDATE pubsubEvent = "_mirrorbits_mirror_update" MIRROR_FILE_UPDATE pubsubEvent = "_mirrorbits_mirror_file_update" PUBSUB_RECONNECTED pubsubEvent = "_mirrorbits_pubsub_reconnected" ) // Pubsub is the internal structure of the publish/subscribe handler type Pubsub struct { r *Redis rconn redis.Conn connlock sync.Mutex extSubscribers map[string][]chan string extSubscribersLock sync.RWMutex stop chan bool wg sync.WaitGroup } // NewPubsub returns a new instance of the publish/subscribe handler func NewPubsub(r *Redis) *Pubsub { pubsub := new(Pubsub) pubsub.r = r pubsub.stop = make(chan bool) pubsub.extSubscribers = make(map[string][]chan string) go pubsub.updateEvents() return pubsub } // Close all the connections to the pubsub server func (p *Pubsub) Close() { close(p.stop) p.connlock.Lock() if p.rconn != nil { // FIXME Calling p.rconn.Close() here will block indefinitely in redigo p.rconn.Send("UNSUBSCRIBE") p.rconn.Send("QUIT") p.rconn.Flush() } p.connlock.Unlock() p.wg.Wait() } // SubscribeEvent allows subscription to a particular kind of events and receive a // notification when an event is dispatched on the given channel. func (p *Pubsub) SubscribeEvent(event pubsubEvent, channel chan string) { p.extSubscribersLock.Lock() defer p.extSubscribersLock.Unlock() listeners := p.extSubscribers[string(event)] listeners = append(listeners, channel) p.extSubscribers[string(event)] = listeners } func (p *Pubsub) updateEvents() { p.wg.Add(1) defer p.wg.Done() disconnected := false connect: for { select { case <-p.stop: return default: } p.connlock.Lock() p.rconn = p.r.Get() if _, err := p.rconn.Do("PING"); err != nil { disconnected = true p.rconn.Close() p.rconn = nil p.connlock.Unlock() if RedisIsLoading(err) { // Doing a PING after (re-connection) prevents cases where redis // is currently loading the dataset and is still not ready. log.Warning("Redis is still loading the dataset in memory") } time.Sleep(500 * time.Millisecond) continue } p.connlock.Unlock() log.Debug("Subscribing pubsub") psc := redis.PubSubConn{Conn: p.rconn} psc.Subscribe(CLUSTER) psc.Subscribe(FILE_UPDATE) psc.Subscribe(MIRROR_UPDATE) psc.Subscribe(MIRROR_FILE_UPDATE) if disconnected == true { // This is a way to keep the cache active while disconnected // from redis but still clear the cache (possibly outdated) // after a successful reconnection. disconnected = false p.handleMessage(string(PUBSUB_RECONNECTED), nil) } for { switch v := psc.Receive().(type) { case redis.Message: //log.Debugf("Redis message on channel %s: message: %s", v.Channel, v.Data) p.handleMessage(v.Channel, v.Data) case redis.Subscription: log.Debugf("Redis subscription on channel %s: %s (%d)", v.Channel, v.Kind, v.Count) case error: select { case <-p.stop: return default: } log.Errorf("Pubsub disconnected: %s", v) psc.Close() p.rconn.Close() time.Sleep(50 * time.Millisecond) disconnected = true goto connect } } } } // Notify subscribers of the new message func (p *Pubsub) handleMessage(channel string, data []byte) { p.extSubscribersLock.RLock() defer p.extSubscribersLock.RUnlock() listeners := p.extSubscribers[channel] for _, listener := range listeners { // Block if the listener is not available listener <- string(data) } } // Publish a message on the pubsub server func Publish(r redis.Conn, event pubsubEvent, message string) error { _, err := r.Do("PUBLISH", string(event), message) return err } // SendPublish add the message to a transaction func SendPublish(r redis.Conn, event pubsubEvent, message string) error { err := r.Send("PUBLISH", string(event), message) return err } videolabs-mirrorbits-441567e/database/redis.go000066400000000000000000000233351523530551300213460ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package database import ( "errors" "fmt" "strconv" "strings" "sync" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/gomodule/redigo/redis" "github.com/rafaeljusto/redigomock" ) const ( redisConnectionTimeout = 200 * time.Millisecond redisReadWriteTimeout = 300 * time.Second ) var ( // ErrUnreachable is returned when the endpoint is not reachable ErrUnreachable = errors.New("redis endpoint unreachable") ) type redisPool interface { Get() redis.Conn Close() error } // Redis is the instance object of the redis database type Redis struct { pool redisPool Pubsub *Pubsub failure bool failureState sync.RWMutex knownMaster string knownMasterLock sync.Mutex stop chan bool ready chan struct{} version string } // NewRedis returns a new instance of the redis database func NewRedis() *Redis { r := NewRedisCustomPool(nil) // Asynchronous db update handler go r.dbUpdateHandler() return r } // NewRedisCustomPool returns a new instance of the redis database // using a custom pool func NewRedisCustomPool(pool redisPool) *Redis { r := &Redis{ stop: make(chan bool), ready: make(chan struct{}), } if pool != nil { // Check if we are running inside `go test` if _, ok := pool.Get().(*redigomock.Conn); ok { // Close ready since we are running a mock close(r.ready) } r.pool = pool } else { r.pool = &redis.Pool{ MaxIdle: 10, IdleTimeout: 240 * time.Second, Dial: func() (redis.Conn, error) { conn, err := r.Connect() switch err { case nil: r.setFailureState(false) default: r.setFailureState(true) } if r.version != "" && ! r.IsAtLeastVersion(core.RedisMinimumVersion) { log.Fatalf("Unsupported Redis version, please upgrade to Redis >= %s", core.RedisMinimumVersion) } return conn, err }, TestOnBorrow: func(c redis.Conn, t time.Time) error { _, err := c.Do("PING") if RedisIsLoading(err) { return nil } return err }, } } go r.connRecover() return r } // Get returns a redis connection from the pool func (r *Redis) Get() redis.Conn { select { case <-r.ready: default: return &NotReadyError{} } return r.pool.Get() } // UnblockedGet returns a redis connection from the pool even // if the database checks and/or upgrade are not finished. func (r *Redis) UnblockedGet() redis.Conn { return r.pool.Get() } // Close closes all connections to the redis database func (r *Redis) Close() { select { case _, _ = <-r.stop: return default: log.Debug("Closing databases connections") r.Pubsub.Close() r.pool.Close() close(r.stop) } } // ConnectPubsub initiates the connection to the pubsub func (r *Redis) ConnectPubsub() { if r.Pubsub == nil { r.Pubsub = NewPubsub(r) } } func (r *Redis) IsAtLeastVersion(version string) bool { return parseVersion(r.version) >= parseVersion(version) } func (r *Redis) askVersion (conn redis.Conn) (string, error) { if conn == nil { return "", ErrUnreachable } info, err := parseInfo(conn.Do("INFO", "server")) if err != nil { return "", err } return info["redis_version"], nil } // Connect initiates a new connection to the redis server func (r *Redis) Connect() (redis.Conn, error) { sentinels := GetConfig().RedisSentinels if len(sentinels) > 0 { if len(GetConfig().RedisSentinelMasterName) == 0 { r.logError("Config: RedisSentinelMasterName cannot be empty!") goto single } for _, s := range sentinels { log.Debugf("Connecting to redis sentinel %s", s.Host) var master []string var masterhost string var cm redis.Conn c, err := r.connectTo(s.Host) if err != nil { r.logError("Sentinel: %s", err.Error()) continue } //AUTH? role, err := r.askRole(c) if err != nil { r.logError("Sentinel: %s", err.Error()) goto closeSentinel } if role != "sentinel" { r.logError("Sentinel: %s is not a sentinel but a %s", s.Host, role) goto closeSentinel } master, err = redis.Strings(c.Do("SENTINEL", "get-master-addr-by-name", GetConfig().RedisSentinelMasterName)) if err == redis.ErrNil { r.logError("Sentinel: %s doesn't know the master-name %s", s.Host, GetConfig().RedisSentinelMasterName) goto closeSentinel } else if err != nil { r.logError("Sentinel: %s", err.Error()) goto closeSentinel } masterhost = fmt.Sprintf("%s:%s", master[0], master[1]) cm, err = r.connectTo(masterhost) if err != nil { r.logError("Redis master: %s", err.Error()) goto closeSentinel } if r.auth(cm) != nil { r.logError("Redis master: auth failed") goto closeMaster } if err = r.selectDB(cm); err != nil { c.Close() return nil, err } role, err = r.askRole(cm) if err != nil { r.logError("Redis master: %s", err.Error()) goto closeMaster } if role != "master" { r.logError("Redis master: %s is not a master but a %s", masterhost, role) goto closeMaster } // Close the connection to the sentinel c.Close() r.printConnectedMaster(masterhost) return cm, nil closeMaster: cm.Close() closeSentinel: c.Close() } } single: if len(GetConfig().RedisAddress) == 0 { if len(sentinels) == 0 { log.Error("No redis master available") } return nil, ErrUnreachable } if len(sentinels) > 0 && r.Failure() == false { log.Warning("No redis master available, trying using the configured RedisAddress as fallback") } c, err := r.connectTo(GetConfig().RedisAddress) if err != nil { return nil, err } if err = r.auth(c); err != nil { c.Close() return nil, err } if err = r.selectDB(c); err != nil { c.Close() return nil, err } role, err := r.askRole(c) if err != nil { r.logError("Redis master: %s", err.Error()) return nil, ErrUnreachable } if role != "master" { r.logError("Redis master: %s is not a master but a %s", GetConfig().RedisAddress, role) return nil, ErrUnreachable } r.version, err = r.askVersion(c) r.printConnectedMaster(GetConfig().RedisAddress) return c, err } func (r *Redis) connectTo(address string) (redis.Conn, error) { return redis.Dial("tcp", address, redis.DialConnectTimeout(redisConnectionTimeout), redis.DialReadTimeout(redisReadWriteTimeout), redis.DialWriteTimeout(redisReadWriteTimeout)) } func (r *Redis) askRole(c redis.Conn) (string, error) { roleReply, err := redis.Values(c.Do("ROLE")) if err != nil { return "", err } role, err := redis.String(roleReply[0], err) return role, err } func (r *Redis) auth(c redis.Conn) (err error) { if GetConfig().RedisPassword != "" { _, err = c.Do("AUTH", GetConfig().RedisPassword) } return } func (r *Redis) selectDB(c redis.Conn) (err error) { _, err = c.Do("SELECT", GetConfig().RedisDB) return } func (r *Redis) logError(format string, args ...any) { if r.Failure() { log.Debugf(format, args...) } else { log.Errorf(format, args...) } } func (r *Redis) printConnectedMaster(address string) { r.knownMasterLock.Lock() defer r.knownMasterLock.Unlock() if address != r.knownMaster && core.Daemon { r.knownMaster = address log.Infof("Connected to redis master %s (version %s)", address, r.version) } else { log.Debugf("Connected to redis master %s (version %s)", address, r.version) } } func (r *Redis) setFailureState(failure bool) { r.failureState.Lock() r.failure = failure r.failureState.Unlock() } // Failure returns true if the connection is in a failure state func (r *Redis) Failure() bool { r.failureState.RLock() defer r.failureState.RUnlock() return r.failure } func (r *Redis) connRecover() { ticker := time.NewTicker(1 * time.Second) for { select { case <-r.stop: return case <-ticker.C: if r.Failure() { if conn := r.Get(); conn != nil { // A successful Get() request will automatically unlock // other services waiting for a working connection. // This is only a way to ensure they wont wait forever. if conn.Err() != nil { log.Warningf("Database is down: %s", conn.Err().Error()) } conn.Close() } } } } } func (r *Redis) dbUpdateHandler() { var logOnce sync.Once again: upneeded, err := r.UpgradeNeeded() if err != nil { time.Sleep(100 * time.Millisecond) goto again } if upneeded { t := time.Now() err = r.Upgrade() if err == ErrAlreadyLocked { logOnce.Do(func() { log.Warning("Database upgrade running. Waiting for completion...") }) time.Sleep(100 * time.Millisecond) goto again } else if err != nil { log.Fatalf("Upgrade failed: %v", err) } log.Infof("Database upgrade successful (took %s), starting normally", time.Since(t).Round(time.Millisecond)) } close(r.ready) } // RedisIsLoading returns true if the error is of type LOADING func RedisIsLoading(err error) bool { // PARSING: "LOADING Redis is loading the dataset in memory" if err != nil && strings.HasPrefix(err.Error(), "LOADING") { return true } return false } func parseVersion(version string) int64 { // We suppport up to 3 components (major, minor, patch) in the version s := strings.Split(version, ".") for len(s) < 3 { s = append(s, "0") } format := fmt.Sprintf("%%s%%0%ds", 2) var v string for _, value := range s { v = fmt.Sprintf(format, v, value) } var result int64 var err error if result, err = strconv.ParseInt(v, 10, 64); err != nil { return -1 } return result } func parseInfo(i any, err error) (map[string]string, error) { v, err := redis.String(i, err) if err != nil { return nil, err } m := make(map[string]string) lines := strings.Split(v, "\r\n") for _, l := range lines { if strings.HasPrefix(l, "#") { continue } kv := strings.SplitN(l, ":", 2) if len(kv) < 2 { continue } m[kv[0]] = kv[1] } return m, nil } videolabs-mirrorbits-441567e/database/redis_test.go000066400000000000000000000017271523530551300224060ustar00rootroot00000000000000// Copyright (c) 2025 Arnaud Rebillout // Licensed under the MIT license package database import ( "fmt" "testing" ) func TestIsAtLeastVersion(t *testing.T) { testsFalse := [] struct { have string want string } { {"", "3.2.0"}, {"2.0", "3.2.0"}, {"2.0.0", "3.2"}, {"2.0.0", "3.2.0"}, } for i, test := range testsFalse { r := Redis{ version: test.have, } t.Run(fmt.Sprintf("testsFalse/%d", i), func(t *testing.T) { if r.IsAtLeastVersion(test.want) { t.Errorf("Expected '%s' < '%s'", test.have, test.want) } }) } testsTrue := [] struct { have string want string } { {"3.2", "3.2.0"}, {"3.2.0", "3.2.0"}, {"6.0", "3.2.0"}, {"6.0.0", "3.2"}, {"6.0.0", "3.2.0"}, } for i, test := range testsTrue { r := Redis{ version: test.have, } t.Run(fmt.Sprintf("testsTrue/%d", i), func(t *testing.T) { if ! r.IsAtLeastVersion(test.want) { t.Errorf("Expected '%s' >= '%s'", test.have, test.want) } }) } } videolabs-mirrorbits-441567e/database/upgrade.go000066400000000000000000000035731523530551300216710ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package database import ( "errors" "time" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database/upgrader" "github.com/gomodule/redigo/redis" ) var ( ErrUnsupportedVersion = errors.New("unsupported database version, please upgrade mirrorbits") ) // UpgradeNeeded returns true if a database upgrade is needed func (r *Redis) UpgradeNeeded() (bool, error) { version, err := r.GetDBFormatVersion() if err != nil { return false, err } if version > core.DBVersion { return false, ErrUnsupportedVersion } return version != core.DBVersion, nil } // GetDBFormatVersion return the current database format version func (r *Redis) GetDBFormatVersion() (int, error) { conn := r.UnblockedGet() defer conn.Close() again: version, err := redis.Int(conn.Do("GET", core.DBVersionKey)) if RedisIsLoading(err) { time.Sleep(time.Millisecond * 100) goto again } else if err == redis.ErrNil { found, err := redis.Bool(conn.Do("EXISTS", "MIRRORS")) if err != nil { return -1, err } if found { return 0, nil } _, err = conn.Do("SET", core.DBVersionKey, core.DBVersion) return core.DBVersion, err } else if err != nil { return -1, err } return version, nil } // Upgrade starts the upgrade of the database format func (r *Redis) Upgrade() error { version, err := r.GetDBFormatVersion() if err != nil { return err } if version > core.DBVersion { return ErrUnsupportedVersion } else if version == core.DBVersion { return nil } lock, err := r.AcquireLock("upgrade") if err != nil { return err } defer lock.Release() for i := version + 1; i <= core.DBVersion; i++ { u := upgrader.GetUpgrader(r, i) if u != nil { log.Warningf("Upgrading database from version %d to version %d...", i-1, i) if err = u.Upgrade(); err != nil { return err } } } return nil } videolabs-mirrorbits-441567e/database/upgrader/000077500000000000000000000000001523530551300215145ustar00rootroot00000000000000videolabs-mirrorbits-441567e/database/upgrader/upgrader.go000066400000000000000000000011431523530551300236530ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package upgrader import ( "github.com/etix/mirrorbits/database/interfaces" v1 "github.com/etix/mirrorbits/database/v1" v2 "github.com/etix/mirrorbits/database/v2" ) // Upgrader is an interface to implement a database upgrade strategy type Upgrader interface { Upgrade() error } // GetUpgrader returns the upgrader for the given target version func GetUpgrader(redis interfaces.Redis, version int) Upgrader { switch version { case 1: return v1.NewUpgraderV1(redis) case 2: return v2.NewUpgraderV2(redis) } return nil } videolabs-mirrorbits-441567e/database/utils.go000066400000000000000000000032631523530551300213760ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package database import ( "errors" "strconv" "github.com/gomodule/redigo/redis" ) func (r *Redis) GetListOfMirrors() (map[int]string, error) { conn, err := r.Connect() if err != nil { return nil, err } defer conn.Close() values, err := redis.Values(conn.Do("HGETALL", "MIRRORS")) if err != nil { return nil, err } mirrors := make(map[int]string, len(values)/2) // Convert the mirror id to int for i := 0; i < len(values); i += 2 { key, okKey := values[i].([]byte) value, okValue := values[i+1].([]byte) if !okKey || !okValue { return nil, errors.New("invalid type for mirrors key") } id, err := strconv.Atoi(string(key)) if err != nil { return nil, errors.New("invalid type for mirrors ID") } mirrors[id] = string(value) } return mirrors, nil } type NetReadyError struct { error } func (n *NetReadyError) Timeout() bool { return false } func (n *NetReadyError) Temporary() bool { return true } func NewNetTemporaryError() NetReadyError { return NetReadyError{ error: errors.New("database not ready"), } } type NotReadyError struct{} func (e *NotReadyError) Close() error { return NewNetTemporaryError() } func (e *NotReadyError) Err() error { return NewNetTemporaryError() } func (e *NotReadyError) Do(commandName string, args ...any) (reply any, err error) { return nil, NewNetTemporaryError() } func (e *NotReadyError) Send(commandName string, args ...any) error { return NewNetTemporaryError() } func (e *NotReadyError) Flush() error { return NewNetTemporaryError() } func (e *NotReadyError) Receive() (reply any, err error) { return nil, NewNetTemporaryError() } videolabs-mirrorbits-441567e/database/v1/000077500000000000000000000000001523530551300202315ustar00rootroot00000000000000videolabs-mirrorbits-441567e/database/v1/version1.go000066400000000000000000000142021523530551300223250ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package v1 import ( "fmt" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database/interfaces" "github.com/gomodule/redigo/redis" ) // NewUpgraderV1 upgrades the database from version 0 to 1 func NewUpgraderV1(redis interfaces.Redis) *Version1 { return &Version1{ Redis: redis, } } type Version1 struct { Redis interfaces.Redis } type actions struct { delete []string rename map[string]string } func (v *Version1) Upgrade() error { a := &actions{ rename: make(map[string]string), } conn := v.Redis.UnblockedGet() defer conn.Close() // Erase previous work keys (previous failed upgrade?) _, err := conn.Do("EVAL", ` local keys = redis.call('keys', ARGV[1]) for i=1,#keys,5000 do redis.call('del', unpack(keys, i, math.min(i+4999, #keys))) end return keys`, 0, "V1_*") if err != nil { return err } m, err := v.CreateMirrorIndex(a) if err != nil { return err } err = v.RenameKeys(a, m) if err != nil { return err } err = v.FixMirrorID(a, m) if err != nil { return err } err = v.RenameStats(a, m) if err != nil { return err } // Start a transaction to atomically and irrevocably set the new version conn.Send("MULTI") for k, v := range a.rename { conn.Send("RENAME", k, v) } for _, d := range a.delete { do := true for _, v := range a.rename { if d == v { // Abort the operation since this would // delete the result of a rename do = false break } } if do { conn.Send("DEL", d) } } conn.Send("SET", core.DBVersionKey, 1) // Finalize the transaction _, err = conn.Do("EXEC") // <-- At this point, if any of the previous mutation failed, it is still // safe to run a previous version of mirrorbits. return err } func (v *Version1) CreateMirrorIndex(a *actions) (map[int]string, error) { m := make(map[int]string) conn := v.Redis.UnblockedGet() defer conn.Close() // Get the v0 list of mirrors mirrors, err := redis.Strings(conn.Do("LRANGE", "MIRRORS", "0", "-1")) if err != nil { return m, err } for _, name := range mirrors { // Create a unique ID for the current mirror id, err := redis.Int(conn.Do("INCR", "LAST_MID")) if err != nil { return m, err } // Assign the ID to the current mirror if _, err = conn.Do("HSET", "V1_MIRRORS", id, name); err != nil { return m, err } m[id] = name } // Prepare for renaming a.rename["V1_MIRRORS"] = "MIRRORS" return m, nil } func (v *Version1) RenameKeys(a *actions, m map[int]string) error { conn := v.Redis.UnblockedGet() defer conn.Close() // Rename all keys to contain the ID instead of the name for id, name := range m { // Get the list of files known to this mirror files, err := redis.Strings(conn.Do("SMEMBERS", fmt.Sprintf("MIRROR_%s_FILES", name))) if err == redis.ErrNil || IsErrNoSuchKey(err) { continue } else if err != nil { return err } // Rename the FILEINFO__ keys for _, file := range files { a.rename[fmt.Sprintf("FILEINFO_%s_%s", name, file)] = fmt.Sprintf("FILEINFO_%d_%s", id, file) } // Rename the remaing global keys a.rename[fmt.Sprintf("MIRROR_%s_FILES", name)] = fmt.Sprintf("MIRRORFILES_%d", id) a.rename[fmt.Sprintf("HANDLEDFILES_%s", name)] = fmt.Sprintf("HANDLEDFILES_%d", id) // MIRROR_%s -> MIRROR_%d is handled by FixMirrorID } // Get the list of files in the local repo files, err := redis.Strings(conn.Do("SMEMBERS", "FILES")) if err != nil && err != redis.ErrNil { return err } // Rename the keys within FILEMIRRORS_* for _, file := range files { // Get the list of mirrors having each file names, err := redis.Strings(conn.Do("SMEMBERS", fmt.Sprintf("FILEMIRRORS_%s", file))) if err != nil { return err } for _, name := range names { var id int for mid, mname := range m { if mname == name { id = mid break } } if id == 0 { continue } conn.Send("SADD", fmt.Sprintf("V1_FILEMIRRORS_%s", file), id) } if err := conn.Flush(); err != nil { return err } // Mark the key for renaming a.rename[fmt.Sprintf("V1_FILEMIRRORS_%s", file)] = fmt.Sprintf("FILEMIRRORS_%s", file) } return nil } func (v *Version1) FixMirrorID(a *actions, m map[int]string) error { conn := v.Redis.UnblockedGet() defer conn.Close() // Replace ID by the new mirror id // Add a field 'name' containing the mirror name for id, name := range m { err := CopyKey(conn, fmt.Sprintf("MIRROR_%s", name), fmt.Sprintf("V1_MIRROR_%d", id)) if err != nil { return err } conn.Send("HSET", fmt.Sprintf("V1_MIRROR_%d", id), "ID", id, "name", name) a.rename[fmt.Sprintf("V1_MIRROR_%d", id)] = fmt.Sprintf("MIRROR_%d", id) a.delete = append(a.delete, fmt.Sprintf("MIRROR_%s", name)) } if err := conn.Flush(); err != nil { return err } return nil } func (v *Version1) RenameStats(a *actions, m map[int]string) error { conn := v.Redis.UnblockedGet() defer conn.Close() keys, err := redis.Strings(conn.Do("KEYS", "STATS_MIRROR_*")) if err != nil && err != redis.ErrNil { return err } for _, key := range keys { // Here we get two formats: // - STATS_MIRROR_* // - STATS_MIRROR_BYTES_* // and each of them with three differents dates (year, year+month, year+month+day) stats, err := redis.StringMap(conn.Do("HGETALL", key)) if err != nil { return err } for identifier, value := range stats { var id int for mid, mname := range m { if mname == identifier { id = mid break } } if id == 0 { // Mirror does not exist anymore // This is expected if mirrors were removed over time continue } conn.Send("HSET", "V1_"+key, id, value) a.rename["V1_"+key] = key } if err := conn.Flush(); err != nil { return err } } return nil } func CopyKey(conn redis.Conn, src, dst string) error { dmp, err := redis.String(conn.Do("DUMP", src)) if err != nil { return err } _, err = conn.Do("RESTORE", dst, 0, dmp, "REPLACE") return err } // IsErrNoSuchKey return true if error is of type "no such key" func IsErrNoSuchKey(err error) bool { // PARSING: "ERR no such key" if err != nil && err.Error() == "ERR no such key" { return true } return false } videolabs-mirrorbits-441567e/database/v2/000077500000000000000000000000001523530551300202325ustar00rootroot00000000000000videolabs-mirrorbits-441567e/database/v2/version2.go000066400000000000000000000073641523530551300223420ustar00rootroot00000000000000// Copyright (c) 2024 Arnaud Rebillout // Licensed under the MIT license package v2 import ( "strings" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database/interfaces" "github.com/gomodule/redigo/redis" ) // NewUpgraderV2 upgrades the database from version 1 to 2 func NewUpgraderV2(redis interfaces.Redis) *Version2 { return &Version2{ Redis: redis, } } type Version2 struct { Redis interfaces.Redis } type actions struct { rename map[string]string } func (v *Version2) Upgrade() error { a := &actions{ rename: make(map[string]string), } conn := v.Redis.UnblockedGet() defer conn.Close() // Erase previous work keys (previous failed upgrade?) _, err := conn.Do("EVAL", ` local keys = redis.call('keys', ARGV[1]) for i=1,#keys,5000 do redis.call('del', unpack(keys, i, math.min(i+4999, #keys))) end return keys`, 0, "V2_*") if err != nil { return err } err = v.UpdateMirrors(a) if err != nil { return err } // Start a transaction to atomically and irrevocably set the new version conn.Send("MULTI") for k, v := range a.rename { conn.Send("RENAME", k, v) } conn.Send("SET", core.DBVersionKey, 2) // Finalize the transaction _, err = conn.Do("EXEC") // <-- At this point, if any of the previous mutation failed, it is still // safe to run a previous version of mirrorbits. return err } func (v *Version2) UpdateMirrors(a *actions) error { conn := v.Redis.UnblockedGet() defer conn.Close() // Get the list of mirrors keys, err := redis.Strings(conn.Do("KEYS", "MIRROR_*")) if err != nil && err != redis.ErrNil { return err } // Iterate on mirrors for _, keyProd := range keys { // Copy the key key := "V2_" + keyProd err := CopyKey(conn, keyProd, key) if err != nil { return err } // Get the http url url, err := redis.String(conn.Do("HGET", key, "http")) if err != nil { return err } // Get the status. Note that the key might not exist if ever // the mirror was never enabled or scanned successfully. up, err := redis.Bool(conn.Do("HGET", key, "up")) if err != nil && err != redis.ErrNil { return err } upExists := true if err == redis.ErrNil { upExists = false } // Get the excluded reason. As above: the key might not exist. reason, err := redis.String(conn.Do("HGET", key, "excludeReason")) if err != nil && err != redis.ErrNil { return err } reasonExists := true if err == redis.ErrNil { reasonExists = false } // Start a transaction to do all the changes in one go conn.Send("MULTI") if strings.HasPrefix(url, "https://") { // Update up key if needed if upExists { conn.Send("HSET", key, "httpsUp", up) conn.Send("HDEL", key, "up") } // Update reason key if needed if reasonExists { conn.Send("HSET", key, "httpsDownReason", reason) conn.Send("HDEL", key, "excludeReason") } } else { // Update up key if needed if upExists { conn.Send("HSET", key, "httpUp", up) conn.Send("HDEL", key, "up") } // Update reason key if needed if reasonExists { conn.Send("HSET", key, "httpDownReason", reason) conn.Send("HDEL", key, "excludeReason") } } // Finalize the transaction _, err = conn.Do("EXEC") if err != nil { return err } // Mark the key for renaming a.rename[key] = keyProd } return nil } func CopyKey(conn redis.Conn, src, dst string) error { // NB: Redis COPY https://redis.io/commands/copy/ is only available // since Redis 6.2, released in Feb 2021. That's a bit too recent, // so let's stick with the DUMP/RESTORE combination implemented in // this function (and copy/pasted from the v1 database upgrade). dmp, err := redis.String(conn.Do("DUMP", src)) if err != nil { return err } _, err = conn.Do("RESTORE", dst, 0, dmp, "REPLACE") return err } videolabs-mirrorbits-441567e/doc.go000066400000000000000000000027641523530551300172440ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license // Mirrorbits is a geographic download redirector for distributing files efficiently across a set of mirrors. // // Prerequisites // // Before diving into the install section ensures you have: // - Redis 2.8.12 (or later) // - libgeoip // - a recent geoip database (see contrib/geoip/) // // Installation // // You can now proceed to the installation by downloading a prebuilt release on // https://github.com/etix/mirrorbits/releases or by building it yourself: // go get github.com/etix/mirrorbits // go install -v github.com/etix/mirrorbits // If you plan to use the web UI be sure to install the templates found on // https://github.com/etix/mirrorbits/tree/master/templates into your system (usually in /usr/share/mirrorbits). // // Configuration // // A sample configuration file can be found in the git repository: // https://github.com/etix/mirrorbits/blob/master/mirrorbits.conf // // Running // // Mirrorbits is a self-contained application and is, at the same time, the server and the cli. // // To run the server: // mirrorbits -D // To run the cli: // mirrorbits help // // Upgrading // // Mirrorbits has a mode called seamless binary upgrade to upgrade the server executable at runtime // without service disruption. Once the binary has been replaced just issue the following // command in the cli: // mirrorbits upgrade // // For more information visit the official page: // https://github.com/etix/mirrorbits/ package main videolabs-mirrorbits-441567e/filesystem/000077500000000000000000000000001523530551300203235ustar00rootroot00000000000000videolabs-mirrorbits-441567e/filesystem/fileinfo.go000066400000000000000000000012051523530551300224430ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package filesystem import ( "time" ) // FileInfo is a struct embedding details about a file served by // the redirector. type FileInfo struct { Path string `redis:"-"` Size int64 `redis:"size" json:",omitempty"` ModTime time.Time `redis:"modTime" json:",omitempty"` Sha1 string `redis:"sha1" json:",omitempty"` Sha256 string `redis:"sha256" json:",omitempty"` Md5 string `redis:"md5" json:",omitempty"` } // NewFileInfo returns a new FileInfo object func NewFileInfo(path string) FileInfo { return FileInfo{ Path: path, } } videolabs-mirrorbits-441567e/filesystem/fs.go000066400000000000000000000025211523530551300212620ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package filesystem import ( "errors" "path/filepath" "strings" ) var ( // ErrOutsideRepo is returned when the target file is outside of the repository ErrOutsideRepo = errors.New("target file outside repository") ) // EvaluateFilePath sanitize and validate the file against the local repository func EvaluateFilePath(repository, urlpath string) (string, error) { fpath := repository + urlpath // Get the absolute file path fpath, err := filepath.Abs(fpath) if err != nil { return "", err } // Check if absolute path is within the repository if !IsInRepository(repository, fpath) { return "", ErrOutsideRepo } // Evaluate symlinks targetPath, err := filepath.EvalSymlinks(fpath) if err != nil { return "", err } if targetPath != fpath { targetPath, err = filepath.Abs(targetPath) if err != nil { return "", err } if !IsInRepository(repository, targetPath) { return "", ErrOutsideRepo } return targetPath[len(repository):], nil } return fpath[len(repository):], nil } // IsInRepository ensures that the given file path is contained in the repository func IsInRepository(repository, filePath string) bool { if filePath == repository { return true } if strings.HasPrefix(filePath, repository+"/") { return true } return false } videolabs-mirrorbits-441567e/filesystem/hash.go000066400000000000000000000031221523530551300215730ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package filesystem import ( "bufio" "crypto/md5" "crypto/sha1" "crypto/sha256" "encoding/hex" "hash" "io" "os" . "github.com/etix/mirrorbits/config" ) // HashFile generates a human readable hash of the given file path func HashFile(path string) (hashes FileInfo, err error) { f, err := os.Open(path) if err != nil { return } defer f.Close() reader := bufio.NewReader(f) var writers []io.Writer if GetConfig().Hashes.SHA1 { hsha1 := newHasher(sha1.New(), &hashes.Sha1) defer hsha1.Close() writers = append(writers, hsha1) } if GetConfig().Hashes.SHA256 { hsha256 := newHasher(sha256.New(), &hashes.Sha256) defer hsha256.Close() writers = append(writers, hsha256) } if GetConfig().Hashes.MD5 { hmd5 := newHasher(md5.New(), &hashes.Md5) defer hmd5.Close() writers = append(writers, hmd5) } if len(writers) == 0 { return } w := io.MultiWriter(writers...) _, err = io.Copy(w, reader) if err != nil { return } return } type hasher struct { hash.Hash output *string } func newHasher(hash hash.Hash, output *string) hasher { return hasher{ Hash: hash, output: output, } } func (h hasher) Close() error { *h.output = hex.EncodeToString(h.Sum(nil)) return nil } // Sha256sum generates a human readable sha256 hash of the given file path func Sha256sum(path string) ([]byte, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return nil, err } return h.Sum(nil), nil } videolabs-mirrorbits-441567e/go.mod000066400000000000000000000020741523530551300172500ustar00rootroot00000000000000module github.com/etix/mirrorbits require ( github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/etix/goftp v0.0.0-20170217140226-0c13163a1028 github.com/golang/protobuf v1.3.2 github.com/gomodule/redigo v0.0.0-20181026001555-e8fc0692a7e2 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/oschwald/maxminddb-golang v1.5.0 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 github.com/youtube/vitess v0.0.0-20181105031612-54855ec7b369 golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 golang.org/x/term v0.1.0 google.golang.org/grpc v1.27.1 gopkg.in/tylerb/graceful.v1 v1.2.15 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/kr/pretty v0.1.0 // indirect github.com/stretchr/testify v1.4.0 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect golang.org/x/text v0.3.2 // indirect google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect vitess.io/vitess v2.1.1+incompatible // indirect ) go 1.18 videolabs-mirrorbits-441567e/go.sum000066400000000000000000000217001523530551300172720ustar00rootroot00000000000000cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etix/goftp v0.0.0-20170217140226-0c13163a1028 h1:hO2NDwWjaY+FjWoZdMLapjkxt9Gpnmjb4ZdfOaQi9nI= github.com/etix/goftp v0.0.0-20170217140226-0c13163a1028/go.mod h1:broujVOEKwPL7fT3Xa1mJzV2aIqT1keOqSVEPqEO61Y= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gomodule/redigo v0.0.0-20181026001555-e8fc0692a7e2 h1:Gzyurvlb8eehpl7l2YLkMddyOXWkdQN7wU5x5l/xM9s= github.com/gomodule/redigo v0.0.0-20181026001555-e8fc0692a7e2/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/oschwald/maxminddb-golang v1.5.0 h1:rmyoIV6z2/s9TCJedUuDiKht2RN12LWJ1L7iRGtWY64= github.com/oschwald/maxminddb-golang v1.5.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/youtube/vitess v0.0.0-20181105031612-54855ec7b369 h1:Hg7gcIGpsMjVX63qXG6QYpin4kX5WrJ05VSAyxzgxIA= github.com/youtube/vitess v0.0.0-20181105031612-54855ec7b369/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc= golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51 h1:Ex1mq5jaJof+kRnYi3SlYJ8KKa9Ao3NHyIT5XJ1gF6U= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/tylerb/graceful.v1 v1.2.15 h1:1JmOyhKqAyX3BgTXMI84LwT6FOJ4tP2N9e2kwTCM0nQ= gopkg.in/tylerb/graceful.v1 v1.2.15/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2PAbAC/uJ8= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= videolabs-mirrorbits-441567e/http/000077500000000000000000000000001523530551300171165ustar00rootroot00000000000000videolabs-mirrorbits-441567e/http/context.go000066400000000000000000000065241523530551300211400ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package http import ( "net/http" "net/url" "strings" . "github.com/etix/mirrorbits/config" ) // RequestType defines the type of the request type RequestType int // SecureOption is the type that defines TLS requirements type SecureOption int const ( STANDARD RequestType = iota MIRRORLIST FILESTATS MIRRORSTATS CHECKSUM UNDEFINED SecureOption = iota WITHTLS WITHOUTTLS ) // Context represents the context of a request type Context struct { r *http.Request w http.ResponseWriter t Templates v url.Values typ RequestType isMirrorList bool isMirrorStats bool isFileStats bool isChecksum bool isPretty bool secureOption SecureOption } // NewContext returns a new instance of Context func NewContext(w http.ResponseWriter, r *http.Request, t Templates) *Context { c := &Context{r: r, w: w, t: t, v: r.URL.Query()} if c.paramBool("mirrorlist") { c.typ = MIRRORLIST c.isMirrorList = true } else if c.paramBool("stats") { c.typ = FILESTATS c.isFileStats = true } else if c.paramBool("mirrorstats") { c.typ = MIRRORSTATS c.isMirrorStats = true } else if c.paramBool("md5") || c.paramBool("sha1") || c.paramBool("sha256") { c.typ = CHECKSUM c.isChecksum = true } else { c.typ = STANDARD } if c.paramBool("pretty") { c.isPretty = true } // Check for HTTPS requirements proto := strings.ToLower(r.Header.Get("X-Forwarded-Proto")) if proto == "https" { c.secureOption = WITHTLS } else if proto == "http" && GetConfig().AllowHTTPToHTTPSRedirects == false { c.secureOption = WITHOUTTLS } // Check if the query sets (thus overrides) HTTPS requirements v, ok := c.v["https"] if ok { if v[0] == "1" { c.secureOption = WITHTLS } else if v[0] == "0" { c.secureOption = WITHOUTTLS } } return c } // Request returns the underlying http.Request of the current request func (c *Context) Request() *http.Request { return c.r } // ResponseWriter returns the underlying http.ResponseWriter of the current request func (c *Context) ResponseWriter() http.ResponseWriter { return c.w } // Templates returns the instance of precompiled templates func (c *Context) Templates() Templates { return c.t } // Type returns the type of the current request func (c *Context) Type() RequestType { return c.typ } // IsMirrorlist returns true if the mirror list has been requested func (c *Context) IsMirrorlist() bool { return c.isMirrorList } // IsFileStats returns true if the file stats has been requested func (c *Context) IsFileStats() bool { return c.isFileStats } // IsMirrorStats returns true if the mirror stats has been requested func (c *Context) IsMirrorStats() bool { return c.isMirrorStats } // IsChecksum returns true if a checksum has been requested func (c *Context) IsChecksum() bool { return c.isChecksum } // IsPretty returns true if the pretty json has been requested func (c *Context) IsPretty() bool { return c.isPretty } // QueryParam returns the value associated with the given query parameter func (c *Context) QueryParam(key string) string { return c.v.Get(key) } // SecureOption returns the selected secure option func (c *Context) SecureOption() SecureOption { return c.secureOption } func (c *Context) paramBool(key string) bool { _, ok := c.v[key] return ok } videolabs-mirrorbits-441567e/http/gzip.go000066400000000000000000000020241523530551300204140ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package http import ( "io" "net/http" "strings" . "github.com/etix/mirrorbits/config" "github.com/youtube/vitess/go/cgzip" ) type gzipResponseWriter struct { io.Writer http.ResponseWriter typeGuessed bool } func (w *gzipResponseWriter) Write(b []byte) (int, error) { if !w.typeGuessed { if w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", http.DetectContentType(b)) } w.typeGuessed = true } return w.Writer.Write(b) } // NewGzipHandler is an HTTP handler used to compress responses if supported by the client func NewGzipHandler(fn http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if !GetConfig().Gzip || !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { fn(w, r) return } w.Header().Set("Content-Encoding", "gzip") gz, _ := cgzip.NewWriterLevel(w, cgzip.Z_BEST_SPEED) defer gz.Close() fn(&gzipResponseWriter{Writer: gz, ResponseWriter: w}, r) } } videolabs-mirrorbits-441567e/http/http.go000066400000000000000000000437641523530551300204420ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package http import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "html/template" "math/rand" "net" "net/http" "os" "path/filepath" "sort" "strconv" "strings" "sync" "time" systemd "github.com/coreos/go-systemd/daemon" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/logs" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/network" "github.com/etix/mirrorbits/utils" "github.com/gomodule/redigo/redis" "github.com/op/go-logging" "gopkg.in/tylerb/graceful.v1" ) var ( log = logging.MustGetLogger("main") ) // HTTP represents an instance of the HTTP webserver type HTTP struct { geoip *network.GeoIP redis *database.Redis templates Templates Listener *net.Listener server *graceful.Server serverStopChan <-chan struct{} stats *Stats cache *mirrors.Cache engine mirrorSelection Restarting bool stopped bool stoppedMutex sync.Mutex } // Templates is a struct embedding instances of the precompiled templates type Templates struct { *sync.RWMutex mirrorlist *template.Template mirrorstats *template.Template } // HTTPServer is the constructor of the HTTP server func HTTPServer(redis *database.Redis, cache *mirrors.Cache) *HTTP { h := new(HTTP) h.redis = redis h.geoip = network.NewGeoIP() h.templates.RWMutex = new(sync.RWMutex) h.templates.mirrorlist = template.Must(h.LoadTemplates("mirrorlist")) h.templates.mirrorstats = template.Must(h.LoadTemplates("mirrorstats")) h.cache = cache h.stats = NewStats(redis) h.engine = DefaultEngine{} http.Handle("/", NewGzipHandler(h.requestDispatcher)) // Load the GeoIP databases if err := h.geoip.LoadGeoIP(); err != nil { var gerr network.GeoIPError if errors.As(err, &gerr) { for _, e := range gerr.Errors { log.Critical(e.Error()) } if gerr.IsFatal() { if len(GetConfig().Fallbacks) == 0 { log.Fatal("Can't load the GeoIP databases, please set a valid path in the mirrorbits configuration") } else { log.Critical("Can't load the GeoIP databases, all requests will be served by the fallback mirrors") } } else { log.Critical("One or more GeoIP database could not be loaded, service will run in degraded mode") } } } // Initialize the random number generator rand.Seed(time.Now().UnixNano()) return h } // SetListener can be used to set a different listener that should be used by the // HTTP server. This is primarily used during seamless binary upgrade. func (h *HTTP) SetListener(l net.Listener) { h.Listener = &l } // Stop gracefully stops the HTTP server with a timeout to let // the remaining connections finish func (h *HTTP) Stop(timeout time.Duration) { /* Close the server and process remaining connections */ h.stoppedMutex.Lock() defer h.stoppedMutex.Unlock() if h.stopped { return } h.stopped = true h.server.Stop(timeout) } // Terminate terminates the current HTTP server gracefully func (h *HTTP) Terminate() { /* Wait for the server to stop */ select { case <-h.serverStopChan: } /* Commit the latest recorded stats to the database */ h.stats.Terminate() } // StopChan returns a channel that notifies when the server is stopped func (h *HTTP) StopChan() <-chan struct{} { return h.serverStopChan } // Reload the configuration func (h *HTTP) Reload() { // Reload the GeoIP database h.geoip.LoadGeoIP() // Reload the templates h.templates.Lock() if t, err := h.LoadTemplates("mirrorlist"); err == nil { h.templates.mirrorlist = t } else { log.Errorf("could not reload templates 'mirrorlist': %s", err.Error()) } if t, err := h.LoadTemplates("mirrorstats"); err == nil { h.templates.mirrorstats = t } else { log.Errorf("could not reload templates 'mirrorstats': %s", err.Error()) } h.templates.Unlock() } // RunServer is the main function used to start the HTTP server func (h *HTTP) RunServer() (err error) { // If listener isn't nil that means that we're running a seamless // binary upgrade and we have recovered an already running listener if h.Listener == nil { proto := "tcp" address := GetConfig().ListenAddress if strings.HasPrefix(address, "unix:") { proto = "unix" address = strings.TrimPrefix(address, "unix:") } listener, err := net.Listen(proto, address) if err != nil { log.Fatal("Listen: ", err) } h.SetListener(listener) } h.server = &graceful.Server{ // http Server: &http.Server{ Handler: nil, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, MaxHeaderBytes: 1 << 20, }, // graceful Timeout: 10 * time.Second, NoSignalHandling: true, } h.serverStopChan = h.server.StopChan() log.Infof("Service listening on %s", GetConfig().ListenAddress) // Since main blocks here until completion, tell systemd we're ready. // This is a no-op if NOTIFY_SOCKET isn't set. if os.Getenv("NOTIFY_SOCKET") != "" { log.Debug("Notifying systemd of readiness") systemd.SdNotify(false, systemd.SdNotifyReady) } /* Serve until we receive a SIGTERM */ return h.server.Serve(*h.Listener) } func (h *HTTP) requestDispatcher(w http.ResponseWriter, r *http.Request) { h.templates.RLock() ctx := NewContext(w, r, h.templates) h.templates.RUnlock() w.Header().Set("Server", "Mirrorbits/"+core.VERSION) switch ctx.Type() { case MIRRORLIST: fallthrough case STANDARD: h.mirrorHandler(w, r, ctx) case MIRRORSTATS: h.mirrorStatsHandler(w, r, ctx) case FILESTATS: h.fileStatsHandler(w, r, ctx) case CHECKSUM: h.checksumHandler(w, r, ctx) } } func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Context) { //XXX it would be safer to recover in case of panic // Sanitize path urlPath, err := filesystem.EvaluateFilePath(GetConfig().Repository, r.URL.Path) if err != nil { if err == filesystem.ErrOutsideRepo { http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } // Get details about the requested file. Errors are not fatal, and // expected when the database is not ready: fallbacks will handle it. fileInfo, err := h.cache.GetFileInfo(urlPath) if err != nil { //log.Debugf("Error while fetching Fileinfo: %s", err.Error()) } if checkIfModifiedSince(r, fileInfo.ModTime) == condFalse { setLastModified(w, fileInfo.ModTime) writeNotModified(w) return } remoteIP := network.ExtractRemoteIP(r.Header.Get("X-Forwarded-For")) if len(remoteIP) == 0 { remoteIP = network.RemoteIPFromAddr(r.RemoteAddr) } if ctx.IsMirrorlist() { fromip := ctx.QueryParam("fromip") if net.ParseIP(fromip) != nil { remoteIP = fromip } } clientInfo := h.geoip.GetRecord(remoteIP) //TODO return a pointer? mlist, excluded, err := h.engine.Selection(ctx, h.cache, &fileInfo, clientInfo) /* Handle errors */ fallback := false var netErr net.Error if errors.As(err, &netErr) || len(mlist) == 0 { /* Handle fallbacks */ fallbacks := GetConfig().Fallbacks if len(fallbacks) > 0 { fallback = true for i, f := range fallbacks { // Set the absolute URL var absURL string if utils.HasAnyPrefix(f.URL, "http://", "https://") { absURL = f.URL } else if ctx.SecureOption() == WITHOUTTLS { absURL = "http://" + f.URL } else { absURL = "https://" + f.URL } // Create a mirror object and add it to the result mlist = append(mlist, mirrors.Mirror{ ID: i * -1, Name: fmt.Sprintf("fallback%d", i), HttpURL: f.URL, CountryCodes: strings.ToUpper(f.CountryCode), CountryFields: []string{strings.ToUpper(f.CountryCode)}, ContinentCode: strings.ToUpper(f.ContinentCode), AbsoluteURL: absURL}) } sort.Sort(mirrors.ByRank{Mirrors: mlist, ClientInfo: clientInfo}) } else { // No fallback in stock, there's nothing else we can do http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) return } } else if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } results := &mirrors.Results{ FileInfo: fileInfo, MirrorList: mlist, ExcludedList: excluded, ClientInfo: clientInfo, IP: remoteIP, Fallback: fallback, LocalJSPath: GetConfig().LocalJSPath, } var resultRenderer resultsRenderer if ctx.IsMirrorlist() { resultRenderer = &MirrorListRenderer{} } else { switch GetConfig().OutputMode { case "json": resultRenderer = &JSONRenderer{} case "redirect": resultRenderer = &RedirectRenderer{} case "auto": accept := r.Header.Get("Accept") if strings.Index(accept, "application/json") >= 0 { resultRenderer = &JSONRenderer{} } else { resultRenderer = &RedirectRenderer{} } default: http.Error(w, "No page renderer", http.StatusInternalServerError) return } } w.Header().Set("Cache-Control", "private, no-cache") status, err := resultRenderer.Write(ctx, results) if err != nil { http.Error(w, err.Error(), status) } if !ctx.IsMirrorlist() { logs.LogDownload(resultRenderer.Type(), r.Method, status, results, err) if len(mlist) > 0 && r.Method == "GET" && resultRenderer.Type() == "REDIRECT" { timeout := GetConfig().SameDownloadInterval if r.Header.Get("Range") == "" || timeout == 0 { h.stats.CountDownload(mlist[0], fileInfo) } else { downloaderID := remoteIP+"/"+r.Header.Get("User-Agent") hash := sha256.New() hash.Write([]byte(downloaderID)) chk := hex.EncodeToString(hash.Sum(nil)) rconn := h.redis.Get() defer rconn.Close() tempKey := "DOWNLOADED_"+chk+"_"+urlPath prev := "" if h.redis.IsAtLeastVersion("6.2.0") { // Get and set the key in one command. prev, _ = redis.String(rconn.Do("SET", tempKey, 1, "GET", "EX", timeout)) } else { prev, _ = redis.String(rconn.Do("GET", tempKey)) } if prev == "" { // Only count partial requests as a new download if // we haven't had any recently from the same client // (i.e. same (IP, user-agent) hash). This prevents // from counting multiple times a single client // downloading a single file in pieces, such as // torrent clients when files are used as web seeds. h.stats.CountDownload(mlist[0], fileInfo) } if ! h.redis.IsAtLeastVersion("6.2.0") { // Set the key anyway to reset the timer. rconn.Send("SET", tempKey, 1, "EX", timeout) } } } } return } // LoadTemplates pre-loads templates from the configured template directory func (h *HTTP) LoadTemplates(name string) (t *template.Template, err error) { t = template.New("t") t.Funcs(template.FuncMap{ "add": utils.Add, "sizeof": utils.ReadableSize, "version": utils.Version, "hostname": utils.Hostname, "concaturl": utils.ConcatURL, "dateutc": utils.FormattedDateUTC, "iszero": utils.IsZero, }) t, err = t.ParseFiles( filepath.Clean(GetConfig().Templates+"/base.html"), filepath.Clean(fmt.Sprintf("%s/%s.html", GetConfig().Templates, name))) if err != nil { var e *os.PathError if errors.As(err, &e) { log.Fatalf(fmt.Sprintf("Cannot load template %s: %s", e.Path, e.Err.Error())) } else { log.Fatal(err.Error()) } } return t, err } // StatsFileNow is the structure containing the latest stats of a file type StatsFileNow struct { Today int64 Month int64 Year int64 Total int64 } // StatsFilePeriod is the structure containing the stats for the given period type StatsFilePeriod struct { Period string Downloads int64 } // See stats.go header for the storage structure func (h *HTTP) fileStatsHandler(w http.ResponseWriter, r *http.Request, ctx *Context) { var output []byte rconn := h.redis.Get() defer rconn.Close() req := strings.SplitN(ctx.QueryParam("stats"), "-", 3) // Sanity check for _, e := range req { if e == "" { continue } if _, err := strconv.ParseInt(e, 10, 0); err != nil { http.Error(w, "Invalid period", http.StatusBadRequest) return } } if len(req) == 0 || req[0] == "" { fkey := fmt.Sprintf("STATS_FILE_%s", time.Now().Format("2006_01_02")) rconn.Send("MULTI") for i := 0; i < 4; i++ { rconn.Send("HGET", fkey, r.URL.Path) fkey = fkey[:strings.LastIndex(fkey, "_")] } res, err := redis.Values(rconn.Do("EXEC")) if err != nil && err != redis.ErrNil { http.Error(w, err.Error(), http.StatusInternalServerError) return } s := &StatsFileNow{} s.Today, _ = redis.Int64(res[0], err) s.Month, _ = redis.Int64(res[1], err) s.Year, _ = redis.Int64(res[2], err) s.Total, _ = redis.Int64(res[3], err) output, err = json.MarshalIndent(s, "", " ") } else { // Generate the redis key dkey := "STATS_FILE_" for _, e := range req { dkey += fmt.Sprintf("%s_", e) } dkey = dkey[:len(dkey)-1] v, err := redis.Int64(rconn.Do("HGET", dkey, r.URL.Path)) if err != nil && err != redis.ErrNil { http.Error(w, err.Error(), http.StatusInternalServerError) return } s := &StatsFilePeriod{Period: ctx.QueryParam("stats"), Downloads: v} output, err = json.MarshalIndent(s, "", " ") } w.Write(output) } func (h *HTTP) checksumHandler(w http.ResponseWriter, r *http.Request, ctx *Context) { // Sanitize path urlPath, err := filesystem.EvaluateFilePath(GetConfig().Repository, r.URL.Path) if err != nil { if err == filesystem.ErrOutsideRepo { http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) return } // Get details about the requested file fileInfo, err := h.cache.GetFileInfo(urlPath) if err != nil { log.Errorf("Error while fetching Fileinfo: %s", err.Error()) http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) return } var hash string if ctx.paramBool("md5") { hash = fileInfo.Md5 } else if ctx.paramBool("sha1") { hash = fileInfo.Sha1 } else if ctx.paramBool("sha256") { hash = fileInfo.Sha256 } if len(hash) == 0 { http.Error(w, "Hash type not supported", http.StatusNotFound) return } w.Header().Set("Content-Type", "text/plain; charset=UTF-8") w.Write([]byte(fmt.Sprintf("%s %s", hash, filepath.Base(fileInfo.Path)))) return } // MirrorStats contains the stats of a given mirror type MirrorStats struct { ID int Name string Downloads int64 Bytes int64 PercentD float32 PercentB float32 SyncOffset SyncOffset TZOffset time.Duration } // SyncOffset contains the time offset between the mirror and the local repository type SyncOffset struct { Valid bool Value int // in hours HumanReadable string } // MirrorStatsPage contains the values needed to generate the mirrorstats page type MirrorStatsPage struct { List []MirrorStats MirrorList []mirrors.Mirror LocalJSPath string HasTZAdjustement bool } // byDownloadNumbers is a sorting function type byDownloadNumbers struct { mirrorStatsSlice } func (b byDownloadNumbers) Less(i, j int) bool { if b.mirrorStatsSlice[i].Downloads > b.mirrorStatsSlice[j].Downloads { return true } return false } // mirrorStatsSlice is a slice of MirrorStats type mirrorStatsSlice []MirrorStats func (s mirrorStatsSlice) Len() int { return len(s) } func (s mirrorStatsSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (h *HTTP) mirrorStatsHandler(w http.ResponseWriter, r *http.Request, ctx *Context) { rconn := h.redis.Get() defer rconn.Close() // Get all mirrors ID mirrorsMap, err := h.redis.GetListOfMirrors() if err != nil { http.Error(w, "Cannot fetch the list of mirrors", http.StatusInternalServerError) return } var mirrorsIDs []int for id := range mirrorsMap { // We need a common order to iterate the // results from Redis. mirrorsIDs = append(mirrorsIDs, id) } rconn.Send("MULTI") // Get all mirrors stats for _, id := range mirrorsIDs { today := time.Now().UTC().Format("2006_01_02") rconn.Send("HGET", "STATS_MIRROR_"+today, id) rconn.Send("HGET", "STATS_MIRROR_BYTES_"+today, id) } stats, err := redis.Values(rconn.Do("EXEC")) if err != nil { http.Error(w, "Cannot fetch stats", http.StatusInternalServerError) return } var hasTZAdjustement bool var maxdownloads int64 var maxbytes int64 var results []MirrorStats var index int64 mlist := make([]mirrors.Mirror, 0, len(mirrorsIDs)) for _, id := range mirrorsIDs { mirror, err := h.cache.GetMirror(id) if err != nil { continue } mlist = append(mlist, mirror) var downloads int64 if v, _ := redis.String(stats[index], nil); v != "" { downloads, _ = strconv.ParseInt(v, 10, 64) } var bytes int64 if v, _ := redis.String(stats[index+1], nil); v != "" { bytes, _ = strconv.ParseInt(v, 10, 64) } if downloads > maxdownloads { maxdownloads = downloads } if bytes > maxbytes { maxbytes = bytes } var lastModTime time.Time if !mirror.LastModTime.IsZero() { lastModTime = mirror.LastModTime.Time } elapsed := time.Since(lastModTime) tzoffset, _ := time.ParseDuration(fmt.Sprintf("%dms", mirror.TZOffset)) if tzoffset != 0 { hasTZAdjustement = true } s := MirrorStats{ ID: id, Name: mirror.Name, Downloads: downloads, Bytes: bytes, SyncOffset: SyncOffset{ Valid: !lastModTime.IsZero(), Value: int(elapsed.Hours()), HumanReadable: utils.FuzzyTimeStr(elapsed), }, TZOffset: tzoffset, } results = append(results, s) index += 2 } sort.Sort(byDownloadNumbers{results}) for i := 0; i < len(results); i++ { results[i].PercentD = float32(results[i].Downloads) * 100 / float32(maxdownloads) results[i].PercentB = float32(results[i].Bytes) * 100 / float32(maxbytes) } w.Header().Set("Content-Type", "text/html; charset=utf-8") err = ctx.Templates().mirrorstats.ExecuteTemplate(w, "base", MirrorStatsPage{results, mlist, GetConfig().LocalJSPath, hasTZAdjustement}) if err != nil { log.Errorf("HTTP error: %s", err.Error()) http.Error(w, err.Error(), http.StatusInternalServerError) return } } videolabs-mirrorbits-441567e/http/http_test.go000066400000000000000000000346071523530551300214750ustar00rootroot00000000000000// Copyright (c) 2025 Arnaud Rebillout // Licensed under the MIT license package http import ( "errors" "net" "net/http" "net/http/httptest" "net/http/httputil" "os" "path" "reflect" "strings" "syscall" "testing" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/mirrors" . "github.com/etix/mirrorbits/testing" "github.com/rafaeljusto/redigomock" ) var ( fallbackURL = "http://fallback.mirror/" mirrorURL = "http://example.mirror/" testFile = "/testy.tgz" testFileSize = "48" testFileModTime = "2025-06-01 06:00:00.123456789 +0000 UTC" testFileSha256 = "1235a5b376903794b373d84ed615bb36013e70ed6aebf30b2f4823321d5182ec" testFileLastModified = "Sun, 01 Jun 2025 06:00:00 GMT" ) // Join URL and a path func urlJoinPath(url, filepath string) string { return url + strings.TrimLeft(filepath, "/") } // Create an empty file within a directory, fail if it already exists func makeEmptyFile(dir, filename string) error { filePath := path.Join(dir, filename) fileFlags := os.O_CREATE|os.O_EXCL|os.O_WRONLY f, err := os.OpenFile(filePath, fileFlags, 0644) if err != nil { return err } return f.Close() } // Make a request func makeRequest(method, url string, headers map[string]string) *http.Request { req := httptest.NewRequest(method, url, nil) for k, v := range headers { req.Header.Set(k, v) } return req } // Make a response, as returned by mirrorbits func makeResponse(code int, headers map[string]string) *http.Response { var resp http.Response switch code { case 302: resp = http.Response{ Status: "302 Found", StatusCode: 302, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: http.Header{ "Cache-Control": {"private, no-cache"}, "Content-Type": {"text/html; charset=utf-8"}, "Server": {"Mirrorbits/"+core.VERSION}, }, ContentLength: -1, } case 304: resp = http.Response{ Status: "304 Not Modified", StatusCode: 304, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: http.Header{ "Server": {"Mirrorbits/"+core.VERSION}, }, ContentLength: -1, } case 403: resp = http.Response{ Status: "403 Forbidden", StatusCode: 403, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: http.Header{ "Content-Type": {"text/plain; charset=utf-8"}, "Server": {"Mirrorbits/"+core.VERSION}, "X-Content-Type-Options": {"nosniff"}, }, ContentLength: -1, } case 404: resp = http.Response{ Status: "404 Not Found", StatusCode: 404, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: http.Header{ "Content-Type": {"text/plain; charset=utf-8"}, "Server": {"Mirrorbits/"+core.VERSION}, "X-Content-Type-Options": {"nosniff"}, }, ContentLength: -1, } default: resp = http.Response{} } for k, v := range headers { resp.Header.Set(k, v) } return &resp } // Do a request and return the response func doRequest(h *HTTP, method string, url string, headers map[string]string) (*http.Response) { req := makeRequest(method, url, headers) recorder := httptest.NewRecorder() // Note: requestDispatcher calls mirrorHandler h.requestDispatcher(recorder, req) return recorder.Result() } // Check if two http.Response are equal, excluding the body func respEqual(r1 *http.Response, r2 *http.Response) bool { r1Body, r2Body := r1.Body, r2.Body r1.Body, r2.Body = nil, nil res := reflect.DeepEqual(r1, r2) r1.Body, r2.Body = r1Body, r2Body return res } // Dump a response, excluding the body, no error checking func dump(resp *http.Response) string { dump, _ := httputil.DumpResponse(resp, false) return string(dump) } // Return the following error: // dial tcp 127.0.0.1:6379: connect: connection refused func connectionRefusedError() error { ip := net.ParseIP("127.0.0.1") tcpAddr := &net.TCPAddr{IP: ip, Port: 6379} var addr net.Addr = tcpAddr syscallErr := os.NewSyscallError("connect", syscall.ECONNREFUSED) return &net.OpError{Op: "dial", Net: "tcp", Addr: addr, Err: syscallErr} } // Return the following error: // LOADING Redis is loading the dataset in memory func redisIsLoadingError() error { return errors.New("LOADING Redis is loading the dataset in memory") } // A pair consisting of a redis command, and its expected result type mockedCmd struct { Cmd []string Res any } // Register a list of mocked redis commands func mockCommands(mock *redigomock.Conn, commands []mockedCmd) { for _, item := range commands { // Craft arguments for mock.Command, then mock args := []any{} for _, arg := range item.Cmd[1:] { args = append(args, arg) } cmd := mock.Command(item.Cmd[0], args...) // Add an expectation switch item.Res.(type) { case error: cmd.ExpectError(item.Res.(error)) case []string: cmd.ExpectStringSlice(item.Res.([]string)...) case map[string]string: cmd.ExpectMap(item.Res.(map[string]string)) default: // unknown type? that's a programming error } } } // Wrapper around redigomock.ExpectationsWereMet() to return a slice of errors func getMockErrors(mock *redigomock.Conn) (result []error) { err := mock.ExpectationsWereMet() if err != nil { lines := strings.Split(err.Error(), "\n") for _, line := range lines { line := strings.TrimSpace(line) if line == "" { continue } // A PING command might or might not have been sent during // the tests (due to `ConnectPubsub()` I believe). Since we // don't mock it, we must filter it out from the errors. if strings.HasPrefix(line, "command PING ") && strings.HasSuffix(line, " not registered in redigomock library") { continue } result = append(result, errors.New(line)) } } return } // Context for a test type testContext struct { TestDir string RepoDir string MockedConn *redigomock.Conn MirrorCache *mirrors.Cache Server *HTTP } // Prepare a test, return the context func prepareTest(t *testing.T, filenames []string) (testContext, error) { // Create a temporary directory for test data testDir := t.TempDir() // Create the repo directory, along with dummy files repoDir := testDir + "/repo" err := os.Mkdir(repoDir, 0755) if err != nil { return testContext{}, err } for _, f := range filenames { err = makeEmptyFile(repoDir, f) if err != nil { return testContext{}, err } } // Create the templates directory, along with dummy templates templatesDir := testDir + "/templates" err = os.Mkdir(templatesDir, 0755) if err != nil { return testContext{}, err } templates := []string{"base.html", "mirrorlist.html", "mirrorstats.html"} for _, f := range templates { err = makeEmptyFile(templatesDir, f) if err != nil { return testContext{}, err } } // Set mirrorbits configuration SetConfiguration(&Configuration{ Repository: repoDir, Templates: templatesDir, OutputMode: "redirect", MaxLinkHeaders: 5, Fallbacks: []Fallback{ {URL: fallbackURL}, }, }) // Reset the default server before each test. Must be done before // creating a HTTPServer instance, otherwise we run into: // // panic: http: multiple registrations for / // // Cf. https://stackoverflow.com/a/40790728/ http.DefaultServeMux = new(http.ServeMux) // Setup HTTP server mock, conn := PrepareRedisTest() conn.ConnectPubsub() cache := mirrors.NewCache(conn) h := HTTPServer(conn, cache) // Ready for testing! return testContext { TestDir: testDir, RepoDir: repoDir, MockedConn: mock, MirrorCache: cache, Server: h, }, nil } // Test 4xx return codes from MirrorHandler. // // Those HTTP codes are triggered when the file requested doesn't even exist in // the local repo. Mirrorbits doesn't query the database in those cases, so // there's no need to mock redis commands. func TestMirrorHandler4xx(t *testing.T) { // Prepare ctx, err := prepareTest(t, []string{}) if err != nil { t.Fatal(err) } noHeader := map[string]string{} // Request a file that doesn't exist on the local repo // -> return 404 "Not Found" resp := doRequest(ctx.Server, "GET", "/foobar", noHeader) want := makeResponse(404, noHeader) if !respEqual(want, resp) { t.Fatalf("Expected: %v, got: %v", want, resp) } // Request a file outside of the local repo // -> return 403 "Forbidden" resp = doRequest(ctx.Server, "GET", "/../foobar", noHeader) want = makeResponse(403, noHeader) if !respEqual(want, resp) { t.Fatalf("Expected: %v, got: %v", want, resp) } // Request a file while the repo directory doesn't even exist // -> return 404 "Not Found" if err = os.Remove(ctx.RepoDir); err != nil { t.Fatal(err) } resp = doRequest(ctx.Server, "GET", "/foobar", noHeader) want = makeResponse(404, noHeader) if !respEqual(want, resp) { t.Fatalf("Expected: %v, got: %v", want, resp) } } var mockedCmds302Fallback = [][]mockedCmd{ // Database is unreachable (redis error "connection refused") { { Cmd: []string{"HMGET", "FILE_"+testFile, "size", "modTime", "sha1", "sha256", "md5"}, Res: connectionRefusedError(), }, }, // Database is loading { { Cmd: []string{"HMGET", "FILE_"+testFile, "size", "modTime", "sha1", "sha256", "md5"}, Res: redisIsLoadingError(), }, }, // Database is reachable. File exists in the local repo, but is not // found in the database (in real-life, it means that the local repo // was updated with new files, but mirrorbits didn't rescan it yet) { { Cmd: []string{"HMGET", "FILE_"+testFile, "size", "modTime", "sha1", "sha256", "md5"}, Res: []string{"", "", "", "", "", ""}, }, }, // Database is reachable, file exists in the local repo, and is also // present in the database, however no mirror have this file yet { { Cmd: []string{"HMGET", "FILE_"+testFile, "size", "modTime", "sha1", "sha256", "md5"}, Res: []string{testFileSize, testFileModTime, "", testFileSha256, ""}, }, { Cmd: []string{"SMEMBERS", "FILEMIRRORS_"+testFile}, Res: []string{}, }, }, } var mockedCmds302Mirror = [][]mockedCmd{ // Database is reachable, file exists in the local repo, is also // present in the database, and is found on a mirror. // // Note: At startup, mirrorbits says "Can't load the GeoIP databases, // all requests will be served by the fallback mirrors". Well it // doesn't seem to be true, as this test case shows. { { Cmd: []string{"HMGET", "FILE_"+testFile, "size", "modTime", "sha1", "sha256", "md5"}, Res: []string{testFileSize, testFileModTime, "", testFileSha256, ""}, }, { Cmd: []string{"SMEMBERS", "FILEMIRRORS_"+testFile}, Res: []string{"42"}, }, { Cmd: []string{"HGETALL", "MIRROR_42"}, Res: map[string]string{ "ID": "42", "http": mirrorURL, "enabled": "true", "httpUp": "true", }, }, { Cmd: []string{"HMGET", "FILEINFO_42_"+testFile, "size", "modTime", "sha1", "sha256", "md5"}, Res: []string{testFileSize, testFileModTime, "", "", ""}, }, }, } var mockedCmds304 = [][]mockedCmd{ // File exists in the database, and is older than the If-Modified-Since // request header, so mirrorbits returns early and doesn't even check // if mirrors have the file. { { Cmd: []string{"HMGET", "FILE_"+testFile, "size", "modTime", "sha1", "sha256", "md5"}, Res: []string{testFileSize, testFileModTime, "", testFileSha256, ""}, }, }, } // Test 3xx status codes. // // Mocking redis can be tricky. If we forget to mock a command, we'll get an // error of the type: // // command [...] not registered in redigomock library // // However a redis error makes mirrorbits bail out early from mirror selection, // and in turns it triggers a fallback redirection. So from the outside, all we // know is that yes, mirrorbits returned a fallback redirect, and maybe that's // what we expect, so the test pass, but in fact it passed _because_ we forgot // to mock a redis command! // // That's why it's not enough to just check if mocked commands were called, we // also need to make sure that redigomock didn't return any error that were // unexpected. func TestMirrorHandler3xx(t *testing.T) { // Prepare ctx, err := prepareTest(t, []string{testFile}) if err != nil { t.Fatal(err) } // Define tests tests := map[string]struct { MockedCommands [][]mockedCmd RequestHeaders map[string]string Response *http.Response } { // Test various scenarios that lead to a fallback redirection "fallback_redirect": { MockedCommands: mockedCmds302Fallback, Response: makeResponse(302, map[string]string{ "Location": urlJoinPath(fallbackURL, testFile), }), }, // Same as above, but this time passing a If-Modified-Since // header, set to an old date, so no consequence on the result "fallback_redirect_old_if_modified_since": { MockedCommands: mockedCmds302Fallback, RequestHeaders: map[string]string{ "If-Modified-Since": "Tue, 01 Jun 1999 00:00:00 GMT", }, Response: makeResponse(302, map[string]string{ "Location": urlJoinPath(fallbackURL, testFile), }), }, // Test mirror redirection "mirror_redirect": { MockedCommands: mockedCmds302Mirror, Response: makeResponse(302, map[string]string{ "Location": urlJoinPath(mirrorURL, testFile), }), }, // Same as above, but with a old If-Modified-Since "mirror_redirect_old_if_modified_since": { MockedCommands: mockedCmds302Mirror, RequestHeaders: map[string]string{ "If-Modified-Since": "Tue, 01 Jun 1999 00:00:00 GMT", }, Response: makeResponse(302, map[string]string{ "Location": urlJoinPath(mirrorURL, testFile), }), }, // Test "304 Not Modified" by setting a If-Modified-Since header // that is newer that the test file modification time "not_modified": { MockedCommands: mockedCmds304, RequestHeaders: map[string]string{ "If-Modified-Since": "Wed, 04 Jun 2025 02:12:35 GMT", }, Response: makeResponse(304, map[string]string{ "Last-Modified": testFileLastModified, }), }, } // Run tests for name, tt := range tests { t.Run(name, func(t *testing.T) { for i, commands := range tt.MockedCommands { // Register mocked commands mockCommands(ctx.MockedConn, commands) // Request the file resp := doRequest(ctx.Server, "GET", testFile, tt.RequestHeaders) // Check that mocking went fine for _, err := range getMockErrors(ctx.MockedConn) { t.Errorf("#%d: %s", i, err) } // Check that response is as expected if !respEqual(tt.Response, resp) { //t.Errorf("#%d: Expected: %v, got: %v", i, tt.Response, resp) t.Errorf("#%d: Expected:\n%sGot:\n%s", i, dump(tt.Response), dump(resp)) } // Cleanup ctx.MockedConn.Clear() ctx.MirrorCache.Clear() } }) } } videolabs-mirrorbits-441567e/http/ifmodifiedsince.go000066400000000000000000000065321523530551300225740ustar00rootroot00000000000000// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found below. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Note: The source code below was picked from go/src/net/http/fs.go package http import ( "net/http" "time" ) // condResult is the result of an HTTP request precondition check. // See https://tools.ietf.org/html/rfc7232 section 3. type condResult int const ( condNone condResult = iota condTrue condFalse ) func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { if r.Method != "GET" && r.Method != "HEAD" { return condNone } ims := r.Header.Get("If-Modified-Since") if ims == "" || isZeroTime(modtime) { return condNone } t, err := http.ParseTime(ims) if err != nil { return condNone } // The Last-Modified header truncates sub-second precision so // the modtime needs to be truncated too. modtime = modtime.Truncate(time.Second) if modtime.Before(t) || modtime.Equal(t) { return condFalse } return condTrue } var unixEpochTime = time.Unix(0, 0) // isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). func isZeroTime(t time.Time) bool { return t.IsZero() || t.Equal(unixEpochTime) } func setLastModified(w http.ResponseWriter, modtime time.Time) { if !isZeroTime(modtime) { w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) } } func writeNotModified(w http.ResponseWriter) { // RFC 7232 section 4.1: // a sender SHOULD NOT generate representation metadata other than the // above listed fields unless said metadata exists for the purpose of // guiding cache updates (e.g., Last-Modified might be useful if the // response does not have an ETag field). h := w.Header() delete(h, "Content-Type") delete(h, "Content-Length") delete(h, "Content-Encoding") if h.Get("Etag") != "" { delete(h, "Last-Modified") } w.WriteHeader(http.StatusNotModified) } videolabs-mirrorbits-441567e/http/pagerenderer.go000066400000000000000000000077671523530551300221310ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package http import ( "bytes" "encoding/json" "errors" "fmt" "net/http" "sort" "strconv" "strings" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/mirrors" ) var ( // ErrTemplatesNotFound is returned cannot be loaded ErrTemplatesNotFound = errors.New("please set a valid path to the templates directory") ) // resultsRenderer is the interface for all result renderers type resultsRenderer interface { Write(ctx *Context, results *mirrors.Results) (int, error) Type() string } // JSONRenderer is used to render JSON formatted details about the current request type JSONRenderer struct{} // Type returns the type of renderer func (w *JSONRenderer) Type() string { return "JSON" } // Write is used to write the result to the ResponseWriter func (w *JSONRenderer) Write(ctx *Context, results *mirrors.Results) (statusCode int, err error) { if ctx.IsPretty() { output, err := json.MarshalIndent(results, "", " ") if err != nil { return http.StatusInternalServerError, err } ctx.ResponseWriter().Header().Set("Content-Type", "application/json; charset=utf-8") ctx.ResponseWriter().Header().Set("Content-Length", strconv.Itoa(len(output))) ctx.ResponseWriter().Write(output) } else { ctx.ResponseWriter().Header().Set("Content-Type", "application/json; charset=utf-8") err = json.NewEncoder(ctx.ResponseWriter()).Encode(results) if err != nil { return http.StatusInternalServerError, err } } return http.StatusOK, nil } // RedirectRenderer is a basic renderer that redirects the user to the first mirror in the list type RedirectRenderer struct{} // Type returns the type of renderer func (w *RedirectRenderer) Type() string { return "REDIRECT" } // Write is used to write the result to the ResponseWriter func (w *RedirectRenderer) Write(ctx *Context, results *mirrors.Results) (statusCode int, err error) { if len(results.MirrorList) > 0 { ctx.ResponseWriter().Header().Set("Content-Type", "text/html; charset=utf-8") path := strings.TrimPrefix(results.FileInfo.Path, "/") mh := len(results.MirrorList) maxheaders := GetConfig().MaxLinkHeaders if mh > maxheaders+1 { mh = maxheaders + 1 } if mh >= 1 { // Generate the header alternative links for i, m := range results.MirrorList[1:mh] { var countryCode string if len(m.CountryFields) > 0 { countryCode = strings.ToLower(m.CountryFields[0]) } ctx.ResponseWriter().Header().Add("Link", fmt.Sprintf("<%s>; rel=duplicate; pri=%d; geo=%s", m.AbsoluteURL+path, i+1, countryCode)) } } // Finally issue the redirect http.Redirect(ctx.ResponseWriter(), ctx.Request(), results.MirrorList[0].AbsoluteURL+path, http.StatusFound) return http.StatusFound, nil } // No mirror returned for this request http.NotFound(ctx.ResponseWriter(), ctx.Request()) return http.StatusNotFound, nil } // MirrorListRenderer is used to render the mirrorlist page using the HTML templates type MirrorListRenderer struct{} // Type returns the type of renderer func (w *MirrorListRenderer) Type() string { return "MIRRORLIST" } // Write is used to write the result to the ResponseWriter func (w *MirrorListRenderer) Write(ctx *Context, results *mirrors.Results) (statusCode int, err error) { if ctx.Templates().mirrorlist == nil { // No templates found for the mirrorlist return http.StatusInternalServerError, ErrTemplatesNotFound } // Sort the exclude reasons by message so they appear grouped sort.Sort(mirrors.ByExcludeReason{Mirrors: results.ExcludedList}) // Create a temporary output buffer to render the page var buf bytes.Buffer ctx.ResponseWriter().Header().Set("Content-Type", "text/html; charset=utf-8") // Render the page into the buffer err = ctx.Templates().mirrorlist.ExecuteTemplate(&buf, "base", results) if err != nil { // Something went wrong, discard the buffer return http.StatusInternalServerError, err } // Write the buffer to the socket buf.WriteTo(ctx.ResponseWriter()) return http.StatusOK, nil } videolabs-mirrorbits-441567e/http/selection.go000066400000000000000000000243641523530551300214430ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package http import ( "errors" "fmt" "math" "math/rand" "sort" "strings" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/network" "github.com/etix/mirrorbits/utils" ) var ( ErrInvalidFileInfo = errors.New("Invalid file info (modtime is zero)") ) type mirrorSelection interface { // Selection must return an ordered list of selected mirror, // a list of rejected mirrors and and an error code. Selection(*Context, *mirrors.Cache, *filesystem.FileInfo, network.GeoIPRecord) (mirrors.Mirrors, mirrors.Mirrors, error) } // DefaultEngine is the default algorithm used for mirror selection type DefaultEngine struct{} // Selection returns an ordered list of selected mirror, a list of rejected mirrors and and an error code func (h DefaultEngine) Selection(ctx *Context, cache *mirrors.Cache, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord) (mlist mirrors.Mirrors, excluded mirrors.Mirrors, err error) { // Bail out early if we don't have valid file details if fileInfo.ModTime.IsZero() { err = ErrInvalidFileInfo return } // Prepare and return the list of all potential mirrors mlist, err = cache.GetMirrors(fileInfo.Path, clientInfo) if err != nil { return } // Filter the list of mirrors mlist, excluded, closestMirror, farthestMirror := Filter(mlist, ctx.SecureOption(), fileInfo, clientInfo) if !clientInfo.IsValid() { // Shuffle the list //XXX Should we use the fallbacks instead? for i := range mlist { j := rand.Intn(i + 1) mlist[i], mlist[j] = mlist[j], mlist[i] } // Shortcut if !ctx.IsMirrorlist() { // Reduce the number of mirrors to process mlist = mlist[:utils.Min(5, len(mlist))] } return } // We're not interested in divisions by zero if closestMirror == 0 { closestMirror = math.SmallestNonzeroFloat32 } /* Weight distribution for random selection [Probabilistic weight] */ // Compute score for each mirror and return the mirrors eligible for weight distribution. // This includes: // - mirrors found in a 1.5x (configurable) range from the closest mirror // - mirrors targeting the given country (as primary or secondary) // - mirrors being in the same AS number totalScore := 0 baseScore := int(farthestMirror) weights := map[int]int{} for i := 0; i < len(mlist); i++ { m := &mlist[i] if m.Distance > closestMirror*GetConfig().WeightDistributionRange { log.Debugf("[%s] ignored: too far (%.1f; maximum: %.1f)", m.Name, m.Distance, closestMirror*GetConfig().WeightDistributionRange) m.ComputedScore = 0 continue } if GetConfig().GeographicalSort { m.ComputedScore = baseScore - int(m.Distance) + 1 if m.Distance <= closestMirror*GetConfig().WeightDistributionRange { score := (float32(baseScore) - m.Distance) if !network.IsPrimaryCountry(clientInfo, m.CountryFields) { score /= 2 m.ComputedScore += int(score) } else if network.IsPrimaryCountry(clientInfo, m.CountryFields) { m.ComputedScore += int(float32(baseScore) - (m.Distance * 5)) } else if network.IsAdditionalCountry(clientInfo, m.CountryFields) { m.ComputedScore += int(float32(baseScore) - closestMirror) } if m.Asnum == clientInfo.ASNum { m.ComputedScore += baseScore / 2 } floatingScore := float64(m.ComputedScore) + (float64(m.ComputedScore) * (float64(m.Score) / 100)) + 0.5 // The minimum allowed score is 1 m.ComputedScore = int(math.Max(floatingScore, 1)) if m.ComputedScore > baseScore { // The weight must always be > 0 to not break the randomization below totalScore += m.ComputedScore - baseScore weights[m.ID] = m.ComputedScore - baseScore } } } else { // GetConfig().GeographicalSort == false if m.Score <= 0 { m.ComputedScore = 0 } else { totalScore += m.Score m.Weight = float32(m.Score) m.ComputedScore = m.Score weights[m.ID] = m.Score } } } // Get the final number of mirrors selected for weight distribution selected := len(weights) // Sort mirrors by computed score sort.Sort(mirrors.ByComputedScore{Mirrors: mlist}) if selected > 1 { if ctx.IsMirrorlist() { // Don't reorder the results, just set the percentage for i := 0; i < selected; i++ { id := mlist[i].ID for j := 0; j < len(mlist); j++ { if mlist[j].ID == id { mlist[j].Weight = float32(float64(weights[id]) * 100 / float64(totalScore)) break } } } } else { // Randomize the order of the selected mirrors considering their weights weightedMirrors := make([]mirrors.Mirror, selected) rest := totalScore for i := 0; i < selected; i++ { var id int rv := rand.Int31n(int32(rest)) s := 0 for k, v := range weights { s += v if int32(s) > rv { id = k break } } for _, m := range mlist { if m.ID == id { m.Weight = float32(float64(weights[id]) * 100 / float64(totalScore)) weightedMirrors[i] = m break } } rest -= weights[id] delete(weights, id) } // Replace the head of the list by its reordered counterpart mlist = append(weightedMirrors, mlist[selected:]...) // Reduce the number of mirrors to return v := math.Min(math.Min(5, float64(selected)), float64(len(mlist))) mlist = mlist[:int(v)] } } else if selected == 1 && len(mlist) > 0 { mlist[0].Weight = 100 } return } // Filter mirror list, return the list of mirrors candidates for redirection, // and the list of mirrors that were excluded. Also return the distance of the // closest and farthest mirrors. func Filter(mlist mirrors.Mirrors, secureOption SecureOption, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord) (accepted mirrors.Mirrors, excluded mirrors.Mirrors, closestMirror float32, farthestMirror float32) { // Check if this file is allowed to be outdated checkSize := true maxOutdated := time.Duration(0) config := GetConfig().AllowOutdatedFiles for _, c := range config { if strings.HasPrefix(fileInfo.Path, c.Prefix) { checkSize = false maxOutdated = time.Duration(c.Minutes) * time.Minute break } } accepted = make([]mirrors.Mirror, 0, len(mlist)) excluded = make([]mirrors.Mirror, 0, len(mlist)) for _, m := range mlist { // Is it enabled? if !m.Enabled { m.ExcludeReason = "Disabled" goto discard } // Is the procol requested supported by the mirror? // Is the mirror up for this protocol? switch secureOption { case WITHTLS: // HTTPS explicitly requested m.AbsoluteURL = ensureAbsolute(m.HttpURL, "https") httpsSupported := !strings.HasPrefix(m.HttpURL, "http://") if !httpsSupported { m.ExcludeReason = "Not HTTPS" } else if !m.HttpsUp { m.ExcludeReason = either(m.HttpsDownReason, "Down") } else { break } goto discard case WITHOUTTLS: // HTTP explicitly requested m.AbsoluteURL = ensureAbsolute(m.HttpURL, "http") httpSupported := !strings.HasPrefix(m.HttpURL, "https://") if !httpSupported { m.ExcludeReason = "Not HTTP" } else if !m.HttpUp { m.ExcludeReason = either(m.HttpDownReason, "Down") } else { break } goto discard default: // Any protocol will do - favor HTTPS if avail var httpReason, httpsReason string m.AbsoluteURL = ensureAbsolute(m.HttpURL, "https") httpsSupported := !strings.HasPrefix(m.HttpURL, "http://") if !httpsSupported { httpsReason = "Not HTTPS" } else if !m.HttpsUp { httpsReason = either(m.HttpsDownReason, "Down") } else { break } m.AbsoluteURL = ensureAbsolute(m.HttpURL, "http") httpSupported := !strings.HasPrefix(m.HttpURL, "https://") if !httpSupported { httpReason = "Not HTTP" } else if !m.HttpUp { httpReason = either(m.HttpDownReason, "Down") } else { break } if httpReason == httpsReason { m.ExcludeReason = httpReason } else { m.ExcludeReason = httpReason + " / " + httpsReason } goto discard } // Is it the same size / modtime as source? if m.FileInfo != nil { if checkSize && m.FileInfo.Size != fileInfo.Size { m.ExcludeReason = "File size mismatch" goto discard } if !m.FileInfo.ModTime.IsZero() { mModTime := m.FileInfo.ModTime if GetConfig().FixTimezoneOffsets { offset := time.Duration(m.TZOffset) * time.Millisecond mModTime = mModTime.Add(offset) } precision := m.LastSuccessfulSyncPrecision.Duration() mModTime = mModTime.Truncate(precision) lModTime := fileInfo.ModTime.Truncate(precision) delta := lModTime.Sub(mModTime) if delta < 0 || delta > maxOutdated { m.ExcludeReason = fmt.Sprintf("Mod time mismatch (diff: %s)", delta) goto discard } } } // Is it configured to serve its continent only? if m.ContinentOnly { if !clientInfo.IsValid() || clientInfo.ContinentCode != m.ContinentCode { m.ExcludeReason = "Continent only" goto discard } } // Is it configured to serve its country only? if m.CountryOnly { if !clientInfo.IsValid() || !utils.IsInSlice(clientInfo.CountryCode, m.CountryFields) { m.ExcludeReason = "Country only" goto discard } } // Is it in the same AS number? if m.ASOnly { if !clientInfo.IsValid() || clientInfo.ASNum != m.Asnum { m.ExcludeReason = "AS only" goto discard } } // Is the user's country code allowed on this mirror? if clientInfo.IsValid() && utils.IsInSlice(clientInfo.CountryCode, m.ExcludedCountryFields) { m.ExcludeReason = "User's country restriction" goto discard } // Keep track of the closest and farthest mirrors if len(accepted) == 0 { closestMirror = m.Distance } else if m.Distance < closestMirror { closestMirror = m.Distance } if m.Distance > farthestMirror { farthestMirror = m.Distance } accepted = append(accepted, m) continue discard: excluded = append(excluded, m) } return } // ensureAbsolute returns the url 'as is' if it's absolute (ie. it starts with // a scheme), otherwise it prepends '://' and returns the result. func ensureAbsolute(url string, scheme string) string { if utils.HasAnyPrefix(url, "http://", "https://") { return url } return scheme + "://" + url } // either returns s if it's not empty, d otherwise func either(s string, d string) string { if s != "" { return s } return d } videolabs-mirrorbits-441567e/http/selection_test.go000066400000000000000000000275261523530551300225050ustar00rootroot00000000000000// Copyright (c) 2024 Arnaud Rebillout // Licensed under the MIT license package http import ( "fmt" "testing" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/network" "github.com/etix/mirrorbits/mirrors" ) var noFileInfo *filesystem.FileInfo var noClientInfo network.GeoIPRecord func TestMain(m *testing.M) { noFileInfo = nil noClientInfo = network.GeoIPRecord{} SetConfiguration(&Configuration{ FixTimezoneOffsets: false, }) m.Run() } // Helper to check the results of the Filter function on a single mirror func checkResultsSingle(t *testing.T, a mirrors.Mirrors, x mirrors.Mirrors, reason string, url string) { t.Helper() var m mirrors.Mirror // If reason was set, we expect the mirror to have been excluded, while // no reason means that the mirror should have been accepted. if reason == "" { if len(a) != 1 || len(x) != 0 { t.Fatalf("There should be 1 mirror accepted and 0 mirror excluded") } m = a[0] } else { if len(a) != 0 || len(x) != 1 { t.Fatalf("There should be 0 mirror accepted and 1 mirror excluded") } m = x[0] } // Test that the field ExcludeReason was set as expected, or is unset // in case the mirror was accepted. if m.ExcludeReason != reason { t.Fatalf("Invalid ExcludeReason, expected '%s', got '%s'", reason, m.ExcludeReason) } // The field AbsoluteURL is expected to be set, except in the case when // the mirror is disabled. if m.ExcludeReason != "Disabled" && m.AbsoluteURL != url { t.Fatalf("Invalid AbsoluteURL, expected '%s', got '%s'", url, m.AbsoluteURL) } } // Helper to test the Filter function on a single mirror func testFilterSingle(t *testing.T, m mirrors.Mirror, secureOption SecureOption, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord, reason string) { t.Helper() mlist := mirrors.Mirrors{m} a, x, _, _ := Filter(mlist, secureOption, fileInfo, clientInfo) checkResultsSingle(t, a, x, reason, m.HttpURL) } // Helper to test the Filter function on a single mirror, with a specific AbsoluteURL func testFilterSingleAbsoluteURL(t *testing.T, m mirrors.Mirror, secureOption SecureOption, fileInfo *filesystem.FileInfo, clientInfo network.GeoIPRecord, reason string, url string) { t.Helper() mlist := mirrors.Mirrors{m} a, x, _, _ := Filter(mlist, secureOption, fileInfo, clientInfo) checkResultsSingle(t, a, x, reason, url) } func TestFilter(t *testing.T) { // Test that a mirror that is disabled is rejected m1 := mirrors.Mirror{ HttpURL: "http://m1.mirror", } t.Run("disabled", func(t *testing.T) { testFilterSingle(t, m1, UNDEFINED, noFileInfo, noClientInfo, "Disabled") }) // Given that a mirror is enabled, test that it's rejected when the // requested protocol is not available (either it's not supported by // the mirror, or it's down). tests1 := map[string]struct { secureOption SecureOption mirrorURL string excludeReason string absoluteURL string } { "want_https_but_http_only": { secureOption: WITHTLS, mirrorURL: "http://m1.mirror", excludeReason: "Not HTTPS", absoluteURL: "http://m1.mirror", }, "want_https_has_https_but_down": { secureOption: WITHTLS, mirrorURL: "https://m1.mirror", excludeReason: "Down", absoluteURL: "https://m1.mirror", }, "want_https_has_any_but_down": { secureOption: WITHTLS, mirrorURL: "m1.mirror", excludeReason: "Down", absoluteURL: "https://m1.mirror", }, "want_http_but_https_only": { secureOption: WITHOUTTLS, mirrorURL: "https://m1.mirror", excludeReason: "Not HTTP", absoluteURL: "https://m1.mirror", }, "want_http_has_http_but_down": { secureOption: WITHOUTTLS, mirrorURL: "http://m1.mirror", excludeReason: "Down", absoluteURL: "http://m1.mirror", }, "want_http_has_any_but_down": { secureOption: WITHOUTTLS, mirrorURL: "m1.mirror", excludeReason: "Down", absoluteURL: "http://m1.mirror", }, "want_any_has_http_only_but_down": { secureOption: UNDEFINED, mirrorURL: "http://m1.mirror", excludeReason: "Down / Not HTTPS", absoluteURL: "http://m1.mirror", }, "want_any_has_https_only_but_down": { secureOption: UNDEFINED, mirrorURL: "https://m1.mirror", excludeReason: "Not HTTP / Down", absoluteURL: "https://m1.mirror", }, "want_any_has_any_but_down": { secureOption: UNDEFINED, mirrorURL: "m1.mirror", excludeReason: "Down", absoluteURL: "http://m1.mirror", }, } for name, test := range tests1 { m1 := mirrors.Mirror{ HttpURL: test.mirrorURL, Enabled: true, } t.Run(name, func(t *testing.T) { testFilterSingleAbsoluteURL(t, m1, test.secureOption, noFileInfo, noClientInfo, test.excludeReason, test.absoluteURL) }) } // Given that a mirror is enabled and the protocol requested is available, // test that a mirror is rejected when the requested file is not valid // (wrong size or mod time). testfile := &filesystem.FileInfo{ Path: "/test/file.tgz", Size: 43000, ModTime: time.Now(), } tests2 := map[string]struct { fileSize int64 fileModTime time.Time excludeReason string } { "wrong_size": { fileSize: 12345, fileModTime: testfile.ModTime, excludeReason: "File size mismatch", }, "wrong_mod_time_newer_on_mirror": { fileSize: testfile.Size, fileModTime: testfile.ModTime.Add(time.Second * 10), excludeReason: "Mod time mismatch (diff: -10s)", }, "wrong_mod_time_older_on_mirror": { fileSize: testfile.Size, fileModTime: testfile.ModTime.Add(time.Second * -10), excludeReason: "Mod time mismatch (diff: 10s)", }, } for name, test := range tests2 { m1 := mirrors.Mirror{ HttpURL: "https://m1.mirror", Enabled: true, HttpsUp: true, FileInfo: &filesystem.FileInfo{ Path: "/test/file.tgz", Size: test.fileSize, ModTime: test.fileModTime, }, } t.Run(name, func(t *testing.T) { testFilterSingle(t, m1, WITHTLS, testfile, noClientInfo, test.excludeReason) }) } // Given that a mirror is enabled, the protocol requested is available // and the file on the mirror is valid, test that a mirror is rejected // when the client doesn't meet the geolocation requirements. clientInfo := network.GeoIPRecord{ ContinentCode: "EU", CountryCode: "FR", ASNum: 4444, } tests3 := map[string]struct { continentOnly bool continentCode string countryOnly bool countryCodes string asOnly bool asNum uint excludedCountryCodes string excludeReason string } { "wrong_continent": { continentOnly: true, continentCode: "NA", excludeReason: "Continent only", }, "wrong_country": { countryOnly: true, countryCodes: "UK", excludeReason: "Country only", }, "wrong_countries": { countryOnly: true, countryCodes: "FI NO SE", excludeReason: "Country only", }, "wrong_as": { asOnly: true, asNum: 5555, excludeReason: "AS only", }, "excluded_country": { excludedCountryCodes: "FR", excludeReason: "User's country restriction", }, "excluded_countries": { excludedCountryCodes: "ES FR IT PT", excludeReason: "User's country restriction", }, } for name, test := range tests3 { m1 := mirrors.Mirror{ HttpURL: "https://m1.mirror", Enabled: true, HttpsUp: true, FileInfo: testfile, ContinentOnly: test.continentOnly, ContinentCode: test.continentCode, CountryOnly: test.countryOnly, CountryCodes: test.countryCodes, ASOnly: test.asOnly, Asnum: test.asNum, ExcludedCountryCodes: test.excludedCountryCodes, } m1.Prepare() t.Run(name, func(t *testing.T) { testFilterSingle(t, m1, WITHTLS, testfile, clientInfo, test.excludeReason) }) } // Given valid mirrors, test that the distances returned are correct. tests4 := map[string]struct { distances []float32 extrema []float32 } { "no_mirror": { distances: []float32{}, extrema: []float32{0, 0}, }, "one_mirror": { distances: []float32{10}, extrema: []float32{10, 10}, }, "some_mirrors": { distances: []float32{30, 20, 10}, extrema: []float32{10, 30}, }, } for name, test := range tests4 { mlist := make([]mirrors.Mirror, 0, 5) for i, d := range test.distances { m := mirrors.Mirror{ HttpURL: fmt.Sprintf("https://m%d.mirror", i), Enabled: true, HttpsUp: true, FileInfo: testfile, Distance: d, } mlist = append(mlist, m) } t.Run(name, func(t *testing.T) { a, x, closest, farthest := Filter(mlist, WITHTLS, testfile, clientInfo) if len(a) != len(mlist) || len(x) != 0 { t.Fatalf("There should be %d mirror(s) accepted and 0 mirror excluded", len(mlist)) } if closest != test.extrema[0] || farthest != test.extrema[1] { t.Fatalf("Wrong results for [closest farthest], expected %v, got %v", test.extrema, []float32{closest, farthest}) } }) } } func TestFilterAllowOutdatedFiles(t *testing.T) { // Given a file that is outdated on a mirror, test that the mirror is // rejected, unless the configuration setting AllowOutdatedFiles is set // correctly in order to accept this file. testfile := &filesystem.FileInfo{ Path: "/test/file.tgz", Size: 43000, ModTime: time.Now(), } configValues := [][]OutdatedFilesConfig{ []OutdatedFilesConfig{}, []OutdatedFilesConfig{{ Prefix: "/test/", Minutes: 1, }}, []OutdatedFilesConfig{{ Prefix: "/wrong/", Minutes: 2, }}, []OutdatedFilesConfig{{ Prefix: "/test/", Minutes: 2, }}, } tests := map[string]struct { fileSize int64 fileModTime time.Time excludeReason []string } { "outdated_same_size": { fileSize: testfile.Size, fileModTime: testfile.ModTime.Add(-100 * time.Second), excludeReason: []string{ "Mod time mismatch (diff: 1m40s)", "Mod time mismatch (diff: 1m40s)", "Mod time mismatch (diff: 1m40s)", "", }, }, "outdated_different_size": { fileSize: 12345, fileModTime: testfile.ModTime.Add(-100 * time.Second), excludeReason: []string{ "File size mismatch", "Mod time mismatch (diff: 1m40s)", "File size mismatch", "", }, }, } for idx, configValue := range configValues { SetConfiguration(&Configuration{ AllowOutdatedFiles: configValue, }) for name, test := range tests { m1 := mirrors.Mirror{ HttpURL: fmt.Sprintf("https://m%d.mirror", 1), Enabled: true, HttpsUp: true, FileInfo: &filesystem.FileInfo{ Path: testfile.Path, Size: test.fileSize, ModTime: test.fileModTime, }, } t.Run(name, func(t *testing.T) { testFilterSingle(t, m1, WITHTLS, testfile, noClientInfo, test.excludeReason[idx]) }) } } } func TestFilterFixTimezoneOffsets(t *testing.T) { // Given a mirror with a 1-hour timezone offset, test that the mirror // is rejected unless 1) the TZOffset of the mirror is set correctly, // and 2) the configuration setting FixTimezoneOffsets is enabled. var offset int64 = 3600 modTime := time.Now() outdatedModTime := modTime.Add(time.Duration(-offset) * time.Second) fileRequested := &filesystem.FileInfo{ Path: "/test/file.tgz", Size: 43000, ModTime: modTime, } fileOnMirror := &filesystem.FileInfo{ Path: "/test/file.tgz", Size: 43000, ModTime: outdatedModTime, } configValues := []bool{false, true} tests := map[string]struct { tzoffset int64 excludeReason []string } { "tzoffset_unset": { tzoffset: 0, excludeReason: []string{ "Mod time mismatch (diff: 1h0m0s)", "Mod time mismatch (diff: 1h0m0s)", }, }, "tzoffset_set": { tzoffset: offset * 1000, excludeReason: []string{ "Mod time mismatch (diff: 1h0m0s)", "", }, }, } for idx, configValue := range configValues { SetConfiguration(&Configuration{ FixTimezoneOffsets: configValue, }) for name, test := range tests { m1 := mirrors.Mirror{ HttpURL: fmt.Sprintf("https://m%d.mirror", 1), Enabled: true, HttpsUp: true, FileInfo: fileOnMirror, TZOffset: test.tzoffset, } t.Run(name, func(t *testing.T) { testFilterSingle(t, m1, WITHTLS, fileRequested, noClientInfo, test.excludeReason[idx]) }) } } } videolabs-mirrorbits-441567e/http/stats.go000066400000000000000000000101121523530551300205760ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package http import ( "errors" "fmt" "strconv" "strings" "sync" "time" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/mirrors" ) /* Total (all files, all mirrors): STATS_TOTAL List of hashes for a file: STATS_FILE = path -> value All time STATS_FILE_[year] = path -> value By year STATS_FILE_[year]_[month] = path -> value By month STATS_FILE_[year]_[month]_[day] = path -> value By day List of hashes for a mirror: STATS_MIRROR = mirror -> value All time STATS_MIRROR_[year] = mirror -> value By year STATS_MIRROR_[year]_[month] = mirror -> value By month STATS_MIRROR_[year]_[month]_[day] = mirror -> value By day */ var ( errEmptyFileError = errors.New("stats: file parameter is empty") errUnknownMirror = errors.New("stats: unknown mirror") ) // Stats is the internal structure for the download stats type Stats struct { r *database.Redis countChan chan countItem mapStats map[string]int64 stop chan bool wg sync.WaitGroup downgraded bool } type countItem struct { mirrorID int filepath string size int64 time time.Time } // NewStats returns an instance of the stats counter func NewStats(redis *database.Redis) *Stats { s := &Stats{ r: redis, countChan: make(chan countItem, 1000), mapStats: make(map[string]int64), stop: make(chan bool), } go s.processCountDownload() return s } // Terminate stops the stats handler and commit results to the database func (s *Stats) Terminate() { close(s.stop) log.Notice("Saving stats") s.wg.Wait() } // CountDownload is a lightweight method used to count a new download for a specific file and mirror func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo) error { if m.Name == "" { return errUnknownMirror } if fileinfo.Path == "" { return errEmptyFileError } s.countChan <- countItem{m.ID, fileinfo.Path, fileinfo.Size, time.Now().UTC()} return nil } // Process all stacked download messages func (s *Stats) processCountDownload() { s.wg.Add(1) pushTicker := time.NewTicker(500 * time.Millisecond) for { select { case <-s.stop: s.pushStats() s.wg.Done() return case c := <-s.countChan: date := c.time.Format("2006_01_02|") // Includes separator s.mapStats["f"+date+c.filepath]++ s.mapStats["m"+date+strconv.Itoa(c.mirrorID)]++ s.mapStats["s"+date+strconv.Itoa(c.mirrorID)] += c.size case <-pushTicker.C: s.pushStats() } } } // Push the resulting stats on redis func (s *Stats) pushStats() { if len(s.mapStats) <= 0 { return } rconn := s.r.Get() defer rconn.Close() if rconn.Err() != nil { if s.downgraded == false { log.Warningf("Uncommited stats kept in-memory: %v", rconn.Err()) } s.downgraded = true return } rconn.Send("MULTI") for k, v := range s.mapStats { if v == 0 { continue } separator := strings.Index(k, "|") if separator <= 0 { log.Critical("Stats: separator not found") continue } typ := k[:1] date := k[1:separator] object := k[separator+1:] if typ == "f" { // File fkey := fmt.Sprintf("STATS_FILE_%s", date) for i := 0; i < 4; i++ { rconn.Send("HINCRBY", fkey, object, v) fkey = fkey[:strings.LastIndex(fkey, "_")] } // Increase the total too rconn.Send("INCRBY", "STATS_TOTAL", v) } else if typ == "m" { // Mirror mkey := fmt.Sprintf("STATS_MIRROR_%s", date) for i := 0; i < 4; i++ { rconn.Send("HINCRBY", mkey, object, v) mkey = mkey[:strings.LastIndex(mkey, "_")] } } else if typ == "s" { // Bytes mkey := fmt.Sprintf("STATS_MIRROR_BYTES_%s", date) for i := 0; i < 4; i++ { rconn.Send("HINCRBY", mkey, object, v) mkey = mkey[:strings.LastIndex(mkey, "_")] } } else { log.Warning("Stats: unknown type", typ) } } _, err := rconn.Do("EXEC") if err != nil { log.Errorf("Stats: could not save stats to redis: %s", err.Error()) return } s.downgraded = false // Clear the map s.mapStats = make(map[string]int64) } videolabs-mirrorbits-441567e/logs/000077500000000000000000000000001523530551300171035ustar00rootroot00000000000000videolabs-mirrorbits-441567e/logs/logs.go000066400000000000000000000110261523530551300203760ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package logs import ( "bytes" "fmt" "io" stdlog "log" "os" "runtime" "strconv" "strings" "sync" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/mirrors" "github.com/op/go-logging" ) var ( log = logging.MustGetLogger("main") rlogger runtimeLogger dlogger downloadsLogger ) type runtimeLogger struct { f *os.File } type downloadsLogger struct { sync.RWMutex l *stdlog.Logger f io.WriteCloser } func (d *downloadsLogger) Close() { if d.f != nil { d.f.Close() d.f = nil } d.l = nil } // ReloadLogs will reopen the logs to allow rotations func ReloadLogs() { ReloadRuntimeLogs() if core.Daemon { ReloadDownloadLogs() } } func isTerminal(f *os.File) bool { stat, _ := f.Stat() if (stat.Mode() & os.ModeCharDevice) != 0 { return true } return false } // ReloadRuntimeLogs reopens the runtime logs for writing func ReloadRuntimeLogs() { if rlogger.f == os.Stderr && core.RunLog == "" { // Logger already set up and connected to the console. // Don't reload to avoid breaking journald. return } if rlogger.f != nil && rlogger.f != os.Stderr { rlogger.f.Close() } if core.RunLog != "" { var err error rlogger.f, _, err = openLogFile(core.RunLog) if err != nil { fmt.Fprintln(os.Stderr, "Cannot open log file for writing") rlogger.f = os.Stderr } } else { rlogger.f = os.Stderr } logBackend := logging.NewLogBackend(rlogger.f, "", 0) logBackend.Color = isTerminal(rlogger.f) //TODO make color optional logging.SetBackend(logBackend) if core.Debug { logging.SetFormatter(logging.MustStringFormatter("%{shortfile:-20s}%{time:2006/01/02 15:04:05.000 MST} %{message}")) logging.SetLevel(logging.DEBUG, "main") } else { logging.SetFormatter(logging.MustStringFormatter("%{time:2006/01/02 15:04:05.000 MST} %{message}")) logging.SetLevel(logging.INFO, "main") } } func openLogFile(logfile string) (*os.File, bool, error) { newfile := true s, _ := os.Stat(logfile) if s != nil && s.Size() > 0 { newfile = false } f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664) if err != nil { return nil, false, err } return f, newfile, nil } func setDownloadLogWriter(writer io.Writer, createHeader bool) { dlogger.l = stdlog.New(writer, "", stdlog.Ldate|stdlog.Lmicroseconds) if createHeader { var buf bytes.Buffer hostname, _ := os.Hostname() fmt.Fprintf(&buf, "# Log file created at: %s\n", time.Now().Format("2006/01/02 15:04:05")) fmt.Fprintf(&buf, "# Running on machine: %s\n", hostname) fmt.Fprintf(&buf, "# Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH) writer.Write(buf.Bytes()) } } // ReloadDownloadLogs reopens the download logs for writing func ReloadDownloadLogs() { dlogger.Lock() defer dlogger.Unlock() dlogger.Close() if GetConfig().LogDir == "" { return } logfile := GetConfig().LogDir + "/downloads.log" f, createHeader, err := openLogFile(logfile) if err != nil { log.Criticalf("Cannot open log file %s", logfile) return } setDownloadLogWriter(f, createHeader) } // LogDownload writes a download result to the logs func LogDownload(typ string, method string, statuscode int, p *mirrors.Results, err error) { dlogger.RLock() defer dlogger.RUnlock() if dlogger.l == nil { // Logs are disabled return } var path, ip string if p != nil { path = p.FileInfo.Path ip = p.IP } errstr := "" if err != nil { errstr = err.Error() } line := fmt.Sprintf("%s %d %s \"%s\" ip:%s", typ, statuscode, method, path, ip) if (statuscode == 302 || statuscode == 200) && p != nil && len(p.MirrorList) > 0 { var distance, countries string m := p.MirrorList[0] distance = strconv.FormatFloat(float64(m.Distance), 'f', 2, 32) countries = strings.Join(m.CountryFields, ",") fallback := "" if p.Fallback == true { fallback = " fallback:true" } sameASNum := "" if m.Asnum > 0 && m.Asnum == p.ClientInfo.ASNum { sameASNum = "same" } line += fmt.Sprintf(" mirror:%s%s %sasn:%d distance:%skm countries:%s", m.Name, fallback, sameASNum, m.Asnum, distance, countries) } else if statuscode == 404 && p != nil { // nothing to add to the log line } else if statuscode == 500 && p != nil { mirrorName := "unknown" if len(p.MirrorList) > 0 { mirrorName = p.MirrorList[0].Name } line += fmt.Sprintf(" mirror:%s error:%s", mirrorName, errstr) } else { line += fmt.Sprintf(" error:%s", errstr) } dlogger.l.Print(line) } videolabs-mirrorbits-441567e/logs/logs_test.go000066400000000000000000000171471523530551300214470ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package logs import ( "bytes" "errors" "io" "os" "reflect" "strings" "testing" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/network" "github.com/op/go-logging" ) type CloseTester struct { closed bool } func (c *CloseTester) Write(p []byte) (n int, err error) { return 0, err } func (c *CloseTester) Close() error { c.closed = true return nil } func TestDownloadsLogger_Close(t *testing.T) { f := &CloseTester{} dlogger.f = f if f.closed == true { t.Fatalf("Precondition failed") } dlogger.Close() if f.closed == false { t.Fatalf("Should be closed") } if dlogger.l != nil || dlogger.f != nil { t.Fatalf("Should be nil") } } func TestIsTerminal(t *testing.T) { stat, _ := os.Stdout.Stat() if (stat.Mode() & os.ModeCharDevice) == 0 { t.Skip("Cannot test without a valid terminal") } if !isTerminal(os.Stdout) { t.Fatalf("The current terminal is supposed to support colors") } f, err := os.CreateTemp("", "mirrorbits-tests") if err != nil { t.Errorf("Unable to create a temporary file: %s", err.Error()) return } defer os.Remove(f.Name()) if isTerminal(f) { t.Fatalf("The given file cannot be a terminal") } } func TestReloadRuntimeLogs(t *testing.T) { rlogger.f = nil ReloadRuntimeLogs() if rlogger.f == nil { t.Fatalf("The logger output must be setup") } if rlogger.f != os.Stderr { t.Fatalf("The logger output is expected to be Stderr") } if logging.GetLevel("main") != logging.INFO { t.Fatalf("Log level is supposed to be INFO by default") } ptr := reflect.ValueOf(rlogger.f).Pointer() ReloadRuntimeLogs() if reflect.ValueOf(rlogger.f).Pointer() != ptr { t.Fatalf("The logger must not be reloaded when writing on Stderr") } /* */ core.RunLog = "/" ReloadRuntimeLogs() if rlogger.f != os.Stderr { t.Fatalf("Opening an invalid file must fallback to Stderr") } /* */ f, err := os.CreateTemp("", "mirrorbits-tests") if err != nil { t.Errorf("Unable to create a temporary file: %s", err.Error()) return } defer os.Remove(f.Name()) core.RunLog = f.Name() core.Debug = true ReloadRuntimeLogs() if logging.GetLevel("main") != logging.DEBUG { t.Fatalf("Log level is supposed to be DEBUG") } if rlogger.f == os.Stderr { t.Fatalf("The output is expected to be a file, not Stderr") } /* */ testString := "Testing42" log.Error(testString) buf, _ := io.ReadAll(f) if !strings.Contains(string(buf), testString) { t.Fatalf("The log doesn't contain the string %s", testString) } /* */ core.RunLog = "" ReloadRuntimeLogs() if rlogger.f != os.Stderr { t.Fatalf("The output is expected to be Stderr") } } func TestOpenLogFile(t *testing.T) { path := t.TempDir() f, newfile, err := openLogFile(path + "/test1.log") if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if newfile == false { t.Fatalf("Expected new file") } content := []byte("It works!") n, err := f.Write(content) if err != nil { t.Fatalf("Unexpected write error: %s", err.Error()) } if n != len(content) { t.Fatalf("Invalid number of bytes written") } f.Close() /* Reopen file to check newfile */ f, newfile, err = openLogFile(path + "/test1.log") if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if newfile == true { t.Fatalf("Expected newfile to be false") } f.Close() /* Open invalid file */ f, _, err = openLogFile("") if err == nil { t.Fatalf("Error expected while opening invalid file") } f.Close() } func TestSetDownloadLogWriter(t *testing.T) { if dlogger.l != nil || dlogger.f != nil { t.Fatalf("Precondition failed") } var buf bytes.Buffer setDownloadLogWriter(&buf, true) if dlogger.l == nil { t.Fatalf("Logger not created") } if buf.Len() == 0 { t.Fatalf("Buffer empty, expected header") } if !strings.HasPrefix(buf.String(), "#") { t.Fatalf("Header doesn't starts with '#'") } buf.Reset() /* */ setDownloadLogWriter(&buf, false) if buf.Len() != 0 { t.Fatalf("Expected no content") } } func TestReloadDownloadLogs(t *testing.T) { // Not implemented because of GetConfig() // TODO need abstraction for GetConfig() } //type xResults struct { // FileInfo filesystem.FileInfo // MapURL string `json:"-"` // IP string // ClientInfo network.GeoIPRecord // MirrorList Mirrors // ExcludedList Mirrors `json:",omitempty"` // Fallback bool `json:",omitempty"` //} func TestLogDownload(t *testing.T) { var buf bytes.Buffer dlogger.Close() // The next line isn't supposed to crash. LogDownload("", "GET", 500, nil, nil) setDownloadLogWriter(&buf, true) buf.Reset() // The next few lines arent't supposed to crash. LogDownload("", "GET", 200, nil, nil) LogDownload("", "GET", 302, nil, nil) LogDownload("", "GET", 404, nil, nil) LogDownload("", "GET", 500, nil, nil) LogDownload("", "GET", 501, nil, nil) if c := strings.Count(buf.String(), "\n"); c != 5 { t.Fatalf("Invalid number of lines, got %d, expected 5", c) } buf.Reset() /* Test a log line with 200 status code */ p := &mirrors.Results{ FileInfo: filesystem.FileInfo{ Path: "/test/file.tgz", }, MirrorList: mirrors.Mirrors{ mirrors.Mirror{ ID: 1, Name: "m1", Asnum: 444, Distance: 99, CountryFields: []string{"FR", "UK", "DE"}, }, mirrors.Mirror{ ID: 2, Name: "m2", }, }, IP: "192.168.0.1", ClientInfo: network.GeoIPRecord{ ASNum: 444, }, Fallback: true, } LogDownload("JSON", "GET", 200, p, nil) expected := "JSON 200 GET \"/test/file.tgz\" ip:192.168.0.1 mirror:m1 fallback:true sameasn:444 distance:99.00km countries:FR,UK,DE\n" if !strings.HasSuffix(buf.String(), expected) { t.Fatalf("Invalid log line:\nGot:\n%#vs\nExpected:\n%#v", buf.String(), expected) } buf.Reset() /* Test a log line with 404 status code */ p = &mirrors.Results{ FileInfo: filesystem.FileInfo{ Path: "/test/file.tgz", }, IP: "192.168.0.1", } LogDownload("JSON", "GET", 404, p, nil) expected = "JSON 404 GET \"/test/file.tgz\" ip:192.168.0.1\n" if !strings.HasSuffix(buf.String(), expected) { t.Fatalf("Invalid log line:\nGot:\n%#vs\nExpected:\n%#v", buf.String(), expected) } buf.Reset() /* Test a log line with 500 status code */ p = &mirrors.Results{ MirrorList: mirrors.Mirrors{ mirrors.Mirror{ ID: 1, Name: "m1", }, mirrors.Mirror{ ID: 2, Name: "m2", }, }, } LogDownload("JSON", "GET", 500, p, errors.New("test error")) expected = "JSON 500 GET \"\" ip: mirror:m1 error:test error\n" if !strings.HasSuffix(buf.String(), expected) { t.Fatalf("Invalid log line:\nGot:\n%#vs\nExpected:\n%#v", buf.String(), expected) } buf.Reset() /* Test a log line with 501 status code */ p = &mirrors.Results{ FileInfo: filesystem.FileInfo{ Path: "/test/file.tgz", }, IP: "192.168.0.1", } LogDownload("JSON", "GET", 501, p, errors.New("test error")) expected = "JSON 501 GET \"/test/file.tgz\" ip:192.168.0.1 error:test error\n" if !strings.HasSuffix(buf.String(), expected) { t.Fatalf("Invalid log line:\nGot:\n%#vs\nExpected:\n%#v", buf.String(), expected) } buf.Reset() /* Make sure we don't trip when there's a %s (aka a "verb") in the request */ p = &mirrors.Results{ FileInfo: filesystem.FileInfo{ Path: "/test/%s/hacked", }, IP: "192.168.0.1", } LogDownload("JSON", "GET", 404, p, nil) expected = "JSON 404 GET \"/test/%s/hacked\" ip:192.168.0.1\n" if !strings.HasSuffix(buf.String(), expected) { t.Fatalf("Invalid log line:\nGot:\n%#vs\nExpected:\n%#v", buf.String(), expected) } buf.Reset() } videolabs-mirrorbits-441567e/main.go000066400000000000000000000075721523530551300174250ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package main import ( "fmt" "os" "os/signal" "runtime/pprof" "strings" "syscall" "time" "github.com/etix/mirrorbits/cli" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/daemon" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/http" "github.com/etix/mirrorbits/logs" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/process" "github.com/etix/mirrorbits/rpc" "github.com/op/go-logging" ) var ( log = logging.MustGetLogger("main") ) func main() { core.Parseflags() if core.CpuProfile != "" { f, err := os.Create(core.CpuProfile) if err != nil { log.Fatal(err) } defer f.Close() pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } if core.Daemon { LoadConfig() logs.ReloadLogs() process.WritePidFile() // Show our nice welcome logo fmt.Printf(core.Banner+"\n\n", core.VERSION) /* Setup RPC */ rpcs := new(rpc.CLI) if err := rpcs.Start(); err != nil { log.Fatal(fmt.Errorf("rpc error: %w", err)) } /* Connect to the database */ r := database.NewRedis() r.ConnectPubsub() rpcs.SetDatabase(r) c := mirrors.NewCache(r) rpcs.SetCache(c) h := http.HTTPServer(r, c) /* Start the background monitor */ m := daemon.NewMonitor(r, c) if core.Monitor { go m.MonitorLoop() } /* Handle SIGNALS */ k := make(chan os.Signal, 1) rpcs.SetSignals(k) signal.Notify(k, syscall.SIGINT, // Terminate syscall.SIGTERM, // Terminate syscall.SIGQUIT, // Stop gracefully syscall.SIGHUP, // Reload config syscall.SIGUSR1, // Reopen log files syscall.SIGUSR2, // Seamless binary upgrade ) go func() { for { sig := <-k switch sig { case syscall.SIGINT: fallthrough case syscall.SIGTERM: process.RemovePidFile() os.Exit(0) case syscall.SIGQUIT: m.Stop() rpcs.Close() if h.Listener != nil { log.Notice("Waiting for running tasks to finish...") h.Stop(5 * time.Second) } else { process.RemovePidFile() os.Exit(0) } case syscall.SIGHUP: listenAddress := GetConfig().ListenAddress if err := ReloadConfig(); err != nil { log.Warningf("SIGHUP Received: %s\n", err) } else { log.Notice("SIGHUP Received: Reloading configuration...") } if GetConfig().ListenAddress != listenAddress { h.Restarting = true h.Stop(1 * time.Second) } h.Reload() logs.ReloadLogs() case syscall.SIGUSR1: log.Notice("SIGUSR1 Received: Re-opening logs...") logs.ReloadLogs() case syscall.SIGUSR2: log.Notice("SIGUSR2 Received: Seamless binary upgrade...") rpcs.Close() err := process.Relaunch(*h.Listener) if err != nil { log.Errorf("Relaunch failed: %s\n", err) } } } }() // Recover an existing listener (see process.go) if l, ppid, err := process.Recover(); err == nil { h.SetListener(l) go func() { time.Sleep(100 * time.Millisecond) process.KillParent(ppid) }() } /* Finally start the HTTP server */ var err error for { err = h.RunServer() if h.Restarting { h.Restarting = false continue } // This check is ugly but there's still no way to detect this error by type if err != nil && strings.Contains(err.Error(), "use of closed network connection") { // This error is expected during a graceful shutdown err = nil } break } log.Debug("Waiting for monitor termination") m.Wait() log.Debug("Terminating server") h.Terminate() r.Close() process.RemovePidFile() if err != nil { log.Fatal(err) } else { log.Notice("Server stopped gracefully.") } } else { args := os.Args[len(os.Args)-core.NArg:] if err := cli.ParseCommands(args...); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) os.Exit(1) } } os.Exit(0) } videolabs-mirrorbits-441567e/mirrorbits.conf000066400000000000000000000116121523530551300212030ustar00rootroot00000000000000# vim: set ft=yaml: ################### ##### GENERAL ##### ################### ## Path to the local repository # Repository: /srv/repo ## Path to the templates (default autodetect) # Templates: /usr/share/mirrorbits/ ## A local path or URL containing the JavaScript used by the templates. ## If this is not set (the default), the JavaScript will just be loaded ## from the usual CDNs. See also `contrib/localjs/fetchfiles.sh`. # LocalJSPath: ## Path where to store download logs (comment to disable) # LogDir: /var/log/mirrorbits ## Path to the GeoIP2 mmdb databases # GeoipDatabasePath: /usr/share/GeoIP/ ## OutputMode can take on the three values: ## - redirect: HTTP redirect to the destination file on the selected mirror ## - json: return a json document for pre-treatment by an application ## - auto: based on the Accept HTTP header # OutputMode: auto ## Enable Gzip compression # Gzip: false ## Allow redirecting HTTP requests to HTTPS mirrors. If ever a mirror supports ## both, HTTPS is favored. In other words, this setting forces HTTPS when ## possible, thus making the implicit assumption that the client supports it. # AllowHTTPToHTTPSRedirects: true ## Interval in seconds between which 2 range downloads of a given file ## from a same origin (hashed (IP, user-agent) couple) are considered ## to be the same download. In particular, download statistics are not ## incremented for this file. # SameDownloadInterval: 600 ## Host and port to listen on # ListenAddress: :8080 ## Host and port to listen for the CLI RPC # RPCListenAddress: localhost:3390 ## Password for restricting access to the CLI (optional) # RPCPassword: #################### ##### DATABASE ##### #################### ## Redis host and port # RedisAddress: 10.0.0.1:6379 ## Redis password (if any) # RedisPassword: supersecure ## Redis database ID (if any) # RedisDB: 0 ## Redis sentinel name (only if using sentinel) # RedisSentinelMasterName: mirrorbits ## List of Redis sentinel hosts (only if using sentinel) # RedisSentinels: # - Host: 10.0.0.1:26379 # - Host: 10.0.0.2:26379 # - Host: 10.0.0.3:26379 ############################ ##### LOCAL REPOSITORY ##### ############################ ## Relative path to the trace file within the repository (optional). ## The file must contain the number of seconds since epoch and should ## be updated every minute (or so) with a cron on the master repository. # TraceFileLocation: /trace ## Interval between two scans of the local repository. ## The repository scan will index new and removed files and collect file ## sizes and checksums. ## This should, more or less, match the frequency where the local repo ## is updated. # RepositoryScanInterval: 5 ## Enable or disable specific hashing algorithms # Hashes: # SHA256: On # SHA1: Off # MD5: Off ################### ##### MIRRORS ##### ################### ## Maximum number of concurrent mirror synchronization to do (rsync/ftp) # ConcurrentSync: 5 ## Interval in minutes between mirror scan # ScanInterval: 30 ## Interval in minutes between mirrors HTTP health checks # CheckInterval: 1 ## Allow a mirror to issue an HTTP redirect. ## Setting this to true will disable the mirror if a redirect is detected. # DisallowRedirects: false ## Disable a mirror if an active file is missing (HTTP 404) # DisableOnMissingFile: false ## Allow some files to be outdated on the mirrors. ## When the requested file matches any of the rules below, the file is allowed ## to be outdated at most Minutes minutes, and the file size is not checked. ## This might be desirable if the repository contains some files that are ## updated in-place, to prevent Mirrorbits from redirecting all the traffic to ## fallback mirrors for those files when they are modified. # AllowOutdatedFiles: # - Prefix: /dists/ # Minutes: 540 ## Adjust the weight/range of the geographic distribution # WeightDistributionRange: 1.5 ## Maximum number of alternative links to return in the HTTP header # MaxLinkHeaders: 10 ## Automatically fix timezone offsets. ## Enable this if one or more mirrors are always excluded because their ## last-modification-time mismatch. This option will try to guess the ## offset and adjust the mod time accordingly. ## Affected mirrors will need to be rescanned after enabling this feature. # FixTimezoneOffsets: false ## List of mirrors to use as fallback which will be used in case mirrorbits ## is unable to answer a request because the database is unreachable. ## Note: Mirrorbits will redirect to one of these mirrors based on the user ## location but won't be able to know if the mirror has the requested file. ## Therefore only put your most reliable and up-to-date mirrors here. ## Note: Omit the scheme if you want to support both http and https. # Fallbacks: # - URL: https://fallback1.mirror/repo/ # CountryCode: fr # ContinentCode: eu # - URL: https://fallback2.mirror/repo/ # CountryCode: us # ContinentCode: na videolabs-mirrorbits-441567e/mirrors/000077500000000000000000000000001523530551300176345ustar00rootroot00000000000000videolabs-mirrorbits-441567e/mirrors/cache.go000066400000000000000000000171201523530551300212270ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package mirrors import ( "fmt" "strconv" "strings" "time" "unsafe" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/network" "github.com/etix/mirrorbits/utils" "github.com/gomodule/redigo/redis" ) // Cache implements a local caching mechanism of type LRU for content available in the // redis database that is automatically invalidated if the object is updated in Redis. type Cache struct { r *database.Redis fiCache *LRUCache fmCache *LRUCache mCache *LRUCache fimCache *LRUCache mirrorUpdateEvent chan string fileUpdateEvent chan string mirrorFileUpdateEvent chan string pubsubReconnectedEvent chan string invalidationEvent chan string } type fileInfoValue struct { value filesystem.FileInfo } func (f *fileInfoValue) Size() int { return int(unsafe.Sizeof(f.value)) } type fileMirrorValue struct { value []int } func (f *fileMirrorValue) Size() int { return cap(f.value) } type mirrorValue struct { value Mirror } func (f *mirrorValue) Size() int { return int(unsafe.Sizeof(f.value)) } // NewCache constructs a new instance of Cache func NewCache(r *database.Redis) *Cache { if r == nil || r.Pubsub == nil { return nil } c := &Cache{ r: r, } // Create the LRU c.fiCache = NewLRUCache(1024000) c.fmCache = NewLRUCache(2048000) c.mCache = NewLRUCache(1024000) c.fimCache = NewLRUCache(4096000) // Create event channels c.mirrorUpdateEvent = make(chan string, 10) c.fileUpdateEvent = make(chan string, 10) c.mirrorFileUpdateEvent = make(chan string, 10) c.pubsubReconnectedEvent = make(chan string) c.invalidationEvent = make(chan string, 10) // Subscribe to events c.r.Pubsub.SubscribeEvent(database.MIRROR_UPDATE, c.mirrorUpdateEvent) c.r.Pubsub.SubscribeEvent(database.FILE_UPDATE, c.fileUpdateEvent) c.r.Pubsub.SubscribeEvent(database.MIRROR_FILE_UPDATE, c.mirrorFileUpdateEvent) c.r.Pubsub.SubscribeEvent(database.PUBSUB_RECONNECTED, c.pubsubReconnectedEvent) go func() { for { //FIXME add a close channel select { case data := <-c.mirrorUpdateEvent: c.mCache.Delete(data) select { case c.invalidationEvent <- data: default: // Non-blocking } case data := <-c.fileUpdateEvent: c.fiCache.Delete(data) case data := <-c.mirrorFileUpdateEvent: s := strings.SplitN(data, " ", 2) c.fmCache.Delete(s[1]) c.fimCache.Delete(fmt.Sprintf("%s|%s", s[0], s[1])) case <-c.pubsubReconnectedEvent: c.Clear() } } }() return c } // Clear clears the local cache func (c *Cache) Clear() { c.fiCache.Clear() c.fmCache.Clear() c.mCache.Clear() c.fimCache.Clear() } // GetMirrorInvalidationEvent returns a channel that contains ID of mirrors // that have just been invalidated. This function is supposed to have only // ONE reader and is made to avoid a race for MIRROR_UPDATE events between // a mirror invalidation and a mirror being fetched from the cache. func (c *Cache) GetMirrorInvalidationEvent() <-chan string { return c.invalidationEvent } // GetFileInfo returns file information for a given file either from the cache // or directly from the database if the object is not yet stored in the cache. func (c *Cache) GetFileInfo(path string) (f filesystem.FileInfo, err error) { v, ok := c.fiCache.Get(path) if ok { f = v.(*fileInfoValue).value } else { f, err = c.fetchFileInfo(path) } return } func (c *Cache) fetchFileInfo(path string) (f filesystem.FileInfo, err error) { rconn := c.r.Get() defer rconn.Close() f.Path = path // Path is not stored in the object instance in redis reply, err := redis.Strings(rconn.Do("HMGET", fmt.Sprintf("FILE_%s", path), "size", "modTime", "sha1", "sha256", "md5")) if err != nil { return } f.Size, _ = strconv.ParseInt(reply[0], 10, 64) f.ModTime, _ = time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", reply[1]) f.Sha1 = reply[2] f.Sha256 = reply[3] f.Md5 = reply[4] c.fiCache.Set(path, &fileInfoValue{value: f}) return } // GetMirrors returns all the mirrors serving a given file either from the cache // or directly from the database if the object is not yet stored in the cache. func (c *Cache) GetMirrors(path string, clientInfo network.GeoIPRecord) (mirrors []Mirror, err error) { var mirrorsIDs []int v, ok := c.fmCache.Get(path) if ok { mirrorsIDs = v.(*fileMirrorValue).value } else { mirrorsIDs, err = c.fetchFileMirrors(path) if err != nil { return } } mirrors = make([]Mirror, 0, len(mirrorsIDs)) for _, id := range mirrorsIDs { var mirror Mirror var fileInfo filesystem.FileInfo v, ok := c.mCache.Get(strconv.Itoa(id)) if ok { mirror = v.(*mirrorValue).value } else { //TODO execute missing items in a MULTI query mirror, err = c.fetchMirror(id) if err != nil { return } } v, ok = c.fimCache.Get(fmt.Sprintf("%d|%s", id, path)) if ok { fileInfo = v.(*fileInfoValue).value } else { fileInfo, err = c.fetchFileInfoMirror(id, path) if err != nil { return } } if fileInfo.Size >= 0 { mirror.FileInfo = &fileInfo } // Add the path in the results so we can access it from the templates mirror.FileInfo.Path = path if clientInfo.IsValid() { mirror.Distance = utils.GetDistanceKm(clientInfo.Latitude, clientInfo.Longitude, mirror.Latitude, mirror.Longitude) } else { mirror.Distance = 0 } mirrors = append(mirrors, mirror) } return } func (c *Cache) fetchFileMirrors(path string) (ids []int, err error) { rconn := c.r.Get() defer rconn.Close() ids, err = redis.Ints(rconn.Do("SMEMBERS", fmt.Sprintf("FILEMIRRORS_%s", path))) if err != nil { return } c.fmCache.Set(path, &fileMirrorValue{value: ids}) return } func (c *Cache) fetchMirror(mirrorID int) (mirror Mirror, err error) { rconn := c.r.Get() defer rconn.Close() reply, err := redis.Values(rconn.Do("HGETALL", fmt.Sprintf("MIRROR_%d", mirrorID))) if err != nil { return } if len(reply) == 0 { err = redis.ErrNil return } err = redis.ScanStruct(reply, &mirror) if err != nil { return } mirror.Prepare() c.mCache.Set(strconv.Itoa(mirrorID), &mirrorValue{value: mirror}) return } func (c *Cache) GetFileInfoMirror(mirrorID int, path string) (f filesystem.FileInfo, err error) { var fileInfo filesystem.FileInfo v, ok := c.fimCache.Get(fmt.Sprintf("%d|%s", mirrorID, path)) if ok { fileInfo = v.(*fileInfoValue).value } else { fileInfo, err = c.fetchFileInfoMirror(mirrorID, path) if err != nil { return } } return fileInfo, nil } func (c *Cache) fetchFileInfoMirror(id int, path string) (f filesystem.FileInfo, err error) { rconn := c.r.Get() defer rconn.Close() f.Path = path // Path is not stored in the object instance in redis reply, err := redis.Strings(rconn.Do("HMGET", fmt.Sprintf("FILEINFO_%d_%s", id, path), "size", "modTime", "sha1", "sha256", "md5")) if err != nil { return } // Note: as of today, only the size is stored by the scanners // all other fields are left blank. f.Size, _ = strconv.ParseInt(reply[0], 10, 64) f.ModTime, _ = time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", reply[1]) f.Sha1 = reply[2] f.Sha256 = reply[3] f.Md5 = reply[4] c.fimCache.Set(fmt.Sprintf("%d|%s", id, path), &fileInfoValue{value: f}) return } // GetMirror returns all information about a given mirror either from the cache // or directly from the database if the object is not yet stored in the cache. func (c *Cache) GetMirror(id int) (mirror Mirror, err error) { v, ok := c.mCache.Get(strconv.Itoa(id)) if ok { mirror = v.(*mirrorValue).value } else { mirror, err = c.fetchMirror(id) if err != nil { return } } return } videolabs-mirrorbits-441567e/mirrors/cache_test.go000066400000000000000000000343041523530551300222710ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package mirrors import ( "fmt" "reflect" "strconv" "testing" "time" "unsafe" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/network" . "github.com/etix/mirrorbits/testing" _ "github.com/rafaeljusto/redigomock" ) func TestNewCache(t *testing.T) { _, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(nil) if c != nil { t.Fatalf("Expected invalid instance") } c = NewCache(conn) if c == nil { t.Fatalf("No valid instance returned") } } type TestValue struct { value string } func (f *TestValue) Size() int { return int(unsafe.Sizeof(f.value)) } func TestCache_Clear(t *testing.T) { _, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) c.fiCache.Set("test", &TestValue{"42"}) c.fmCache.Set("test", &TestValue{"42"}) c.mCache.Set("test", &TestValue{"42"}) c.fimCache.Set("test", &TestValue{"42"}) c.Clear() if _, ok := c.fiCache.Get("test"); ok { t.Fatalf("Value shouldn't be present") } if _, ok := c.fmCache.Get("test"); ok { t.Fatalf("Value shouldn't be present") } if _, ok := c.mCache.Get("test"); ok { t.Fatalf("Value shouldn't be present") } if _, ok := c.fimCache.Get("test"); ok { t.Fatalf("Value shouldn't be present") } } func assertFileInfoEqual(t *testing.T, actual *filesystem.FileInfo, expected *filesystem.FileInfo) { t.Helper() if actual.Path != expected.Path { t.Fatalf("Path doesn't match, expected %#v got %#v", expected.Path, actual.Path) } if actual.Size != expected.Size { t.Fatalf("Size doesn't match, expected %#v got %#v", expected.Size, actual.Size) } if !actual.ModTime.Equal(expected.ModTime) { t.Fatalf("ModTime doesn't match, expected %s got %s", expected.ModTime.String(), actual.ModTime.String()) } if actual.Sha1 != expected.Sha1 { t.Fatalf("Sha1 doesn't match, expected %#v got %#v", expected.Sha1, actual.Sha1) } if actual.Sha256 != expected.Sha256 { t.Fatalf("Sha256 doesn't match, expected %#v got %#v", expected.Sha256, actual.Sha256) } if actual.Md5 != expected.Md5 { t.Fatalf("Md5 doesn't match, expected %#v got %#v", expected.Md5, actual.Md5) } } func TestCache_fetchFileInfo(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) testfile := filesystem.FileInfo{ Path: "/test/file.tgz", Size: 43000, ModTime: time.Now(), Sha1: "3ce963aea2d6f23fe915063f8bba21888db0ddfa", Sha256: "1c8e38c7e03e4d117eba4f82afaf6631a9b79f4c1e9dec144d4faf1d109aacda", Md5: "2c98ec39f49da6ddd9cfa7b1d7342afe", } f, err := c.fetchFileInfo(testfile.Path) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetFileinfo := mock.Command("HMGET", "FILE_"+testfile.Path, "size", "modTime", "sha1", "sha256", "md5").Expect([]any{ []byte(strconv.FormatInt(testfile.Size, 10)), []byte(testfile.ModTime.Format("2006-01-02 15:04:05.999999999 -0700 MST")), []byte(testfile.Sha1), []byte(testfile.Sha256), []byte(testfile.Md5), }) f, err = c.fetchFileInfo(testfile.Path) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFileinfo) < 1 { t.Fatalf("HMGET not executed") } assertFileInfoEqual(t, &f, &testfile) _, ok := c.fiCache.Get(testfile.Path) if !ok { t.Fatalf("Not stored in cache") } } func TestCache_fetchFileInfo_non_existing(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) testfile := filesystem.FileInfo{ Path: "/test/file.tgz", Size: 0, ModTime: time.Time{}, Sha1: "", Sha256: "", Md5: "", } f, err := c.fetchFileInfo(testfile.Path) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetFileinfo := mock.Command("HMGET", "FILE_"+testfile.Path, "size", "modTime", "sha1", "sha256", "md5").Expect([]any{ []byte(""), []byte(""), []byte(""), []byte(""), []byte(""), }) f, err = c.fetchFileInfo(testfile.Path) // fetchFileInfo on a non-existing file doesn't yield Redis.ErrNil if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFileinfo) < 1 { t.Fatalf("HMGET not executed") } assertFileInfoEqual(t, &f, &testfile) // Non-existing file are also stored in cache _, ok := c.fiCache.Get(testfile.Path) if !ok { t.Fatalf("Not stored in cache") } } func TestCache_GetFileInfo(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) testfile := filesystem.FileInfo{ Path: "/test/file.tgz", Size: 43000, ModTime: time.Now(), Sha1: "3ce963aea2d6f23fe915063f8bba21888db0ddfa", Sha256: "1c8e38c7e03e4d117eba4f82afaf6631a9b79f4c1e9dec144d4faf1d109aacda", Md5: "2c98ec39f49da6ddd9cfa7b1d7342afe", } _, err := c.GetFileInfo(testfile.Path) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetFileinfo := mock.Command("HMGET", "FILE_"+testfile.Path, "size", "modTime", "sha1", "sha256", "md5").Expect([]any{ []byte(strconv.FormatInt(testfile.Size, 10)), []byte(testfile.ModTime.Format("2006-01-02 15:04:05.999999999 -0700 MST")), []byte(testfile.Sha1), []byte(testfile.Sha256), []byte(testfile.Md5), }) f, err := c.GetFileInfo(testfile.Path) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFileinfo) < 1 { t.Fatalf("HMGET not executed") } assertFileInfoEqual(t, &f, &testfile) f, err = c.GetFileInfo(testfile.Path) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFileinfo) > 1 { t.Fatalf("Cache not used, request expected to be done once") } assertFileInfoEqual(t, &f, &testfile) } func TestCache_GetFileInfo_non_existing(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) testfile := filesystem.FileInfo{ Path: "/test/file.tgz", Size: 0, ModTime: time.Time{}, Sha1: "", Sha256: "", Md5: "", } _, err := c.GetFileInfo(testfile.Path) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetFileinfo := mock.Command("HMGET", "FILE_"+testfile.Path, "size", "modTime", "sha1", "sha256", "md5").Expect([]any{ []byte(""), []byte(""), []byte(""), []byte(""), []byte(""), }) f, err := c.GetFileInfo(testfile.Path) // GetFileInfo on a non-existing file doesn't yield Redis.ErrNil if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFileinfo) < 1 { t.Fatalf("HMGET not executed") } assertFileInfoEqual(t, &f, &testfile) f, err = c.GetFileInfo(testfile.Path) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } // Non-existing file are also stored in cache if mock.Stats(cmdGetFileinfo) > 1 { t.Fatalf("Cache not used, request expected to be done once") } assertFileInfoEqual(t, &f, &testfile) } func TestCache_fetchFileMirrors(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) filename := "/test/file.tgz" _, err := c.fetchFileMirrors(filename) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetFilemirrors := mock.Command("SMEMBERS", "FILEMIRRORS_"+filename).Expect([]any{ []byte("9"), []byte("2"), []byte("5"), }) ids, err := c.fetchFileMirrors(filename) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFilemirrors) < 1 { t.Fatalf("SMEMBERS not executed") } if len(ids) != 3 { t.Fatalf("Invalid number of items returned") } _, ok := c.fmCache.Get(filename) if !ok { t.Fatalf("Not stored in cache") } } func TestCache_fetchMirror(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) testmirror := Mirror{ ID: 1, Name: "m1", HttpURL: "http://m1.mirror", RsyncURL: "rsync://m1.mirror", FtpURL: "ftp://m1.mirror", SponsorName: "m1sponsor", SponsorURL: "m1sponsorurl", SponsorLogoURL: "m1sponsorlogourl", AdminName: "m1adminname", AdminEmail: "m1adminemail", CustomData: "m1customdata", ContinentOnly: true, CountryOnly: false, ASOnly: true, Score: 0, Latitude: -20.0, Longitude: 55.0, ContinentCode: "EU", CountryCodes: "FR UK", Asnum: 444, Comment: "m1comment", Enabled: true, HttpUp: true, HttpsUp: true, } _, err := c.fetchMirror(testmirror.ID) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetMirror := mock.Command("HGETALL", "MIRROR_1").ExpectMap(map[string]string{ "ID": strconv.Itoa(testmirror.ID), "name": testmirror.Name, "http": testmirror.HttpURL, "rsync": testmirror.RsyncURL, "ftp": testmirror.FtpURL, "sponsorName": testmirror.SponsorName, "sponsorURL": testmirror.SponsorURL, "sponsorLogo": testmirror.SponsorLogoURL, "adminName": testmirror.AdminName, "adminEmail": testmirror.AdminEmail, "customData": testmirror.CustomData, "continentOnly": strconv.FormatBool(testmirror.ContinentOnly), "countryOnly": strconv.FormatBool(testmirror.CountryOnly), "asOnly": strconv.FormatBool(testmirror.ASOnly), "score": strconv.FormatInt(int64(testmirror.Score), 10), "latitude": fmt.Sprintf("%f", testmirror.Latitude), "longitude": fmt.Sprintf("%f", testmirror.Longitude), "continentCode": testmirror.ContinentCode, "countryCodes": testmirror.CountryCodes, "asnum": strconv.FormatInt(int64(testmirror.Asnum), 10), "comment": testmirror.Comment, "enabled": strconv.FormatBool(testmirror.Enabled), "httpUp": strconv.FormatBool(testmirror.HttpUp), "httpsUp": strconv.FormatBool(testmirror.HttpsUp), }) m, err := c.fetchMirror(testmirror.ID) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetMirror) < 1 { t.Fatalf("HGETALL not executed") } // This is required to reach DeepEqual(ity) testmirror.Prepare() if !reflect.DeepEqual(testmirror, m) { t.Fatalf("Result is different") } _, ok := c.mCache.Get(strconv.Itoa(testmirror.ID)) if !ok { t.Fatalf("Not stored in cache") } } func TestCache_fetchFileInfoMirror(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) testfile := filesystem.FileInfo{ Path: "/test/file.tgz", Size: 44000, ModTime: time.Now(), Sha1: "3ce963aea2d6f23fe915063f8bba21888db0ddfa", Sha256: "1c8e38c7e03e4d117eba4f82afaf6631a9b79f4c1e9dec144d4faf1d109aacda", Md5: "2c98ec39f49da6ddd9cfa7b1d7342afe", } _, err := c.fetchFileInfoMirror(1, testfile.Path) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetFileinfomirror := mock.Command("HMGET", "FILEINFO_1_"+testfile.Path, "size", "modTime", "sha1", "sha256", "md5").Expect([]any{ []byte(strconv.FormatInt(testfile.Size, 10)), []byte(testfile.ModTime.String()), []byte(testfile.Sha1), []byte(testfile.Sha256), []byte(testfile.Md5), }) _, err = c.fetchFileInfoMirror(1, testfile.Path) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFileinfomirror) < 1 { t.Fatalf("HMGET not executed") } _, ok := c.fimCache.Get("1|" + testfile.Path) if !ok { t.Fatalf("Not stored in cache") } } func TestCache_GetMirror(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) testmirror := 1 _, err := c.GetMirror(testmirror) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetMirror := mock.Command("HGETALL", "MIRROR_1").ExpectMap(map[string]string{ "ID": strconv.Itoa(testmirror), }) m, err := c.GetMirror(testmirror) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetMirror) < 1 { t.Fatalf("HGETALL not executed") } // Results are already checked by TestCache_fetchMirror // We only need to check one of them if m.ID != testmirror { t.Fatalf("Result is different") } _, ok := c.mCache.Get(strconv.Itoa(testmirror)) if !ok { t.Fatalf("Not stored in cache") } } func TestCache_GetMirrors(t *testing.T) { mock, conn := PrepareRedisTest() conn.ConnectPubsub() c := NewCache(conn) filename := "/test/file.tgz" clientInfo := network.GeoIPRecord{ CountryCode: "FR", Latitude: 48.8567, Longitude: 2.3508, } _, err := c.GetMirrors(filename, clientInfo) if err == nil { t.Fatalf("Error expected, mock command not yet registered") } cmdGetFilemirrors := mock.Command("SMEMBERS", "FILEMIRRORS_"+filename).Expect([]any{ []byte("1"), []byte("2"), }) cmdGetMirrorM1 := mock.Command("HGETALL", "MIRROR_1").ExpectMap(map[string]string{ "ID": "1", "latitude": "52.5167", "longitude": "13.3833", }) cmdGetMirrorM2 := mock.Command("HGETALL", "MIRROR_2").ExpectMap(map[string]string{ "ID": "2", "latitude": "51.5072", "longitude": "0.1275", }) cmdGetFileinfomirrorM1 := mock.Command("HMGET", "FILEINFO_1_"+filename, "size", "modTime", "sha1", "sha256", "md5").Expect([]any{ []byte("44000"), []byte(""), []byte(""), []byte(""), []byte(""), }) cmdGetFileinfomirrorM2 := mock.Command("HMGET", "FILEINFO_2_"+filename, "size", "modTime", "sha1", "sha256", "md5").Expect([]any{ []byte("44000"), []byte(""), []byte(""), []byte(""), []byte(""), }) mirrors, err := c.GetMirrors(filename, clientInfo) if err != nil { t.Fatalf("Unexpected error: %s", err.Error()) } if mock.Stats(cmdGetFilemirrors) < 1 { t.Fatalf("cmdGetFilemirrors not called") } if mock.Stats(cmdGetMirrorM1) < 1 { t.Fatalf("cmdGetMirrorM1 not called") } if mock.Stats(cmdGetMirrorM2) < 1 { t.Fatalf("cmdGetMirrorM2 not called") } if mock.Stats(cmdGetFileinfomirrorM1) < 1 { t.Fatalf("cmdGetFileinfomirrorM1 not called") } if mock.Stats(cmdGetFileinfomirrorM2) < 1 { t.Fatalf("cmdGetFileinfomirrorM2 not called") } if len(mirrors) != 2 { t.Fatalf("Invalid number of mirrors returned") } if int(mirrors[0].Distance) != int(876) { t.Fatalf("Distance between user and m1 is wrong, got %d, expected 876", int(mirrors[0].Distance)) } if int(mirrors[1].Distance) != int(334) { t.Fatalf("Distance between user and m2 is wrong, got %d, expected 334", int(mirrors[1].Distance)) } } videolabs-mirrorbits-441567e/mirrors/logs.go000066400000000000000000000145141523530551300211340ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package mirrors import ( "encoding/json" "fmt" "time" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" "github.com/gomodule/redigo/redis" "github.com/op/go-logging" ) var ( log = logging.MustGetLogger("main") ) type LogType uint const ( _ LogType = iota LOGTYPE_ERROR LOGTYPE_ADDED LOGTYPE_EDITED LOGTYPE_ENABLED LOGTYPE_DISABLED LOGTYPE_STATECHANGED LOGTYPE_SCANSTARTED LOGTYPE_SCANCOMPLETED ) func typeToInstance(typ LogType) LogAction { switch LogType(typ) { case LOGTYPE_ERROR: return &LogError{} case LOGTYPE_ADDED: return &LogAdded{} case LOGTYPE_EDITED: return &LogEdited{} case LOGTYPE_ENABLED: return &LogEnabled{} case LOGTYPE_DISABLED: return &LogDisabled{} case LOGTYPE_STATECHANGED: return &LogStateChanged{} case LOGTYPE_SCANSTARTED: return &LogScanStarted{} case LOGTYPE_SCANCOMPLETED: return &LogScanCompleted{} default: } return nil } type LogAction interface { GetType() LogType GetMirrorID() int GetTimestamp() time.Time GetOutput() string } type LogCommonAction struct { Type LogType MirrorID int Timestamp time.Time } func (l LogCommonAction) GetType() LogType { return l.Type } func (l LogCommonAction) GetMirrorID() int { return l.MirrorID } func (l LogCommonAction) GetTimestamp() time.Time { return l.Timestamp } type LogError struct { LogCommonAction Err string } func (l *LogError) GetOutput() string { return fmt.Sprintf("Error: %s", l.Err) } func NewLogError(id int, err error) LogAction { return &LogError{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_ERROR, MirrorID: id, Timestamp: time.Now(), }, Err: err.Error(), } } type LogAdded struct { LogCommonAction } func (l *LogAdded) GetOutput() string { return "Mirror added" } func NewLogAdded(id int) LogAction { return &LogAdded{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_ADDED, MirrorID: id, Timestamp: time.Now(), }, } } type LogEdited struct { LogCommonAction } func (l *LogEdited) GetOutput() string { return "Mirror edited" } func NewLogEdited(id int) LogAction { return &LogEdited{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_EDITED, MirrorID: id, Timestamp: time.Now(), }, } } type LogEnabled struct { LogCommonAction } func (l *LogEnabled) GetOutput() string { return "Mirror enabled" } func NewLogEnabled(id int) LogAction { return &LogEnabled{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_ENABLED, MirrorID: id, Timestamp: time.Now(), }, } } type LogDisabled struct { LogCommonAction } func (l *LogDisabled) GetOutput() string { return "Mirror disabled" } func NewLogDisabled(id int) LogAction { return &LogDisabled{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_DISABLED, MirrorID: id, Timestamp: time.Now(), }, } } type LogStateChanged struct { LogCommonAction Proto Protocol Up bool Reason string } func (l *LogStateChanged) GetOutput() string { var mirror string switch l.Proto { case HTTP: mirror = "HTTP mirror" case HTTPS: mirror = "HTTPS mirror" default: mirror = "Mirror" } if l.Up == false { if len(l.Reason) == 0 { return mirror + " is down" } return mirror + " is down: " + l.Reason } return mirror + " is up" } func NewLogStateChanged(id int, proto Protocol, up bool, reason string) LogAction { return &LogStateChanged{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_STATECHANGED, MirrorID: id, Timestamp: time.Now(), }, Proto: proto, Up: up, Reason: reason, } } type LogScanStarted struct { LogCommonAction Typ core.ScannerType } func (l *LogScanStarted) GetOutput() string { switch l.Typ { case core.RSYNC: return "RSYNC scan started" case core.FTP: return "FTP scan started" default: return "Scan started using a unknown protocol" } } func NewLogScanStarted(id int, typ core.ScannerType) LogAction { return &LogScanStarted{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_SCANSTARTED, MirrorID: id, Timestamp: time.Now(), }, Typ: typ, } } type LogScanCompleted struct { LogCommonAction FilesIndexed int64 KnownIndexed int64 Removed int64 TZOffset int64 } func (l *LogScanCompleted) GetOutput() string { output := fmt.Sprintf("Scan completed: %d files (%d known), %d removed", l.FilesIndexed, l.KnownIndexed, l.Removed) if l.TZOffset != 0 { offset, _ := time.ParseDuration(fmt.Sprintf("%dms", l.TZOffset)) output += fmt.Sprintf(" (corrected timezone offset: %s)", offset) } return output } func NewLogScanCompleted(id int, files, known, removed, tzoffset int64) LogAction { return &LogScanCompleted{ LogCommonAction: LogCommonAction{ Type: LOGTYPE_SCANCOMPLETED, MirrorID: id, Timestamp: time.Now(), }, FilesIndexed: files, KnownIndexed: known, Removed: removed, TZOffset: tzoffset, } } func PushLog(r *database.Redis, logAction LogAction) error { conn := r.Get() defer conn.Close() key := fmt.Sprintf("MIRRORLOGS_%d", logAction.GetMirrorID()) value, err := json.Marshal(logAction) if err != nil { return err } _, err = conn.Do("RPUSH", key, value) return err } func ReadLogs(r *database.Redis, mirrorid, max int) ([]string, error) { conn := r.Get() defer conn.Close() if max <= 0 { // Get the latest 500 events by default max = 500 } key := fmt.Sprintf("MIRRORLOGS_%d", mirrorid) lines, err := redis.Strings(conn.Do("LRANGE", key, max*-1, -1)) if err != nil { return nil, err } outputs := make([]string, 0, len(lines)) for _, line := range lines { var objmap map[string]any err = json.Unmarshal([]byte(line), &objmap) if err != nil { log.Warningf("Unable to parse mirror log line: %s", err) continue } typf, ok := objmap["Type"].(float64) if !ok { log.Warning("Unable to parse mirror log line") continue } // Truncate the received float64 back to int typ := int(typf) action := typeToInstance(LogType(typ)) if action == nil { log.Warning("Unknown mirror log action") continue } err = json.Unmarshal([]byte(line), action) if err != nil { log.Warningf("Unable to unmarshal mirror log line: %s", err) continue } line := fmt.Sprintf("%s: %s", action.GetTimestamp().Format("2006-01-02 15:04:05 MST"), action.GetOutput()) outputs = append(outputs, line) } return outputs, nil } videolabs-mirrorbits-441567e/mirrors/lru.go000066400000000000000000000140251523530551300207670ustar00rootroot00000000000000/* Copyright 2012, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package mirrors // Implementation of an LRU cache in golang import ( "container/list" "fmt" "sync" "time" ) // LRUCache is the internal structure of the cache type LRUCache struct { mu sync.Mutex // list & table of *entry objects list *list.List table map[string]*list.Element // Our current size, in bytes. Obviously a gross simplification and low-grade // approximation. size uint64 // How many bytes we are limiting the cache to. capacity uint64 } // Value that go into LRUCache need to satisfy this interface. type Value interface { Size() int } // Item contains the key and value that goes into the cache type Item struct { Key string Value Value } type entry struct { key string value Value size int timeAccessed time.Time } // NewLRUCache return a new instance of the cache func NewLRUCache(capacity uint64) *LRUCache { return &LRUCache{ list: list.New(), table: make(map[string]*list.Element), capacity: capacity, } } // Get a value from cache func (lru *LRUCache) Get(key string) (v Value, ok bool) { lru.mu.Lock() defer lru.mu.Unlock() element := lru.table[key] if element == nil { return nil, false } lru.moveToFront(element) return element.Value.(*entry).value, true } // Set a key and associated value into the cache func (lru *LRUCache) Set(key string, value Value) { lru.mu.Lock() defer lru.mu.Unlock() if element := lru.table[key]; element != nil { lru.updateInplace(element, value) } else { lru.addNew(key, value) } } // SetIfAbsent sets a key into the cache only if it doesn't exist yet func (lru *LRUCache) SetIfAbsent(key string, value Value) { lru.mu.Lock() defer lru.mu.Unlock() if element := lru.table[key]; element != nil { lru.moveToFront(element) } else { lru.addNew(key, value) } } // Delete the key and associated value from the cache func (lru *LRUCache) Delete(key string) bool { lru.mu.Lock() defer lru.mu.Unlock() element := lru.table[key] if element == nil { return false } lru.list.Remove(element) delete(lru.table, key) lru.size -= uint64(element.Value.(*entry).size) return true } // Clear the cache func (lru *LRUCache) Clear() { lru.mu.Lock() defer lru.mu.Unlock() lru.list.Init() lru.table = make(map[string]*list.Element) lru.size = 0 } // SetCapacity sets the capacity of the cache func (lru *LRUCache) SetCapacity(capacity uint64) { lru.mu.Lock() defer lru.mu.Unlock() lru.capacity = capacity lru.checkCapacity() } // Stats return stats about the caching structure func (lru *LRUCache) Stats() (length, size, capacity uint64, oldest time.Time) { lru.mu.Lock() defer lru.mu.Unlock() if lastElem := lru.list.Back(); lastElem != nil { oldest = lastElem.Value.(*entry).timeAccessed } return uint64(lru.list.Len()), lru.size, lru.capacity, oldest } // StatsJSON returns the stats as JSON func (lru *LRUCache) StatsJSON() string { if lru == nil { return "{}" } l, s, c, o := lru.Stats() return fmt.Sprintf("{\"Length\": %v, \"Size\": %v, \"Capacity\": %v, \"OldestAccess\": \"%v\"}", l, s, c, o) } // Keys returns all the keys available in the cache func (lru *LRUCache) Keys() []string { lru.mu.Lock() defer lru.mu.Unlock() keys := make([]string, 0, lru.list.Len()) for e := lru.list.Front(); e != nil; e = e.Next() { keys = append(keys, e.Value.(*entry).key) } return keys } // Items returns all the items available in the cache func (lru *LRUCache) Items() []Item { lru.mu.Lock() defer lru.mu.Unlock() items := make([]Item, 0, lru.list.Len()) for e := lru.list.Front(); e != nil; e = e.Next() { v := e.Value.(*entry) items = append(items, Item{Key: v.key, Value: v.value}) } return items } func (lru *LRUCache) updateInplace(element *list.Element, value Value) { valueSize := value.Size() sizeDiff := valueSize - element.Value.(*entry).size element.Value.(*entry).value = value element.Value.(*entry).size = valueSize lru.size += uint64(sizeDiff) lru.moveToFront(element) lru.checkCapacity() } func (lru *LRUCache) moveToFront(element *list.Element) { lru.list.MoveToFront(element) element.Value.(*entry).timeAccessed = time.Now() } func (lru *LRUCache) addNew(key string, value Value) { newEntry := &entry{key, value, value.Size(), time.Now()} element := lru.list.PushFront(newEntry) lru.table[key] = element lru.size += uint64(newEntry.size) lru.checkCapacity() } func (lru *LRUCache) checkCapacity() { // Partially duplicated from Delete for lru.size > lru.capacity { delElem := lru.list.Back() delValue := delElem.Value.(*entry) lru.list.Remove(delElem) delete(lru.table, delValue.key) lru.size -= uint64(delValue.size) } } videolabs-mirrorbits-441567e/mirrors/lru_test.go000066400000000000000000000106051523530551300220260ustar00rootroot00000000000000// Copyright 2012, Google Inc. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package mirrors import ( "testing" ) type CacheValue struct { size int } func (cv *CacheValue) Size() int { return cv.size } func TestInitialState(t *testing.T) { cache := NewLRUCache(5) l, sz, c, _ := cache.Stats() if l != 0 { t.Errorf("length = %v, want 0", l) } if sz != 0 { t.Errorf("size = %v, want 0", sz) } if c != 5 { t.Errorf("capacity = %v, want 5", c) } } func TestSetInsertsValue(t *testing.T) { cache := NewLRUCache(100) data := &CacheValue{0} key := "key" cache.Set(key, data) v, ok := cache.Get(key) if !ok || v.(*CacheValue) != data { t.Errorf("Cache has incorrect value: %v != %v", data, v) } } func TestGetValueWithMultipleTypes(t *testing.T) { cache := NewLRUCache(100) data := &CacheValue{0} key := "key" cache.Set(key, data) v, ok := cache.Get("key") if !ok || v.(*CacheValue) != data { t.Errorf("Cache has incorrect value for \"key\": %v != %v", data, v) } v, ok = cache.Get(string([]byte{'k', 'e', 'y'})) if !ok || v.(*CacheValue) != data { t.Errorf("Cache has incorrect value for []byte {'k','e','y'}: %v != %v", data, v) } } func TestSetUpdatesSize(t *testing.T) { cache := NewLRUCache(100) emptyValue := &CacheValue{0} key := "key1" cache.Set(key, emptyValue) if _, sz, _, _ := cache.Stats(); sz != 0 { t.Errorf("cache.Size() = %v, expected 0", sz) } someValue := &CacheValue{20} key = "key2" cache.Set(key, someValue) if _, sz, _, _ := cache.Stats(); sz != 20 { t.Errorf("cache.Size() = %v, expected 20", sz) } } func TestSetWithOldKeyUpdatesValue(t *testing.T) { cache := NewLRUCache(100) emptyValue := &CacheValue{0} key := "key1" cache.Set(key, emptyValue) someValue := &CacheValue{20} cache.Set(key, someValue) v, ok := cache.Get(key) if !ok || v.(*CacheValue) != someValue { t.Errorf("Cache has incorrect value: %v != %v", someValue, v) } } func TestSetWithOldKeyUpdatesSize(t *testing.T) { cache := NewLRUCache(100) emptyValue := &CacheValue{0} key := "key1" cache.Set(key, emptyValue) if _, sz, _, _ := cache.Stats(); sz != 0 { t.Errorf("cache.Size() = %v, expected %v", sz, 0) } someValue := &CacheValue{20} cache.Set(key, someValue) expected := uint64(someValue.size) if _, sz, _, _ := cache.Stats(); sz != expected { t.Errorf("cache.Size() = %v, expected %v", sz, expected) } } func TestGetNonExistent(t *testing.T) { cache := NewLRUCache(100) if _, ok := cache.Get("crap"); ok { t.Error("Cache returned a crap value after no inserts.") } } func TestDelete(t *testing.T) { cache := NewLRUCache(100) value := &CacheValue{1} key := "key" if cache.Delete(key) { t.Error("Item unexpectedly already in cache.") } cache.Set(key, value) if !cache.Delete(key) { t.Error("Expected item to be in cache.") } if _, sz, _, _ := cache.Stats(); sz != 0 { t.Errorf("cache.Size() = %v, expected 0", sz) } if _, ok := cache.Get(key); ok { t.Error("Cache returned a value after deletion.") } } func TestClear(t *testing.T) { cache := NewLRUCache(100) value := &CacheValue{1} key := "key" cache.Set(key, value) cache.Clear() if _, sz, _, _ := cache.Stats(); sz != 0 { t.Errorf("cache.Size() = %v, expected 0 after Clear()", sz) } } func TestCapacityIsObeyed(t *testing.T) { size := uint64(3) cache := NewLRUCache(size) value := &CacheValue{1} // Insert up to the cache's capacity. cache.Set("key1", value) cache.Set("key2", value) cache.Set("key3", value) if _, sz, _, _ := cache.Stats(); sz != size { t.Errorf("cache.Size() = %v, expected %v", sz, size) } // Insert one more; something should be evicted to make room. cache.Set("key4", value) if _, sz, _, _ := cache.Stats(); sz != size { t.Errorf("post-evict cache.Size() = %v, expected %v", sz, size) } } func TestLRUIsEvicted(t *testing.T) { size := uint64(3) cache := NewLRUCache(size) cache.Set("key1", &CacheValue{1}) cache.Set("key2", &CacheValue{1}) cache.Set("key3", &CacheValue{1}) // lru: [key3, key2, key1] // Look up the elements. This will rearrange the LRU ordering. cache.Get("key3") cache.Get("key2") cache.Get("key1") // lru: [key1, key2, key3] cache.Set("key0", &CacheValue{1}) // lru: [key0, key1, key2] // The least recently used one should have been evicted. if _, ok := cache.Get("key3"); ok { t.Error("Least recently used element was not evicted.") } } videolabs-mirrorbits-441567e/mirrors/mirrors.go000066400000000000000000000264031523530551300216650ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package mirrors import ( "fmt" "math/rand" "strconv" "strings" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/network" "github.com/etix/mirrorbits/utils" "github.com/gomodule/redigo/redis" ) type Protocol uint const ( UNDEFINED Protocol = iota HTTP HTTPS ) func (p Protocol) String() string { switch p { case UNDEFINED: return "undefined" case HTTP: return "HTTP" case HTTPS: return "HTTPS" default: return "unknown" } } // Mirror is the structure representing all the information about a mirror type Mirror struct { ID int `redis:"ID" yaml:"-"` Name string `redis:"name" yaml:"Name"` HttpURL string `redis:"http" yaml:"HttpURL"` RsyncURL string `redis:"rsync" yaml:"RsyncURL"` FtpURL string `redis:"ftp" yaml:"FtpURL"` SponsorName string `redis:"sponsorName" yaml:"SponsorName"` SponsorURL string `redis:"sponsorURL" yaml:"SponsorURL"` SponsorLogoURL string `redis:"sponsorLogo" yaml:"SponsorLogoURL"` AdminName string `redis:"adminName" yaml:"AdminName"` AdminEmail string `redis:"adminEmail" yaml:"AdminEmail"` CustomData string `redis:"customData" yaml:"CustomData"` ContinentOnly bool `redis:"continentOnly" yaml:"ContinentOnly"` CountryOnly bool `redis:"countryOnly" yaml:"CountryOnly"` ASOnly bool `redis:"asOnly" yaml:"ASOnly"` Score int `redis:"score" yaml:"Score"` Latitude float32 `redis:"latitude" yaml:"Latitude"` Longitude float32 `redis:"longitude" yaml:"Longitude"` ContinentCode string `redis:"continentCode" yaml:"ContinentCode"` CountryCodes string `redis:"countryCodes" yaml:"CountryCodes"` ExcludedCountryCodes string `redis:"excludedCountryCodes" yaml:"ExcludedCountryCodes"` Asnum uint `redis:"asnum" yaml:"ASNum"` Comment string `redis:"comment" yaml:"-"` Enabled bool `redis:"enabled" yaml:"Enabled"` HttpUp bool `redis:"httpUp" json:"-" yaml:"-"` HttpsUp bool `redis:"httpsUp" json:"-" yaml:"-"` HttpDownReason string `redis:"httpDownReason" json:",omitempty" yaml:"-"` HttpsDownReason string `redis:"httpsDownReason" json:",omitempty" yaml:"-"` StateSince Time `redis:"stateSince" json:",omitempty" yaml:"-"` AllowRedirects Redirects `redis:"allowredirects" json:",omitempty" yaml:"AllowRedirects"` TZOffset int64 `redis:"tzoffset" json:"-" yaml:"-"` // timezone offset in ms Distance float32 `redis:"-" yaml:"-"` CountryFields []string `redis:"-" json:"-" yaml:"-"` ExcludedCountryFields []string `redis:"-" json:"-" yaml:"-"` Filepath string `redis:"-" json:"-" yaml:"-"` Weight float32 `redis:"-" json:"-" yaml:"-"` ComputedScore int `redis:"-" yaml:"-"` LastSync Time `redis:"lastSync" yaml:"-"` LastSuccessfulSync Time `redis:"lastSuccessfulSync" yaml:"-"` LastSuccessfulSyncProtocol core.ScannerType `redis:"lastSuccessfulSyncProtocol" yaml:"-"` LastSuccessfulSyncPrecision core.Precision `redis:"lastSuccessfulSyncPrecision" yaml:"-"` LastModTime Time `redis:"lastModTime" yaml:"-"` FileInfo *filesystem.FileInfo `redis:"-" json:"-" yaml:"-"` // Details of the requested file on this specific mirror AbsoluteURL string `redis:"-" yaml:"-"` // Absolute HttpURL, guaranteed to start with a scheme ExcludeReason string `redis:"-" json:",omitempty" yaml:"-"` // Reason why the mirror was excluded } // Prepare must be called after retrieval from the database to reformat some values func (m *Mirror) Prepare() { m.CountryFields = strings.Fields(m.CountryCodes) m.ExcludedCountryFields = strings.Fields(m.ExcludedCountryCodes) } // IsHTTPOnly returns true if the mirror has an HTTP address func (m *Mirror) IsHTTPOnly() bool { return strings.HasPrefix(m.HttpURL, "http://") } // IsHTTPSOnly returns true if the mirror has an HTTPS address func (m *Mirror) IsHTTPSOnly() bool { return strings.HasPrefix(m.HttpURL, "https://") } // IsUp returns true if the mirror is up (for a mirror that supports both HTTP // and HTTPS, it means both are up) func (m *Mirror) IsUp() bool { if m.HttpUp == m.HttpsUp { return m.HttpUp } if m.IsHTTPOnly() { return m.HttpUp } if m.IsHTTPSOnly() { return m.HttpsUp } return false } // Mirrors represents a slice of Mirror type Mirrors []Mirror // Len return the number of Mirror in the slice func (s Mirrors) Len() int { return len(s) } // Swap swaps mirrors at index i and j func (s Mirrors) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // ByRank is used to sort a slice of Mirror by their rank type ByRank struct { Mirrors ClientInfo network.GeoIPRecord } // Less compares two mirrors based on their rank func (m ByRank) Less(i, j int) bool { if m.ClientInfo.IsValid() { if m.ClientInfo.ASNum == m.Mirrors[i].Asnum { if m.Mirrors[i].Asnum != m.Mirrors[j].Asnum { return true } } else if m.ClientInfo.ASNum == m.Mirrors[j].Asnum { return false } //TODO Simplify me if m.ClientInfo.CountryCode != "" { if utils.IsInSlice(m.ClientInfo.CountryCode, m.Mirrors[i].CountryFields) { if !utils.IsInSlice(m.ClientInfo.CountryCode, m.Mirrors[j].CountryFields) { return true } } else if utils.IsInSlice(m.ClientInfo.CountryCode, m.Mirrors[j].CountryFields) { return false } } if m.ClientInfo.ContinentCode != "" { if m.ClientInfo.ContinentCode == m.Mirrors[i].ContinentCode { if m.ClientInfo.ContinentCode != m.Mirrors[j].ContinentCode { return true } } else if m.ClientInfo.ContinentCode == m.Mirrors[j].ContinentCode { return false } } return m.Mirrors[i].Distance < m.Mirrors[j].Distance } // Randomize the output if we miss client info return rand.Intn(2) == 0 } // ByComputedScore is used to sort a slice of Mirror by their score type ByComputedScore struct { Mirrors } // Less compares two mirrors based on their score func (b ByComputedScore) Less(i, j int) bool { return b.Mirrors[i].ComputedScore > b.Mirrors[j].ComputedScore } // ByExcludeReason is used to sort a slice of Mirror alphabetically by their exclude reason type ByExcludeReason struct { Mirrors } // Less compares two mirrors based on their exclude reason func (b ByExcludeReason) Less(i, j int) bool { if b.Mirrors[i].ExcludeReason < b.Mirrors[j].ExcludeReason { return true } return false } // EnableMirror enables the given mirror func EnableMirror(r *database.Redis, id int) error { return SetMirrorEnabled(r, id, true) } // DisableMirror disables the given mirror func DisableMirror(r *database.Redis, id int) error { return SetMirrorEnabled(r, id, false) } // SetMirrorEnabled marks a mirror as enabled or disabled func SetMirrorEnabled(r *database.Redis, id int, state bool) error { conn := r.Get() defer conn.Close() key := fmt.Sprintf("MIRROR_%d", id) _, err := conn.Do("HSET", key, "enabled", state) // Publish update if err == nil { database.Publish(conn, database.MIRROR_UPDATE, strconv.Itoa(id)) if state == true { PushLog(r, NewLogEnabled(id)) } else { PushLog(r, NewLogDisabled(id)) } } return err } // MarkMirrorUp marks the given mirror as up func MarkMirrorUp(r *database.Redis, id int, proto Protocol) error { return SetMirrorState(r, id, proto, true, "") } // MarkMirrorDown marks the given mirror as down func MarkMirrorDown(r *database.Redis, id int, proto Protocol, reason string) error { return SetMirrorState(r, id, proto, false, reason) } // SetMirrorState sets the state of a mirror to up or down, over HTTP or HTTPS, // with an optional reason func SetMirrorState(r *database.Redis, id int, proto Protocol, state bool, reason string) error { conn := r.Get() defer conn.Close() key := fmt.Sprintf("MIRROR_%d", id) var upField, reasonField string switch proto { case HTTP: upField, reasonField = "httpUp", "httpDownReason" case HTTPS: upField, reasonField = "httpsUp", "httpsDownReason" default: return fmt.Errorf("Unknown protocol: %s", proto) } previousState, err := redis.Bool(conn.Do("HGET", key, upField)) if err != nil && err != redis.ErrNil { return err } var args []any args = append(args, key, upField, state, reasonField, reason) if state != previousState { args = append(args, "stateSince", time.Now().Unix()) } _, err = conn.Do("HSET", args...) if err == nil { // Publish update database.Publish(conn, database.MIRROR_UPDATE, strconv.Itoa(id)) if state != previousState { PushLog(r, NewLogStateChanged(id, proto, state, reason)) } } return err } // Results is the resulting struct of a request and is // used by the renderers to generate the final page. type Results struct { FileInfo filesystem.FileInfo IP string ClientInfo network.GeoIPRecord MirrorList Mirrors ExcludedList Mirrors `json:",omitempty"` Fallback bool `json:",omitempty"` LocalJSPath string } // Redirects is handling the per-mirror authorization of HTTP redirects type Redirects int // Allowed will return true if redirects are authorized for this mirror func (r *Redirects) Allowed() bool { switch *r { case 1: return true case 2: return false default: return GetConfig().DisallowRedirects == false } } // MarshalYAML converts internal values to YAML func (r Redirects) MarshalYAML() (any, error) { var b *bool switch r { case 1: v := true b = &v case 2: v := false b = &v default: } return b, nil } // UnmarshalYAML converts YAML to internal values func (r *Redirects) UnmarshalYAML(unmarshal func(any) error) error { var b *bool if err := unmarshal(&b); err != nil { return err } if b == nil { *r = 0 } else if *b == true { *r = 1 } else { *r = 2 } return nil } // Time is a structure holding a time.Time object. // It is used to serialize and deserialize a time // held in a redis database. type Time struct { time.Time } // RedisArg serialize the time.Time object func (t Time) RedisArg() any { return t.UTC().Unix() } // RedisScan deserialize the time.Time object func (t *Time) RedisScan(src any) (err error) { switch src := src.(type) { case int64: t.Time = time.Unix(src, 0) case []byte: var i int64 i, err = strconv.ParseInt(string(src), 10, 64) t.Time = time.Unix(i, 0) default: err = fmt.Errorf("cannot convert from %T to %T", src, t) } return err } // FromTime returns a Time from a time.Time func (t Time) FromTime(time time.Time) Time { return Time{ Time: time, } } videolabs-mirrorbits-441567e/mirrors/mirrors_test.go000066400000000000000000000265161523530551300227310ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package mirrors import ( "fmt" "math/rand" "sort" "strings" "testing" "time" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/network" . "github.com/etix/mirrorbits/testing" "github.com/gomodule/redigo/redis" "github.com/rafaeljusto/redigomock" ) func generateSimpleMirrorList(number int) Mirrors { ret := Mirrors{} for i := 0; i < number; i++ { m := Mirror{ ID: i, Name: fmt.Sprintf("M%d", i), } ret = append(ret, m) } return ret } func formatMirrorOrder(mirrors Mirrors) string { buf := "" for _, m := range mirrors { buf += fmt.Sprintf("%s, ", m.Name) } return strings.TrimSuffix(buf, ", ") } func matchingMirrorOrder(m Mirrors, order []int) bool { if len(m) != len(order) { return false } for i, v := range order { if v != m[i].ID { return false } } return true } func TestMirrors_Len(t *testing.T) { m := Mirrors{} if m.Len() != 0 { t.Fatalf("Expected 0, got %d", m.Len()) } m = generateSimpleMirrorList(2) if m.Len() != len(m) { t.Fatalf("Expected %d, got %d", len(m), m.Len()) } } func TestMirrors_Swap(t *testing.T) { m := generateSimpleMirrorList(5) if !matchingMirrorOrder(m, []int{0, 1, 2, 3, 4}) { t.Fatalf("Expected M0 before M1, got %s", formatMirrorOrder(m)) } m.Swap(0, 1) if !matchingMirrorOrder(m, []int{1, 0, 2, 3, 4}) { t.Fatalf("Expected M1 before M0, got %s", formatMirrorOrder(m)) } m.Swap(2, 4) if !matchingMirrorOrder(m, []int{1, 0, 4, 3, 2}) { t.Fatal("Expected M4 at position 2 and M2 at position 4", m) } } func TestByRank_Less(t *testing.T) { rand.Seed(time.Now().UnixNano()) /* */ c := network.GeoIPRecord{} if c.IsValid() { t.Fatalf("GeoIPRecord is supposed to be invalid") } /* */ // Generate two identical slices m1 := generateSimpleMirrorList(50) m2 := generateSimpleMirrorList(50) // Mirrors are identical (besides name) so ByRank is expected // to randomize their order. sort.Sort(ByRank{m1, c}) differences := 0 for i, m := range m1 { if m.ID != m2[i].ID { differences++ } } if differences == 0 { t.Fatalf("Result is supposed to be randomized") } else if differences < 10 { t.Fatalf("Too many similarities, something's wrong?") } // Sort again, just to be sure the result is different m3 := generateSimpleMirrorList(50) sort.Sort(ByRank{m3, c}) differences = 0 for i, m := range m3 { if m.ID != m1[i].ID { differences++ } } if differences == 0 { t.Fatalf("Result is supposed to be different from previous run") } else if differences < 10 { t.Fatalf("Too many similarities, something's wrong?") } /* */ c = network.GeoIPRecord{ CountryCode: "FR", ContinentCode: "EU", ASNum: 4444, } if !c.IsValid() { t.Fatalf("GeoIPRecord is supposed to be valid") } /* asnum */ m := Mirrors{ Mirror{ ID: 1, Name: "M1", Asnum: 6666, }, Mirror{ ID: 2, Name: "M2", Asnum: 5555, }, Mirror{ ID: 3, Name: "M3", Asnum: 4444, }, Mirror{ ID: 4, Name: "M4", Asnum: 6666, }, } sort.Sort(ByRank{m, c}) if !matchingMirrorOrder(m, []int{3, 1, 2, 4}) { t.Fatalf("Order doesn't seem right: %s, expected M3, M1, M2, M4", formatMirrorOrder(m)) } /* distance */ m = Mirrors{ Mirror{ ID: 1, Name: "M1", Distance: 1000.0, }, Mirror{ ID: 2, Name: "M2", Distance: 999.0, }, Mirror{ ID: 3, Name: "M3", Distance: 1000.0, }, Mirror{ ID: 4, Name: "M4", Distance: 888.0, }, } sort.Sort(ByRank{m, c}) if !matchingMirrorOrder(m, []int{4, 2, 1, 3}) { t.Fatalf("Order doesn't seem right: %s, expected M4, M2, M1, M3", formatMirrorOrder(m)) } /* countrycode */ m = Mirrors{ Mirror{ ID: 1, Name: "M1", CountryFields: []string{"IT", "UK"}, }, Mirror{ ID: 2, Name: "M2", CountryFields: []string{"IT", "UK"}, }, Mirror{ ID: 3, Name: "M3", CountryFields: []string{"IT", "FR"}, }, Mirror{ ID: 4, Name: "M4", CountryFields: []string{"FR", "UK"}, }, } sort.Sort(ByRank{m, c}) if !matchingMirrorOrder(m, []int{3, 4, 1, 2}) { t.Fatalf("Order doesn't seem right: %s, expected M3, M4, M1, M2", formatMirrorOrder(m)) } /* continentcode */ c = network.GeoIPRecord{ ContinentCode: "EU", ASNum: 4444, CountryCode: "XX", } m = Mirrors{ Mirror{ ID: 1, Name: "M1", ContinentCode: "NA", }, Mirror{ ID: 2, Name: "M2", ContinentCode: "NA", }, Mirror{ ID: 3, Name: "M3", ContinentCode: "EU", }, Mirror{ ID: 4, Name: "M4", ContinentCode: "NA", }, } sort.Sort(ByRank{m, c}) if !matchingMirrorOrder(m, []int{3, 1, 2, 4}) { t.Fatalf("Order doesn't seem right: %s, expected M3, M1, M2, M4", formatMirrorOrder(m)) } /* */ c = network.GeoIPRecord{ CountryCode: "FR", ContinentCode: "EU", ASNum: 4444, } m = Mirrors{ Mirror{ ID: 1, Name: "M1", Distance: 100.0, CountryFields: []string{"IT", "FR"}, ContinentCode: "EU", }, Mirror{ ID: 2, Name: "M2", Distance: 200.0, CountryFields: []string{"FR", "CH"}, ContinentCode: "EU", }, Mirror{ ID: 3, Name: "M3", Distance: 1000.0, CountryFields: []string{"UK", "DE"}, Asnum: 4444, }, } sort.Sort(ByRank{m, c}) if !matchingMirrorOrder(m, []int{3, 1, 2}) { t.Fatalf("Order doesn't seem right: %s, expected M3, M1, M2", formatMirrorOrder(m)) } } func TestByComputedScore_Less(t *testing.T) { m := Mirrors{ Mirror{ ID: 1, Name: "M1", ComputedScore: 50, }, Mirror{ ID: 2, Name: "M2", ComputedScore: 0, }, Mirror{ ID: 3, Name: "M3", ComputedScore: 2500, }, Mirror{ ID: 4, Name: "M4", ComputedScore: 21, }, } sort.Sort(ByComputedScore{m}) if !matchingMirrorOrder(m, []int{3, 1, 4, 2}) { t.Fatalf("Order doesn't seem right: %s, expected M3, M1, M4, M2", formatMirrorOrder(m)) } } func TestByExcludeReason_Less(t *testing.T) { m := Mirrors{ Mirror{ ID: 1, Name: "M1", ExcludeReason: "x42", }, Mirror{ ID: 2, Name: "M2", ExcludeReason: "x43", }, Mirror{ ID: 3, Name: "M3", ExcludeReason: "Test one", }, Mirror{ ID: 4, Name: "M4", ExcludeReason: "Test two", }, Mirror{ ID: 5, Name: "M5", ExcludeReason: "test three", }, } sort.Sort(ByExcludeReason{m}) if !matchingMirrorOrder(m, []int{3, 4, 5, 1, 2}) { t.Fatalf("Order doesn't seem right: %s, expected M3, M4, M5, M1, M2", formatMirrorOrder(m)) } } func TestEnableMirror(t *testing.T) { mock, conn := PrepareRedisTest() cmdEnable := mock.Command("HSET", "MIRROR_1", "enabled", true).Expect("ok") EnableMirror(conn, 1) if mock.Stats(cmdEnable) != 1 { t.Fatalf("Mirror not enabled") } mock.Command("HSET", "MIRROR_1", "enabled", true).ExpectError(redis.Error("blah")) if EnableMirror(conn, 1) == nil { t.Fatalf("Error expected") } } func TestDisableMirror(t *testing.T) { mock, conn := PrepareRedisTest() cmdDisable := mock.Command("HSET", "MIRROR_1", "enabled", false).Expect("ok") DisableMirror(conn, 1) if mock.Stats(cmdDisable) != 1 { t.Fatalf("Mirror not enabled") } mock.Command("HSET", "MIRROR_1", "enabled", false).ExpectError(redis.Error("blah")) if DisableMirror(conn, 1) == nil { t.Fatalf("Error expected") } } func TestSetMirrorEnabled(t *testing.T) { mock, conn := PrepareRedisTest() cmdPublish := mock.Command("PUBLISH", string(database.MIRROR_UPDATE), redigomock.NewAnyData()).Expect("ok") cmdEnable := mock.Command("HSET", "MIRROR_1", "enabled", true).Expect("ok") SetMirrorEnabled(conn, 1, true) if mock.Stats(cmdEnable) < 1 { t.Fatalf("Mirror not enabled") } else if mock.Stats(cmdEnable) > 1 { t.Fatalf("Mirror enabled more than once") } if mock.Stats(cmdPublish) < 1 { t.Fatalf("Event MIRROR_UPDATE not published") } mock.Command("HSET", "MIRROR_1", "enabled", true).ExpectError(redis.Error("blah")) if SetMirrorEnabled(conn, 1, true) == nil { t.Fatalf("Error expected") } cmdDisable := mock.Command("HSET", "MIRROR_1", "enabled", false).Expect("ok") SetMirrorEnabled(conn, 1, false) if mock.Stats(cmdDisable) != 1 { t.Fatalf("Mirror not disabled") } else if mock.Stats(cmdDisable) > 1 { t.Fatalf("Mirror disabled more than once") } if mock.Stats(cmdPublish) < 2 { t.Fatalf("Event MIRROR_UPDATE not published") } mock.Command("HSET", "MIRROR_1", "enabled", false).ExpectError(redis.Error("blah")) if SetMirrorEnabled(conn, 1, false) == nil { t.Fatalf("Error expected") } } func TestMarkMirrorUp(t *testing.T) { _, conn := PrepareRedisTest() if err := MarkMirrorUp(conn, 1, HTTP); err == nil { t.Fatalf("Error expected but nil returned") } } func TestMarkMirrorDown(t *testing.T) { _, conn := PrepareRedisTest() if err := MarkMirrorDown(conn, 1, HTTP, "test1"); err == nil { t.Fatalf("Error expected but nil returned") } } func TestSetMirrorState(t *testing.T) { mock, conn := PrepareRedisTest() if err := SetMirrorState(conn, 1, HTTP, true, "test1"); err == nil { t.Fatalf("Error expected but nil returned") } cmdPublish := mock.Command("PUBLISH", string(database.MIRROR_UPDATE), redigomock.NewAnyData()).Expect("ok") /* Set HTTP mirror up */ cmdPreviousState := mock.Command("HGET", "MIRROR_1", "httpUp").Expect(int64(0)).Expect(int64(1)) cmdStateSince := mock.Command("HSET", "MIRROR_1", "httpUp", true, "httpDownReason", "test1", "stateSince", redigomock.NewAnyInt()).Expect("ok") cmdState := mock.Command("HSET", "MIRROR_1", "httpUp", true, "httpDownReason", "test2").Expect("ok") if err := SetMirrorState(conn, 1, HTTP, true, "test1"); err != nil { t.Fatalf("Unexpected error: %s", err) } if mock.Stats(cmdPreviousState) < 1 { t.Fatalf("Previous state not tested") } if mock.Stats(cmdStateSince) < 1 { t.Fatalf("New state not set") } else if mock.Stats(cmdStateSince) > 1 { t.Fatalf("State set more than once") } if mock.Stats(cmdPublish) < 1 { t.Fatalf("Event MIRROR_UPDATE not published") } /* Set HTTP mirror up a second time */ if err := SetMirrorState(conn, 1, HTTP, true, "test2"); err != nil { t.Fatalf("Unexpected error: %s", err) } if mock.Stats(cmdStateSince) > 1 || mock.Stats(cmdState) < 1 { t.Fatalf("The value stateSince isn't supposed to be set") } if mock.Stats(cmdPublish) != 2 { t.Fatalf("Event MIRROR_UPDATE should be sent") } /* Set HTTP mirror down */ cmdPreviousState = mock.Command("HGET", "MIRROR_1", "httpUp").Expect(int64(1)) cmdStateSince = mock.Command("HSET", "MIRROR_1", "httpUp", false, "httpDownReason", "test3", "stateSince", redigomock.NewAnyInt()).Expect("ok") if err := SetMirrorState(conn, 1, HTTP, false, "test3"); err != nil { t.Fatalf("Unexpected error: %s", err) } if mock.Stats(cmdPreviousState) < 1 { t.Fatalf("Previous state not tested") } if mock.Stats(cmdStateSince) < 1 { t.Fatalf("New state not set") } else if mock.Stats(cmdStateSince) > 1 { t.Fatalf("State set more than once") } if mock.Stats(cmdPublish) < 2 { t.Fatalf("Event MIRROR_UPDATE not published") } } videolabs-mirrorbits-441567e/network/000077500000000000000000000000001523530551300176305ustar00rootroot00000000000000videolabs-mirrorbits-441567e/network/clusterlock.go000066400000000000000000000042711523530551300225150ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package network import ( "errors" "os" "time" "github.com/etix/mirrorbits/database" "github.com/gomodule/redigo/redis" ) const ( lockTTL = 10 // in seconds lockRefresh = 5 // in seconds ) // ClusterLock holds the internal structure of a ClusterLock type ClusterLock struct { redis *database.Redis key string identifier string done chan struct{} } // NewClusterLock returns a new instance of a ClusterLock. // A ClucterLock is used to maitain a lock on a mirror that is being // scanned. The lock is renewed every lockRefresh seconds and is // automatically released by the redis database every lockTTL seconds // allowing the lock to be released even if the application is killed. func NewClusterLock(redis *database.Redis, key, identifier string) *ClusterLock { return &ClusterLock{ redis: redis, key: key, identifier: identifier, } } // Get tries to obtain an exclusive lock, cluster wide, for the given mirror func (n *ClusterLock) Get() (<-chan struct{}, error) { if n.done != nil { return nil, errors.New("lock already in use") } conn := n.redis.Get() defer conn.Close() if conn.Err() != nil { return nil, conn.Err() } _, err := redis.String(conn.Do("SET", n.key, 1, "NX", "EX", lockTTL)) if err == redis.ErrNil { return nil, nil } else if err != nil { return nil, err } n.done = make(chan struct{}) // Maintain the lock active until release go func() { conn := n.redis.Get() defer conn.Close() for { select { case <-n.done: n.done = nil conn.Do("DEL", n.key) return case <-time.After(lockRefresh * time.Second): result, err := redis.Int(conn.Do("EXPIRE", n.key, lockTTL)) if err != nil { log.Errorf("Renewing lock for %s failed: %s", n.identifier, err) return } else if result == 0 { log.Errorf("Renewing lock for %s failed: lock disappeared", n.identifier) return } if os.Getenv("DEBUG") != "" { log.Debugf("[%s] Lock renewed", n.identifier) } } } }() return n.done, nil } // Release releases the exclusive lock on the mirror func (n *ClusterLock) Release() { close(n.done) } videolabs-mirrorbits-441567e/network/geoip.go000066400000000000000000000112021523530551300212560ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package network import ( "errors" "net" "os" "strings" "sync" "time" . "github.com/etix/mirrorbits/config" "github.com/op/go-logging" "github.com/oschwald/maxminddb-golang" ) var ( // ErrMultipleAddresses is returned when the mirror has more than one address ErrMultipleAddresses = errors.New("the mirror has more than one IP address") log = logging.MustGetLogger("main") ) const ( geoipUpdatedExt = ".updated" ) // GeoIP contains methods to query the GeoIP database type GeoIP struct { sync.RWMutex city *geoipDB asn *geoipDB } // GeoIPRecord defines a GeoIP record for a given IP address type GeoIPRecord struct { // City DB CountryCode string ContinentCode string City string Country string Latitude float32 Longitude float32 // ASN DB ASName string ASNum uint } // Geolocalizer is an interface representing a GeoIP library type Geolocalizer interface { Lookup(ipAddress net.IP, result any) error } // NewGeoIP instanciates a new instance of GeoIP func NewGeoIP() *GeoIP { return &GeoIP{} } // Open the GeoIP database func (g *GeoIP) openDatabase(file string) (*maxminddb.Reader, error) { dbpath := GetConfig().GeoipDatabasePath if dbpath != "" && !strings.HasSuffix(dbpath, "/") { dbpath += "/" } filename := dbpath + file var err error if _, err = os.Stat(filename + geoipUpdatedExt); !os.IsNotExist(err) { filename += geoipUpdatedExt } return maxminddb.Open(filename) } type geoipDB struct { filename string modTime time.Time db Geolocalizer } func (g *GeoIP) loadDB(filename string, geodb **geoipDB, geoiperror *GeoIPError) error { // Increase the loaded counter geoiperror.loaded++ if *geodb == nil { *geodb = &geoipDB{ filename: filename, } } db, err := g.openDatabase(filename) if err != nil { geoiperror.Errors = append(geoiperror.Errors, err) return err } modTime := time.Unix(int64(db.Metadata.BuildEpoch), 0) if (*geodb).modTime.Equal(modTime) { return nil } (*geodb).db = db (*geodb).modTime = modTime log.Infof("Loading %s database (built on %s)", filename, (*geodb).modTime) return nil } // GeoIPError holds errors while loading the different databases type GeoIPError struct { Errors []error loaded int } func (e GeoIPError) Error() string { return "One or more GeoIP database could not be loaded" } // IsFatal returns true if the error is fatal func (e GeoIPError) IsFatal() bool { return e.loaded == len(e.Errors) } // LoadGeoIP loads the GeoIP databases into memory func (g *GeoIP) LoadGeoIP() error { var ret GeoIPError g.Lock() g.loadDB("GeoLite2-City.mmdb", &g.city, &ret) g.loadDB("GeoLite2-ASN.mmdb", &g.asn, &ret) g.Unlock() if len(ret.Errors) > 0 { return ret } return nil } // GetRecord return informations about the given ip address // (works in IPv4 and v6) func (g *GeoIP) GetRecord(ip string) (ret GeoIPRecord) { addr := net.ParseIP(ip) if addr == nil { return GeoIPRecord{} } type CityDb struct { City struct { Names struct { English string `maxminddb:"en"` } `maxminddb:"names"` } `maxminddb:"city"` Country struct { IsoCode string `maxminddb:"iso_code"` Names struct { English string `maxminddb:"en"` } `maxminddb:"names"` } `maxminddb:"country"` Continent struct { Code string `maxminddb:"code"` } `maxminddb:"continent"` Location struct { Latitude float64 `maxminddb:"latitude"` Longitude float64 `maxminddb:"longitude"` } `maxminddb:"location"` } type ASNDb struct { AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"` AutonomousSystemOrg string `maxminddb:"autonomous_system_organization"` } var err error var cityDb CityDb var asnDb ASNDb g.RLock() defer g.RUnlock() if g.city != nil && g.city.db != nil { err = g.city.db.Lookup(addr, &cityDb) if err != nil { return GeoIPRecord{} } ret.CountryCode = cityDb.Country.IsoCode ret.ContinentCode = cityDb.Continent.Code ret.City = cityDb.City.Names.English ret.Country = cityDb.Country.Names.English ret.Latitude = float32(cityDb.Location.Latitude) ret.Longitude = float32(cityDb.Location.Longitude) } if g.asn != nil && g.asn.db != nil { err = g.asn.db.Lookup(addr, &asnDb) if err != nil { return GeoIPRecord{} } ret.ASName = asnDb.AutonomousSystemOrg ret.ASNum = asnDb.AutonomousSystemNumber } return ret } // IsIPv6 returns true if the given address is of version 6 func (g *GeoIP) IsIPv6(ip string) bool { return strings.Contains(ip, ":") } // IsValid returns true if the given address is valid func (g *GeoIPRecord) IsValid() bool { return len(g.CountryCode) > 0 } videolabs-mirrorbits-441567e/network/geoip_test.go000066400000000000000000000071661523530551300223330ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package network import ( "net" "reflect" "strings" "testing" "time" ) type CityDb struct { City struct { Names struct { En string } } Country struct { Iso_Code string Names struct { En string } } Continent struct { Code string } Location struct { Latitude float64 Longitude float64 } } type ASNDb struct { Autonomous_system_number uint Autonomous_system_organization string } func TestNewGeoIP(t *testing.T) { g := NewGeoIP() if g == nil { t.Fatalf("Expected valid pointer, got nil") } } func TestGeoIP_GetRecord(t *testing.T) { g := NewGeoIP() mockcity := &geoipDB{ filename: "city.mmdb", modTime: time.Now(), db: &GeoIPMockCity{}, } mockasn := &geoipDB{ filename: "asn.mmdb", modTime: time.Now(), db: &GeoIPMockASN{}, } g.city = mockcity g.asn = mockasn /* city */ r := g.GetRecord("127.0.0.1") if r.City != "test1" { t.Fatalf("Invalid response got %s, expected test1", r.City) } if r.CountryCode != "test2" { t.Fatalf("Invalid response got %s, expected test2", r.CountryCode) } if r.Country != "test3" { t.Fatalf("Invalid response got %s, expected test3", r.Country) } if r.ContinentCode != "test4" { t.Fatalf("Invalid response got %s, expected test4", r.ContinentCode) } if r.Latitude != 24 { t.Fatalf("Invalid response got %f, expected 24", r.Latitude) } if r.Longitude != 42 { t.Fatalf("Invalid response got %f, expected 42", r.Longitude) } if r.ASNum != 42 { t.Fatalf("Invalid response got %d, expected 42", r.ASNum) } if r.ASName != "forty two" { t.Fatalf("Invalid response got %s, expected forty two", r.ASName) } } func TestIsIPv6(t *testing.T) { g := NewGeoIP() if g.IsIPv6("192.168.0.1") == true { t.Fatalf("Expected ipv4, got ipv6") } if g.IsIPv6("::1") == false { t.Fatalf("Expected ipv6, got ipv4") } if g.IsIPv6("fe80::801a:2cff:fe80:315c") == false { t.Fatalf("Expected ipv6, got ipv4") } } func TestGeoIPRecord_IsValid(t *testing.T) { var r GeoIPRecord if r.IsValid() == true { t.Fatalf("Expected false, got true") } r = GeoIPRecord{ CountryCode: "FR", } if r.IsValid() == false { t.Fatalf("Expected true, got false") } } /* MOCK */ type GeoIPMockCity struct { } func (g *GeoIPMockCity) Lookup(ipAddress net.IP, result any) error { var citydb CityDb citydb.City.Names.En = "test1" citydb.Country.Iso_Code = "test2" citydb.Country.Names.En = "test3" citydb.Continent.Code = "test4" citydb.Location.Latitude = 24 citydb.Location.Longitude = 42 CopyStruct(&citydb, result) return nil } type GeoIPMockASN struct { } func (g *GeoIPMockASN) Lookup(ipAddress net.IP, result any) error { var asnDb ASNDb asnDb.Autonomous_system_number = 42 asnDb.Autonomous_system_organization = "forty two" CopyStruct(&asnDb, result) return nil } func CopyStruct(src any, dst any) { s := reflect.Indirect(reflect.ValueOf(src)) d := reflect.Indirect(reflect.ValueOf(dst)) CopyStructRec(s, d) } func CopyStructRec(s, d reflect.Value) { st := s.Type() dt := d.Type() typeOft1 := s.Type() typeOft2 := d.Type() for i := 0; i < s.NumField(); i++ { sf := s.Field(i) if st.Field(i).Type.Kind() == reflect.Struct { for j := 0; j < d.NumField(); j++ { if typeOft1.Field(i).Name == typeOft2.Field(j).Name { CopyStructRec(s.Field(i), d.Field(j)) goto cont } } } for j := 0; j < d.NumField(); j++ { df := d.Field(j) dtf := dt.Field(j) dsttag := dtf.Tag.Get("maxminddb") if strings.ToLower(typeOft1.Field(i).Name) == strings.ToLower(dsttag) { df.Set(reflect.Value(sf)) break } } cont: } } videolabs-mirrorbits-441567e/network/utils.go000066400000000000000000000035321523530551300213220ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package network import ( "net" "strings" ) // LookupMirrorIP returns the IP address of a mirror and returns an error // if the DNS has more than one address func LookupMirrorIP(host string) (string, error) { addrs, err := net.LookupIP(host) if err != nil { return "", err } // A mirror with multiple IP address is a problem // since we can't determine the exact position of // the server. if len(addrs) > 1 { err = ErrMultipleAddresses } return addrs[0].String(), err } // RemoteIPFromAddr removes the port from a remote address (x.x.x.x:yyyy) func RemoteIPFromAddr(remoteAddr string) string { return remoteAddr[:strings.LastIndex(remoteAddr, ":")] } // ExtractRemoteIP extracts the remote IP from an X-Forwarded-For header func ExtractRemoteIP(XForwardedFor string) string { addresses := strings.Split(XForwardedFor, ",") if len(addresses) > 0 { // The left-most address is supposed to be the original client address. // Each successive are added by proxies. In most cases we should probably // take the last address but in case of optimization services this will // probably not work. For now we'll always take the original one. return strings.TrimSpace(addresses[0]) } return "" } // IsPrimaryCountry returns true if the clientInfo country is the primary country func IsPrimaryCountry(clientInfo GeoIPRecord, list []string) bool { if !clientInfo.IsValid() { return false } if len(list) > 0 && list[0] == clientInfo.CountryCode { return true } return false } // IsAdditionalCountry returns true if the clientInfo country is in list func IsAdditionalCountry(clientInfo GeoIPRecord, list []string) bool { if !clientInfo.IsValid() { return false } for i, b := range list { if i > 0 && b == clientInfo.CountryCode { return true } } return false } videolabs-mirrorbits-441567e/network/utils_test.go000066400000000000000000000030141523530551300223540ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package network import ( "testing" ) func TestRemoteIpFromAddr(t *testing.T) { r := RemoteIPFromAddr("127.0.0.1:8080") if r != "127.0.0.1" { t.Fatalf("Expected '127.0.0.1', got %s", r) } r = RemoteIPFromAddr("[::1]:8080") if r != "[::1]" { t.Fatalf("Expected '[::1]', got %s", r) } r = RemoteIPFromAddr(":8080") if r != "" { t.Fatalf("Expected '', got %s", r) } } func TestExtractRemoteIP(t *testing.T) { r := ExtractRemoteIP("192.168.0.1, 192.168.0.2, 192.168.0.3") if r != "192.168.0.1" { t.Fatalf("Expected '192.168.0.1', got %s", r) } r = ExtractRemoteIP("192.168.0.1,192.168.0.2,192.168.0.3") if r != "192.168.0.1" { t.Fatalf("Expected '192.168.0.1', got %s", r) } } func TestIsPrimaryCountry(t *testing.T) { var b bool list := []string{"FR", "DE", "GR"} clientInfo := GeoIPRecord{ CountryCode: "FR", } b = IsPrimaryCountry(clientInfo, list) if !b { t.Fatal("Expected true, got false") } clientInfo = GeoIPRecord{ CountryCode: "GR", } b = IsPrimaryCountry(clientInfo, list) if b { t.Fatal("Expected false, got true") } } func TestIsAdditionalCountry(t *testing.T) { var b bool list := []string{"FR", "DE", "GR"} clientInfo := GeoIPRecord{ CountryCode: "FR", } b = IsAdditionalCountry(clientInfo, list) if b { t.Fatal("Expected false, got true") } clientInfo = GeoIPRecord{ CountryCode: "GR", } b = IsAdditionalCountry(clientInfo, list) if !b { t.Fatal("Expected true, got false") } } videolabs-mirrorbits-441567e/process/000077500000000000000000000000001523530551300176155ustar00rootroot00000000000000videolabs-mirrorbits-441567e/process/process.go000066400000000000000000000106301523530551300216220ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package process import ( "errors" "fmt" "net" "os" "os/exec" "path" "strconv" "syscall" "github.com/etix/mirrorbits/core" "github.com/op/go-logging" ) var ( // Compile time variable defaultPidFile string ) var ( // ErrInvalidfd is returned when the given file descriptor is invalid ErrInvalidfd = errors.New("invalid file descriptor") log = logging.MustGetLogger("main") ) // Relaunch launches {self} as a child process passing listener details // to provide a seamless binary upgrade. func Relaunch(l net.Listener) error { argv0, err := exec.LookPath(os.Args[0]) if err != nil { return err } if _, err := os.Stat(argv0); err != nil { return err } wd, err := os.Getwd() if err != nil { return err } var file *os.File switch t := l.(type) { case *net.TCPListener: file, err = t.File() case *net.UnixListener: file, err = t.File() default: return ErrInvalidfd } if err != nil { return err } fd := file.Fd() sysfile := file.Name() listener, ok := l.(*net.TCPListener) if ok { listenerFile, err := listener.File() if err != nil { return err } fd = listenerFile.Fd() sysfile = listenerFile.Name() } if fd < uintptr(syscall.Stderr) { return ErrInvalidfd } if err := os.Setenv("OLD_FD", fmt.Sprint(fd)); err != nil { return err } if err := os.Setenv("OLD_NAME", fmt.Sprintf("tcp:%s->", l.Addr().String())); err != nil { return err } if err := os.Setenv("OLD_PPID", fmt.Sprint(syscall.Getpid())); err != nil { return err } files := make([]*os.File, fd+1) files[syscall.Stdin] = os.Stdin files[syscall.Stdout] = os.Stdout files[syscall.Stderr] = os.Stderr files[fd] = os.NewFile(fd, sysfile) p, err := os.StartProcess(argv0, os.Args, &os.ProcAttr{ Dir: wd, Env: os.Environ(), Files: files, Sys: &syscall.SysProcAttr{}, }) if err != nil { return err } log.Infof("Spawned child %d\n", p.Pid) return nil } // Recover from a seamless binary upgrade and use an already // existing listener to take over the connections func Recover() (l net.Listener, ppid int, err error) { var fd uintptr _, err = fmt.Sscan(os.Getenv("OLD_FD"), &fd) if err != nil { return } var i net.Listener i, err = net.FileListener(os.NewFile(fd, os.Getenv("OLD_NAME"))) if err != nil { return } switch i.(type) { case *net.TCPListener: l = i.(*net.TCPListener) case *net.UnixListener: l = i.(*net.UnixListener) default: err = fmt.Errorf("file descriptor is %T not *net.TCPListener or *net.UnixListener", i) return } if err = syscall.Close(int(fd)); err != nil { return } _, err = fmt.Sscan(os.Getenv("OLD_PPID"), &ppid) if err != nil { return } return } // KillParent sends a signal to make the parent exit gracefully with SIGQUIT func KillParent(ppid int) error { log.Info("Asking parent to quit") return syscall.Kill(ppid, syscall.SIGQUIT) } // GetPidLocation finds the location to store our pid file // and fallback to /run if none found func GetPidLocation() string { if core.PidFile == "" { // Runtime rdir := os.Getenv("XDG_RUNTIME_DIR") if rdir == "" { if defaultPidFile == "" { // Compile time return "/run/mirrorbits/mirrorbits.pid" // Fallback } return defaultPidFile } return rdir + "/mirrorbits.pid" } return core.PidFile } // WritePidFile writes the current pid file to disk func WritePidFile() { // Get the pid destination p := GetPidLocation() // Create the whole directory path if err := os.MkdirAll(path.Dir(p), 0755); err != nil { log.Errorf("Unable to write pid file: %v", err) } // Get our own PID and write it pid := strconv.Itoa(os.Getpid()) if err := os.WriteFile(p, []byte(pid), 0644); err != nil { log.Errorf("Unable to write pid file: %v", err) } } // RemovePidFile removes the current pid file func RemovePidFile() { pidFile := GetPidLocation() if _, err := os.Stat(pidFile); !os.IsNotExist(err) { // Ensures we don't remove our forked process pid file // This can happen during seamless binary upgrade if GetRemoteProcPid() == os.Getpid() { if err = os.Remove(pidFile); err != nil { log.Errorf("Unable to remove pid file: %v", err) } } } } // GetRemoteProcPid gets the pid as it appears in the pid file (maybe not ours) func GetRemoteProcPid() int { b, err := os.ReadFile(GetPidLocation()) if err != nil { return -1 } i, err := strconv.ParseInt(string(b), 10, 0) if err != nil { return -1 } return int(i) } videolabs-mirrorbits-441567e/rpc/000077500000000000000000000000001523530551300167235ustar00rootroot00000000000000videolabs-mirrorbits-441567e/rpc/interceptors.go000066400000000000000000000016751523530551300220040ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package rpc import ( "context" . "github.com/etix/mirrorbits/config" grpc "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) func StreamInterceptor(srv any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { if err := authorize(stream.Context()); err != nil { return err } return handler(srv, stream) } func UnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if err := authorize(ctx); err != nil { return nil, err } return handler(ctx, req) } func authorize(ctx context.Context) error { if md, ok := metadata.FromIncomingContext(ctx); ok { if md["password"][0] == GetConfig().RPCPassword { return nil } } return status.Error(codes.Unauthenticated, "access denied") } videolabs-mirrorbits-441567e/rpc/rpc.go000066400000000000000000000520151523530551300200410ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package rpc import ( "errors" "fmt" "log" "net" "net/url" "os" "regexp" "runtime" "strconv" "strings" "sync" "syscall" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/network" "github.com/etix/mirrorbits/scan" "github.com/etix/mirrorbits/utils" "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/empty" "github.com/gomodule/redigo/redis" context "golang.org/x/net/context" grpc "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/reflection" "google.golang.org/grpc/status" "gopkg.in/yaml.v3" ) var ( // ErrNameAlreadyTaken is returned when the request name is already taken by another mirror ErrNameAlreadyTaken = errors.New("name already taken") ) // CLI object handles the server side RPC of the CLI type CLI struct { listener net.Listener server *grpc.Server sig chan<- os.Signal redis *database.Redis cache *mirrors.Cache } func (c *CLI) Start() error { var err error c.listener, err = net.Listen("tcp", GetConfig().RPCListenAddress) if err != nil { return err } c.server = grpc.NewServer( grpc.UnaryInterceptor(UnaryInterceptor), grpc.StreamInterceptor(StreamInterceptor), ) RegisterCLIServer(c.server, c) reflection.Register(c.server) go func() { if err := c.server.Serve(c.listener); err != nil { log.Fatalf("failed to serve rpc: %v", err) } }() return nil } func (c *CLI) Close() error { c.server.Stop() return c.listener.Close() } func (c *CLI) SetSignals(sig chan<- os.Signal) { c.sig = sig } func (c *CLI) SetDatabase(r *database.Redis) { c.redis = r } func (c *CLI) SetCache(cache *mirrors.Cache) { c.cache = cache } func (c *CLI) Ping(context.Context, *empty.Empty) (*empty.Empty, error) { return &empty.Empty{}, nil } func (c *CLI) GetVersion(context.Context, *empty.Empty) (*VersionReply, error) { return &VersionReply{ Version: core.VERSION, Build: core.BUILD + core.DEV, GoVersion: runtime.Version(), OS: runtime.GOOS, Arch: runtime.GOARCH, GoMaxProcs: int32(runtime.GOMAXPROCS(0)), }, nil } func (c *CLI) Upgrade(ctx context.Context, in *empty.Empty) (*empty.Empty, error) { select { case c.sig <- syscall.SIGUSR2: default: return nil, status.Error(codes.Internal, "signal handler not ready") } return &empty.Empty{}, nil } func (c *CLI) Reload(ctx context.Context, in *empty.Empty) (*empty.Empty, error) { select { case c.sig <- syscall.SIGHUP: default: return nil, status.Error(codes.Internal, "signal handler not ready") } return &empty.Empty{}, nil } func (c *CLI) MatchMirror(ctx context.Context, in *MatchRequest) (*MatchReply, error) { if c.redis == nil { return nil, status.Error(codes.Internal, "database not ready") } mirrors, err := c.redis.GetListOfMirrors() if err != nil { return nil, fmt.Errorf("can't fetch the list of mirrors: %w", err) } reply := &MatchReply{ Mirrors: matchMirrorsByPattern(mirrors, in.Pattern), } return reply, nil } // matchMirrorsByPattern returns a list of mirrors: // - if the pattern matches a mirror's name exactly, only that mirror is returned // - otherwise, all mirrors containing the pattern as a substring are returned // - all matches are case-insensitive // This allows a mirror whose name is a substring of other mirror names // (e.g. "fcix.net" vs. "mirror.fcix.net") to still be matched unambiguously. func matchMirrorsByPattern(mirrors map[int]string, pattern string) []*MirrorID { lowerPattern := strings.ToLower(pattern) var matches []*MirrorID for id, name := range mirrors { lowerName := strings.ToLower(name) if lowerName == lowerPattern { return []*MirrorID{{ ID: int32(id), Name: name, }} } if strings.Contains(lowerName, lowerPattern) { matches = append(matches, &MirrorID{ ID: int32(id), Name: name, }) } } return matches } func (c *CLI) ChangeStatus(ctx context.Context, in *ChangeStatusRequest) (*empty.Empty, error) { if in.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } var err error switch in.Enabled { case true: err = mirrors.EnableMirror(c.redis, int(in.ID)) case false: err = mirrors.DisableMirror(c.redis, int(in.ID)) } return &empty.Empty{}, err } func (c *CLI) List(ctx context.Context, in *empty.Empty) (*MirrorListReply, error) { conn, err := c.redis.Connect() if err != nil { return nil, err } defer conn.Close() mirrorsIDs, err := c.redis.GetListOfMirrors() if err != nil { return nil, fmt.Errorf("can't fetch the list of mirrors: %w", err) } conn.Send("MULTI") for id := range mirrorsIDs { conn.Send("HGETALL", fmt.Sprintf("MIRROR_%d", id)) } res, err := redis.Values(conn.Do("EXEC")) if err != nil { return nil, fmt.Errorf("database error: %w", err) } reply := &MirrorListReply{} for _, e := range res { var mirror mirrors.Mirror res, ok := e.([]any) if !ok { return nil, errors.New("typecast failed") } err = redis.ScanStruct([]any(res), &mirror) if err != nil { return nil, fmt.Errorf("scan struct failed: %w", err) } m, err := MirrorToRPC(&mirror) if err != nil { return nil, err } reply.Mirrors = append(reply.Mirrors, m) } return reply, nil } func (c *CLI) MirrorInfo(ctx context.Context, in *MirrorIDRequest) (*Mirror, error) { if in.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } conn, err := c.redis.Connect() if err != nil { return nil, err } defer conn.Close() m, err := redis.Values(conn.Do("HGETALL", fmt.Sprintf("MIRROR_%d", in.ID))) if err != nil { return nil, err } var mi mirrors.Mirror err = redis.ScanStruct(m, &mi) if err != nil { return nil, err } rpcm, err := MirrorToRPC(&mi) if err != nil { return nil, err } return rpcm, nil } func (c *CLI) GeoUpdateMirror(ctx context.Context, in *MirrorIDRequest) (*GeoUpdateMirrorReply, error) { if in.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } conn, err := c.redis.Connect() if err != nil { return nil, err } defer conn.Close() m, err := redis.Values(conn.Do("HGETALL", fmt.Sprintf("MIRROR_%d", in.ID))) if err != nil { return nil, err } var mirror mirrors.Mirror err = redis.ScanStruct(m, &mirror) if err != nil { return nil, err } var u *url.URL if utils.HasAnyPrefix(mirror.HttpURL, "http://", "https://") { u, err = url.Parse(mirror.HttpURL) } else { u, err = url.Parse("http://" + mirror.HttpURL) } if err != nil { return nil, fmt.Errorf("can't parse http url: %w", err) } reply := &GeoUpdateMirrorReply{} ip, err := network.LookupMirrorIP(u.Host) if err == network.ErrMultipleAddresses { reply.Warnings = append(reply.Warnings, "Warning: the hostname returned more than one address. Assuming they're sharing the same location.") } else if err != nil { return nil, fmt.Errorf("IP lookup failed: %w", err) } geo := network.NewGeoIP() if err := geo.LoadGeoIP(); err != nil { return nil, err } geoRec := geo.GetRecord(ip) if geoRec.IsValid() { original := mirror mirror.Latitude = geoRec.Latitude mirror.Longitude = geoRec.Longitude mirror.Asnum = geoRec.ASNum // We need to sanitize, as we're going to do a diff below, // and the mirror fields are sanitized. continent := utils.SanitizeLocationCodes(geoRec.ContinentCode) country := utils.SanitizeLocationCodes(geoRec.CountryCode) // ContinentCode is the easy one. mirror.ContinentCode = continent // CountryCodes needs special care: it might have been modified // by user in order to list the countries that this mirror is // expected to serve. Therefore we check if the country from // the GeoIP record is included in the mirror country(ies), and // if that's the case we don't touch it. if !utils.IsInSlice(country, strings.Fields(mirror.CountryCodes)) { mirror.CountryCodes = country } reply.Mirror, err = MirrorToRPC(&mirror) if err != nil { return nil, err } reply.Diff = createDiff(&original, &mirror) } else { reply.Warnings = append(reply.Warnings, "Warning: unable to guess the geographic location of this mirror") } return reply, nil } func (c *CLI) AddMirror(ctx context.Context, in *Mirror) (*AddMirrorReply, error) { mirror, err := MirrorFromRPC(in) if err != nil { return nil, err } if mirror.ID != 0 { return nil, status.Error(codes.FailedPrecondition, "unexpected ID") } var u *url.URL if utils.HasAnyPrefix(mirror.HttpURL, "http://", "https://") { u, err = url.Parse(mirror.HttpURL) } else { u, err = url.Parse("http://" + mirror.HttpURL) } if err != nil { return nil, fmt.Errorf("can't parse http url: %w", err) } reply := &AddMirrorReply{} ip, err := network.LookupMirrorIP(u.Host) if err == network.ErrMultipleAddresses { reply.Warnings = append(reply.Warnings, "Warning: the hostname returned more than one address. Assuming they're sharing the same location.") } else if err != nil { return nil, fmt.Errorf("IP lookup failed: %w", err) } geo := network.NewGeoIP() if err := geo.LoadGeoIP(); err != nil { return nil, err } geoRec := geo.GetRecord(ip) if geoRec.IsValid() { mirror.Latitude = geoRec.Latitude mirror.Longitude = geoRec.Longitude mirror.ContinentCode = geoRec.ContinentCode mirror.CountryCodes = geoRec.CountryCode mirror.Asnum = geoRec.ASNum reply.Latitude = geoRec.Latitude reply.Longitude = geoRec.Longitude reply.Continent = geoRec.ContinentCode reply.Country = geoRec.Country reply.ASN = fmt.Sprintf("%s (%d)", geoRec.ASName, geoRec.ASNum) } else { reply.Warnings = append(reply.Warnings, "Warning: unable to guess the geographic location of this mirror") } return reply, c.setMirror(mirror) } func (c *CLI) UpdateMirror(ctx context.Context, in *Mirror) (*UpdateMirrorReply, error) { mirror, err := MirrorFromRPC(in) if err != nil { return nil, err } if mirror.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } conn, err := c.redis.Connect() if err != nil { return &UpdateMirrorReply{}, err } defer conn.Close() m, err := redis.Values(conn.Do("HGETALL", fmt.Sprintf("MIRROR_%d", mirror.ID))) if err != nil { return nil, err } var original mirrors.Mirror err = redis.ScanStruct(m, &original) if err != nil { return nil, err } diff := createDiff(&original, mirror) return &UpdateMirrorReply{ Diff: diff, }, c.setMirror(mirror) } func createDiff(mirror1, mirror2 *mirrors.Mirror) (out string) { yamlo, _ := yaml.Marshal(mirror1) yamln, _ := yaml.Marshal(mirror2) splito := strings.Split(string(yamlo), "\n") splitn := strings.Split(string(yamln), "\n") for i, l := range splito { if l != splitn[i] { out += fmt.Sprintf("- %s\n+ %s\n", l, splitn[i]) } } return } func (c *CLI) setMirror(mirror *mirrors.Mirror) error { conn, err := c.redis.Connect() if err != nil { return err } defer conn.Close() mirrorsIDs, err := c.redis.GetListOfMirrors() if err != nil { return fmt.Errorf("can't fetch the list of mirrors: %w", err) } isUpdate := false for id, name := range mirrorsIDs { if id == mirror.ID { isUpdate = true } if mirror.ID != id && name == mirror.Name { return ErrNameAlreadyTaken } } if mirror.ID <= 0 { // Generate a new ID mirror.ID, err = redis.Int(conn.Do("INCR", "LAST_MID")) if err != nil { return fmt.Errorf("failed creating a new id: %w", err) } } // Reformat contry codes mirror.CountryCodes = utils.SanitizeLocationCodes(mirror.CountryCodes) mirror.ExcludedCountryCodes = utils.SanitizeLocationCodes(mirror.ExcludedCountryCodes) // Reformat continent code mirror.ContinentCode = utils.SanitizeLocationCodes(mirror.ContinentCode) // Normalize URLs mirror.HttpURL = utils.NormalizeURL(mirror.HttpURL) mirror.RsyncURL = utils.NormalizeURL(mirror.RsyncURL) mirror.FtpURL = utils.NormalizeURL(mirror.FtpURL) // Save the values back into redis conn.Send("MULTI") conn.Send("HSET", fmt.Sprintf("MIRROR_%d", mirror.ID), "ID", mirror.ID, "name", mirror.Name, "http", mirror.HttpURL, "rsync", mirror.RsyncURL, "ftp", mirror.FtpURL, "sponsorName", mirror.SponsorName, "sponsorURL", mirror.SponsorURL, "sponsorLogo", mirror.SponsorLogoURL, "adminName", mirror.AdminName, "adminEmail", mirror.AdminEmail, "customData", mirror.CustomData, "continentOnly", mirror.ContinentOnly, "countryOnly", mirror.CountryOnly, "asOnly", mirror.ASOnly, "score", mirror.Score, "latitude", mirror.Latitude, "longitude", mirror.Longitude, "continentCode", mirror.ContinentCode, "countryCodes", mirror.CountryCodes, "excludedCountryCodes", mirror.ExcludedCountryCodes, "asnum", mirror.Asnum, "comment", mirror.Comment, "allowredirects", mirror.AllowRedirects, "enabled", mirror.Enabled) // Reset state to down for unsupported protocol if strings.HasPrefix(mirror.HttpURL, "http://") { conn.Send("HSET", fmt.Sprintf("MIRROR_%d", mirror.ID), "httpsUp", false) } else if strings.HasPrefix(mirror.HttpURL, "https://") { conn.Send("HSET", fmt.Sprintf("MIRROR_%d", mirror.ID), "httpUp", false) } // The name of the mirror has been changed. conn.Send("HSET", "MIRRORS", mirror.ID, mirror.Name) _, err = conn.Do("EXEC") if err != nil { return fmt.Errorf("couldn't save the mirror configuration: %w", err) } // Publish update database.Publish(conn, database.MIRROR_UPDATE, strconv.Itoa(mirror.ID)) if isUpdate { // This was an update of an existing mirror mirrors.PushLog(c.redis, mirrors.NewLogEdited(mirror.ID)) } else { // We just added a new mirror mirrors.PushLog(c.redis, mirrors.NewLogAdded(mirror.ID)) } return nil } func (c *CLI) RemoveMirror(ctx context.Context, in *MirrorIDRequest) (*empty.Empty, error) { if in.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } conn, err := c.redis.Connect() if err != nil { return nil, err } defer conn.Close() // First disable the mirror err = mirrors.DisableMirror(c.redis, int(in.ID)) if err != nil { return nil, fmt.Errorf("unable to disable the mirror: %w", err) } // Get all files supported by the given mirror files, err := redis.Strings(conn.Do("SMEMBERS", fmt.Sprintf("MIRRORFILES_%d", in.ID))) if err != nil { return nil, fmt.Errorf("unable to fetch the file list: %w", err) } conn.Send("MULTI") // Remove each FILEINFO / FILEMIRRORS for _, file := range files { conn.Send("DEL", fmt.Sprintf("FILEINFO_%d_%s", in.ID, file)) conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", file), in.ID) conn.Send("PUBLISH", database.MIRROR_FILE_UPDATE, fmt.Sprintf("%d %s", in.ID, file)) } // Remove all other keys conn.Send("DEL", fmt.Sprintf("MIRROR_%d", in.ID), fmt.Sprintf("MIRRORFILES_%d", in.ID), fmt.Sprintf("MIRRORFILESTMP_%d", in.ID), fmt.Sprintf("HANDLEDFILES_%d", in.ID), fmt.Sprintf("SCANNING_%d", in.ID), fmt.Sprintf("MIRRORLOGS_%d", in.ID)) // Remove the last reference conn.Send("HDEL", "MIRRORS", in.ID) _, err = conn.Do("EXEC") if err != nil { return nil, fmt.Errorf("operation failed: %w", err) } // Publish update database.Publish(conn, database.MIRROR_UPDATE, strconv.Itoa(int(in.ID))) return &empty.Empty{}, nil } func (c *CLI) RefreshRepository(ctx context.Context, in *RefreshRepositoryRequest) (*empty.Empty, error) { return &empty.Empty{}, scan.ScanSource(c.redis, in.Rehash, nil) } func (c *CLI) ScanMirror(ctx context.Context, in *ScanMirrorRequest) (*ScanMirrorReply, error) { if in.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } conn, err := c.redis.Connect() if err != nil { return nil, err } defer conn.Close() // Check if the local repository has been scanned already exists, err := redis.Bool(conn.Do("EXISTS", "FILES")) if err != nil { return nil, err } if !exists { return nil, status.Error(codes.FailedPrecondition, "local repository not yet indexed. You should run 'refresh' first!") } key := fmt.Sprintf("MIRROR_%d", in.ID) m, err := redis.Values(conn.Do("HGETALL", key)) if err != nil { return nil, err } var mirror mirrors.Mirror err = redis.ScanStruct(m, &mirror) if err != nil { return nil, err } var wg sync.WaitGroup trace := scan.NewTraceHandler(c.redis, make(<-chan struct{})) wg.Add(1) go func() { defer wg.Done() err := trace.GetLastUpdate(mirror) if err != nil && err != scan.ErrNoTrace { var numError *strconv.NumError if errors.As(err, &numError) { if numError.Err == strconv.ErrSyntax { //log.Warningf("[%s] parsing trace file failed: %s is not a valid timestamp", mirror.Name, strconv.Quote(numError.Num)) return } } else { //log.Warningf("[%s] fetching trace file failed: %s", mirror.Name, err) } } }() err = scan.ErrNoSyncMethod var res *scan.ScanResult if in.Protocol == ScanMirrorRequest_ALL { // Use rsync (if applicable) and fallback to FTP if mirror.RsyncURL != "" { res, err = scan.Scan(core.RSYNC, c.redis, c.cache, mirror.RsyncURL, mirror.ID, ctx.Done()) } if err != nil && mirror.FtpURL != "" { res, err = scan.Scan(core.FTP, c.redis, c.cache, mirror.FtpURL, mirror.ID, ctx.Done()) } } else { // Use the requested protocol if in.Protocol == ScanMirrorRequest_RSYNC && mirror.RsyncURL != "" { res, err = scan.Scan(core.RSYNC, c.redis, c.cache, mirror.RsyncURL, mirror.ID, ctx.Done()) } else if in.Protocol == ScanMirrorRequest_FTP && mirror.FtpURL != "" { res, err = scan.Scan(core.FTP, c.redis, c.cache, mirror.FtpURL, mirror.ID, ctx.Done()) } } if err != nil { return nil, errors.New(fmt.Sprintf("scanning %s failed: %s", mirror.Name, err)) } reply := &ScanMirrorReply{ FilesIndexed: res.FilesIndexed, KnownIndexed: res.KnownIndexed, Removed: res.Removed, TZOffsetMs: res.TZOffsetMs, } // Finally enable the mirror if requested if err == nil && in.AutoEnable == true { if err := mirrors.EnableMirror(c.redis, mirror.ID); err != nil { return nil, fmt.Errorf("couldn't enable the mirror: %w", err) } reply.Enabled = true } wg.Wait() return reply, nil } func (c *CLI) StatsFile(ctx context.Context, in *StatsFileRequest) (*StatsFileReply, error) { conn, err := c.redis.Connect() if err != nil { return nil, err } defer conn.Close() // Convert the timestamps start, err := ptypes.Timestamp(in.DateStart) if err != nil { return nil, err } end, err := ptypes.Timestamp(in.DateEnd) if err != nil { return nil, err } // Compile the regex pattern re, err := regexp.Compile(in.Pattern) if err != nil { return nil, err } // Generate the list of redis key for the period tkcoverage := utils.TimeKeyCoverage(start, end) // Prepare the transaction conn.Send("MULTI") for _, k := range tkcoverage { conn.Send("HGETALL", "STATS_FILE_"+k) } stats, err := redis.Values(conn.Do("EXEC")) if err != nil { return nil, fmt.Errorf("can't fetch stats: %w", err) } reply := &StatsFileReply{ Files: make(map[string]int64), } for _, res := range stats { line, ok := res.([]any) if !ok { return nil, errors.New("typecast failed") } stats := []any(line) for i := 0; i < len(stats); i += 2 { path, _ := redis.String(stats[i], nil) matched := re.MatchString(path) if matched { reqs, _ := redis.Int64(stats[i+1], nil) reply.Files[path] += reqs } } } return reply, nil } func (c *CLI) StatsMirror(ctx context.Context, in *StatsMirrorRequest) (*StatsMirrorReply, error) { if in.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } conn, err := c.redis.Connect() if err != nil { return nil, err } defer conn.Close() // Convert the timestamps start, err := ptypes.Timestamp(in.DateStart) if err != nil { return nil, err } end, err := ptypes.Timestamp(in.DateEnd) if err != nil { return nil, err } // Generate the list of redis key for the period tkcoverage := utils.TimeKeyCoverage(start, end) conn.Send("MULTI") // Fetch the stats for _, k := range tkcoverage { conn.Send("HGET", "STATS_MIRROR_"+k, in.ID) conn.Send("HGET", "STATS_MIRROR_BYTES_"+k, in.ID) } stats, err := redis.Strings(conn.Do("EXEC")) if err != nil { return nil, fmt.Errorf("can't fetch stats: %w", err) } // Fetch the mirror struct m, err := redis.Values(conn.Do("HGETALL", fmt.Sprintf("MIRROR_%d", in.ID))) if err != nil { return nil, fmt.Errorf("can't fetch mirror: %w", err) } reply := &StatsMirrorReply{} var mirror mirrors.Mirror err = redis.ScanStruct(m, &mirror) if err != nil { return nil, fmt.Errorf("stats error: %w", err) } reply.Mirror, err = MirrorToRPC(&mirror) if err != nil { return nil, fmt.Errorf("stats error: %w", err) } for i := 0; i < len(stats); i += 2 { v1, _ := strconv.ParseInt(stats[i], 10, 64) v2, _ := strconv.ParseInt(stats[i+1], 10, 64) reply.Requests += v1 reply.Bytes += v2 } return reply, nil } func (c *CLI) GetMirrorLogs(ctx context.Context, in *GetMirrorLogsRequest) (*GetMirrorLogsReply, error) { if in.ID <= 0 { return nil, status.Error(codes.FailedPrecondition, "invalid mirror id") } lines, err := mirrors.ReadLogs(c.redis, int(in.ID), int(in.MaxResults)) if err != nil { return nil, fmt.Errorf("mirror logs error: %w", err) } return &GetMirrorLogsReply{Line: lines}, nil } videolabs-mirrorbits-441567e/rpc/rpc.pb.go000066400000000000000000002210751523530551300204450ustar00rootroot00000000000000// Code generated by protoc-gen-go. DO NOT EDIT. // source: rpc.proto package rpc import ( context "context" fmt "fmt" proto "github.com/golang/protobuf/proto" empty "github.com/golang/protobuf/ptypes/empty" timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" math "math" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type ScanMirrorRequest_Method int32 const ( ScanMirrorRequest_ALL ScanMirrorRequest_Method = 0 ScanMirrorRequest_FTP ScanMirrorRequest_Method = 1 ScanMirrorRequest_RSYNC ScanMirrorRequest_Method = 2 ) var ScanMirrorRequest_Method_name = map[int32]string{ 0: "ALL", 1: "FTP", 2: "RSYNC", } var ScanMirrorRequest_Method_value = map[string]int32{ "ALL": 0, "FTP": 1, "RSYNC": 2, } func (x ScanMirrorRequest_Method) String() string { return proto.EnumName(ScanMirrorRequest_Method_name, int32(x)) } func (ScanMirrorRequest_Method) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{12, 0} } type VersionReply struct { Version string `protobuf:"bytes,1,opt,name=Version,proto3" json:"Version,omitempty"` Build string `protobuf:"bytes,2,opt,name=Build,proto3" json:"Build,omitempty"` GoVersion string `protobuf:"bytes,3,opt,name=GoVersion,proto3" json:"GoVersion,omitempty"` OS string `protobuf:"bytes,4,opt,name=OS,proto3" json:"OS,omitempty"` Arch string `protobuf:"bytes,5,opt,name=Arch,proto3" json:"Arch,omitempty"` GoMaxProcs int32 `protobuf:"varint,6,opt,name=GoMaxProcs,proto3" json:"GoMaxProcs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *VersionReply) Reset() { *m = VersionReply{} } func (m *VersionReply) String() string { return proto.CompactTextString(m) } func (*VersionReply) ProtoMessage() {} func (*VersionReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{0} } func (m *VersionReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_VersionReply.Unmarshal(m, b) } func (m *VersionReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_VersionReply.Marshal(b, m, deterministic) } func (m *VersionReply) XXX_Merge(src proto.Message) { xxx_messageInfo_VersionReply.Merge(m, src) } func (m *VersionReply) XXX_Size() int { return xxx_messageInfo_VersionReply.Size(m) } func (m *VersionReply) XXX_DiscardUnknown() { xxx_messageInfo_VersionReply.DiscardUnknown(m) } var xxx_messageInfo_VersionReply proto.InternalMessageInfo func (m *VersionReply) GetVersion() string { if m != nil { return m.Version } return "" } func (m *VersionReply) GetBuild() string { if m != nil { return m.Build } return "" } func (m *VersionReply) GetGoVersion() string { if m != nil { return m.GoVersion } return "" } func (m *VersionReply) GetOS() string { if m != nil { return m.OS } return "" } func (m *VersionReply) GetArch() string { if m != nil { return m.Arch } return "" } func (m *VersionReply) GetGoMaxProcs() int32 { if m != nil { return m.GoMaxProcs } return 0 } type MatchRequest struct { Pattern string `protobuf:"bytes,1,opt,name=Pattern,proto3" json:"Pattern,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MatchRequest) Reset() { *m = MatchRequest{} } func (m *MatchRequest) String() string { return proto.CompactTextString(m) } func (*MatchRequest) ProtoMessage() {} func (*MatchRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{1} } func (m *MatchRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MatchRequest.Unmarshal(m, b) } func (m *MatchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MatchRequest.Marshal(b, m, deterministic) } func (m *MatchRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MatchRequest.Merge(m, src) } func (m *MatchRequest) XXX_Size() int { return xxx_messageInfo_MatchRequest.Size(m) } func (m *MatchRequest) XXX_DiscardUnknown() { xxx_messageInfo_MatchRequest.DiscardUnknown(m) } var xxx_messageInfo_MatchRequest proto.InternalMessageInfo func (m *MatchRequest) GetPattern() string { if m != nil { return m.Pattern } return "" } type Mirror struct { ID int32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` HttpURL string `protobuf:"bytes,3,opt,name=HttpURL,proto3" json:"HttpURL,omitempty"` RsyncURL string `protobuf:"bytes,4,opt,name=RsyncURL,proto3" json:"RsyncURL,omitempty"` FtpURL string `protobuf:"bytes,5,opt,name=FtpURL,proto3" json:"FtpURL,omitempty"` SponsorName string `protobuf:"bytes,6,opt,name=SponsorName,proto3" json:"SponsorName,omitempty"` SponsorURL string `protobuf:"bytes,7,opt,name=SponsorURL,proto3" json:"SponsorURL,omitempty"` SponsorLogoURL string `protobuf:"bytes,8,opt,name=SponsorLogoURL,proto3" json:"SponsorLogoURL,omitempty"` AdminName string `protobuf:"bytes,9,opt,name=AdminName,proto3" json:"AdminName,omitempty"` AdminEmail string `protobuf:"bytes,10,opt,name=AdminEmail,proto3" json:"AdminEmail,omitempty"` CustomData string `protobuf:"bytes,11,opt,name=CustomData,proto3" json:"CustomData,omitempty"` ContinentOnly bool `protobuf:"varint,12,opt,name=ContinentOnly,proto3" json:"ContinentOnly,omitempty"` CountryOnly bool `protobuf:"varint,13,opt,name=CountryOnly,proto3" json:"CountryOnly,omitempty"` ASOnly bool `protobuf:"varint,14,opt,name=ASOnly,proto3" json:"ASOnly,omitempty"` Score int32 `protobuf:"varint,15,opt,name=Score,proto3" json:"Score,omitempty"` Latitude float32 `protobuf:"fixed32,16,opt,name=Latitude,proto3" json:"Latitude,omitempty"` Longitude float32 `protobuf:"fixed32,17,opt,name=Longitude,proto3" json:"Longitude,omitempty"` ContinentCode string `protobuf:"bytes,18,opt,name=ContinentCode,proto3" json:"ContinentCode,omitempty"` CountryCodes string `protobuf:"bytes,19,opt,name=CountryCodes,proto3" json:"CountryCodes,omitempty"` ExcludedCountryCodes string `protobuf:"bytes,20,opt,name=ExcludedCountryCodes,proto3" json:"ExcludedCountryCodes,omitempty"` Asnum uint32 `protobuf:"varint,21,opt,name=Asnum,proto3" json:"Asnum,omitempty"` Comment string `protobuf:"bytes,22,opt,name=Comment,proto3" json:"Comment,omitempty"` Enabled bool `protobuf:"varint,23,opt,name=Enabled,proto3" json:"Enabled,omitempty"` HttpUp bool `protobuf:"varint,24,opt,name=HttpUp,proto3" json:"HttpUp,omitempty"` HttpDownReason string `protobuf:"bytes,25,opt,name=HttpDownReason,proto3" json:"HttpDownReason,omitempty"` StateSince *timestamp.Timestamp `protobuf:"bytes,26,opt,name=StateSince,proto3" json:"StateSince,omitempty"` AllowRedirects int32 `protobuf:"varint,27,opt,name=AllowRedirects,proto3" json:"AllowRedirects,omitempty"` LastSync *timestamp.Timestamp `protobuf:"bytes,28,opt,name=LastSync,proto3" json:"LastSync,omitempty"` LastSuccessfulSync *timestamp.Timestamp `protobuf:"bytes,29,opt,name=LastSuccessfulSync,proto3" json:"LastSuccessfulSync,omitempty"` LastModTime *timestamp.Timestamp `protobuf:"bytes,30,opt,name=LastModTime,proto3" json:"LastModTime,omitempty"` HttpsUp bool `protobuf:"varint,31,opt,name=HttpsUp,proto3" json:"HttpsUp,omitempty"` HttpsDownReason string `protobuf:"bytes,32,opt,name=HttpsDownReason,proto3" json:"HttpsDownReason,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Mirror) Reset() { *m = Mirror{} } func (m *Mirror) String() string { return proto.CompactTextString(m) } func (*Mirror) ProtoMessage() {} func (*Mirror) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{2} } func (m *Mirror) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Mirror.Unmarshal(m, b) } func (m *Mirror) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Mirror.Marshal(b, m, deterministic) } func (m *Mirror) XXX_Merge(src proto.Message) { xxx_messageInfo_Mirror.Merge(m, src) } func (m *Mirror) XXX_Size() int { return xxx_messageInfo_Mirror.Size(m) } func (m *Mirror) XXX_DiscardUnknown() { xxx_messageInfo_Mirror.DiscardUnknown(m) } var xxx_messageInfo_Mirror proto.InternalMessageInfo func (m *Mirror) GetID() int32 { if m != nil { return m.ID } return 0 } func (m *Mirror) GetName() string { if m != nil { return m.Name } return "" } func (m *Mirror) GetHttpURL() string { if m != nil { return m.HttpURL } return "" } func (m *Mirror) GetRsyncURL() string { if m != nil { return m.RsyncURL } return "" } func (m *Mirror) GetFtpURL() string { if m != nil { return m.FtpURL } return "" } func (m *Mirror) GetSponsorName() string { if m != nil { return m.SponsorName } return "" } func (m *Mirror) GetSponsorURL() string { if m != nil { return m.SponsorURL } return "" } func (m *Mirror) GetSponsorLogoURL() string { if m != nil { return m.SponsorLogoURL } return "" } func (m *Mirror) GetAdminName() string { if m != nil { return m.AdminName } return "" } func (m *Mirror) GetAdminEmail() string { if m != nil { return m.AdminEmail } return "" } func (m *Mirror) GetCustomData() string { if m != nil { return m.CustomData } return "" } func (m *Mirror) GetContinentOnly() bool { if m != nil { return m.ContinentOnly } return false } func (m *Mirror) GetCountryOnly() bool { if m != nil { return m.CountryOnly } return false } func (m *Mirror) GetASOnly() bool { if m != nil { return m.ASOnly } return false } func (m *Mirror) GetScore() int32 { if m != nil { return m.Score } return 0 } func (m *Mirror) GetLatitude() float32 { if m != nil { return m.Latitude } return 0 } func (m *Mirror) GetLongitude() float32 { if m != nil { return m.Longitude } return 0 } func (m *Mirror) GetContinentCode() string { if m != nil { return m.ContinentCode } return "" } func (m *Mirror) GetCountryCodes() string { if m != nil { return m.CountryCodes } return "" } func (m *Mirror) GetExcludedCountryCodes() string { if m != nil { return m.ExcludedCountryCodes } return "" } func (m *Mirror) GetAsnum() uint32 { if m != nil { return m.Asnum } return 0 } func (m *Mirror) GetComment() string { if m != nil { return m.Comment } return "" } func (m *Mirror) GetEnabled() bool { if m != nil { return m.Enabled } return false } func (m *Mirror) GetHttpUp() bool { if m != nil { return m.HttpUp } return false } func (m *Mirror) GetHttpDownReason() string { if m != nil { return m.HttpDownReason } return "" } func (m *Mirror) GetStateSince() *timestamp.Timestamp { if m != nil { return m.StateSince } return nil } func (m *Mirror) GetAllowRedirects() int32 { if m != nil { return m.AllowRedirects } return 0 } func (m *Mirror) GetLastSync() *timestamp.Timestamp { if m != nil { return m.LastSync } return nil } func (m *Mirror) GetLastSuccessfulSync() *timestamp.Timestamp { if m != nil { return m.LastSuccessfulSync } return nil } func (m *Mirror) GetLastModTime() *timestamp.Timestamp { if m != nil { return m.LastModTime } return nil } func (m *Mirror) GetHttpsUp() bool { if m != nil { return m.HttpsUp } return false } func (m *Mirror) GetHttpsDownReason() string { if m != nil { return m.HttpsDownReason } return "" } type MirrorListReply struct { Mirrors []*Mirror `protobuf:"bytes,1,rep,name=Mirrors,proto3" json:"Mirrors,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MirrorListReply) Reset() { *m = MirrorListReply{} } func (m *MirrorListReply) String() string { return proto.CompactTextString(m) } func (*MirrorListReply) ProtoMessage() {} func (*MirrorListReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{3} } func (m *MirrorListReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MirrorListReply.Unmarshal(m, b) } func (m *MirrorListReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MirrorListReply.Marshal(b, m, deterministic) } func (m *MirrorListReply) XXX_Merge(src proto.Message) { xxx_messageInfo_MirrorListReply.Merge(m, src) } func (m *MirrorListReply) XXX_Size() int { return xxx_messageInfo_MirrorListReply.Size(m) } func (m *MirrorListReply) XXX_DiscardUnknown() { xxx_messageInfo_MirrorListReply.DiscardUnknown(m) } var xxx_messageInfo_MirrorListReply proto.InternalMessageInfo func (m *MirrorListReply) GetMirrors() []*Mirror { if m != nil { return m.Mirrors } return nil } type MirrorID struct { ID int32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MirrorID) Reset() { *m = MirrorID{} } func (m *MirrorID) String() string { return proto.CompactTextString(m) } func (*MirrorID) ProtoMessage() {} func (*MirrorID) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{4} } func (m *MirrorID) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MirrorID.Unmarshal(m, b) } func (m *MirrorID) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MirrorID.Marshal(b, m, deterministic) } func (m *MirrorID) XXX_Merge(src proto.Message) { xxx_messageInfo_MirrorID.Merge(m, src) } func (m *MirrorID) XXX_Size() int { return xxx_messageInfo_MirrorID.Size(m) } func (m *MirrorID) XXX_DiscardUnknown() { xxx_messageInfo_MirrorID.DiscardUnknown(m) } var xxx_messageInfo_MirrorID proto.InternalMessageInfo func (m *MirrorID) GetID() int32 { if m != nil { return m.ID } return 0 } func (m *MirrorID) GetName() string { if m != nil { return m.Name } return "" } type MatchReply struct { Mirrors []*MirrorID `protobuf:"bytes,1,rep,name=Mirrors,proto3" json:"Mirrors,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MatchReply) Reset() { *m = MatchReply{} } func (m *MatchReply) String() string { return proto.CompactTextString(m) } func (*MatchReply) ProtoMessage() {} func (*MatchReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{5} } func (m *MatchReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MatchReply.Unmarshal(m, b) } func (m *MatchReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MatchReply.Marshal(b, m, deterministic) } func (m *MatchReply) XXX_Merge(src proto.Message) { xxx_messageInfo_MatchReply.Merge(m, src) } func (m *MatchReply) XXX_Size() int { return xxx_messageInfo_MatchReply.Size(m) } func (m *MatchReply) XXX_DiscardUnknown() { xxx_messageInfo_MatchReply.DiscardUnknown(m) } var xxx_messageInfo_MatchReply proto.InternalMessageInfo func (m *MatchReply) GetMirrors() []*MirrorID { if m != nil { return m.Mirrors } return nil } type ChangeStatusRequest struct { ID int32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Enabled bool `protobuf:"varint,2,opt,name=Enabled,proto3" json:"Enabled,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ChangeStatusRequest) Reset() { *m = ChangeStatusRequest{} } func (m *ChangeStatusRequest) String() string { return proto.CompactTextString(m) } func (*ChangeStatusRequest) ProtoMessage() {} func (*ChangeStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{6} } func (m *ChangeStatusRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ChangeStatusRequest.Unmarshal(m, b) } func (m *ChangeStatusRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ChangeStatusRequest.Marshal(b, m, deterministic) } func (m *ChangeStatusRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_ChangeStatusRequest.Merge(m, src) } func (m *ChangeStatusRequest) XXX_Size() int { return xxx_messageInfo_ChangeStatusRequest.Size(m) } func (m *ChangeStatusRequest) XXX_DiscardUnknown() { xxx_messageInfo_ChangeStatusRequest.DiscardUnknown(m) } var xxx_messageInfo_ChangeStatusRequest proto.InternalMessageInfo func (m *ChangeStatusRequest) GetID() int32 { if m != nil { return m.ID } return 0 } func (m *ChangeStatusRequest) GetEnabled() bool { if m != nil { return m.Enabled } return false } type MirrorIDRequest struct { ID int32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MirrorIDRequest) Reset() { *m = MirrorIDRequest{} } func (m *MirrorIDRequest) String() string { return proto.CompactTextString(m) } func (*MirrorIDRequest) ProtoMessage() {} func (*MirrorIDRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{7} } func (m *MirrorIDRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MirrorIDRequest.Unmarshal(m, b) } func (m *MirrorIDRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MirrorIDRequest.Marshal(b, m, deterministic) } func (m *MirrorIDRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MirrorIDRequest.Merge(m, src) } func (m *MirrorIDRequest) XXX_Size() int { return xxx_messageInfo_MirrorIDRequest.Size(m) } func (m *MirrorIDRequest) XXX_DiscardUnknown() { xxx_messageInfo_MirrorIDRequest.DiscardUnknown(m) } var xxx_messageInfo_MirrorIDRequest proto.InternalMessageInfo func (m *MirrorIDRequest) GetID() int32 { if m != nil { return m.ID } return 0 } type AddMirrorReply struct { Latitude float32 `protobuf:"fixed32,1,opt,name=Latitude,proto3" json:"Latitude,omitempty"` Longitude float32 `protobuf:"fixed32,2,opt,name=Longitude,proto3" json:"Longitude,omitempty"` Country string `protobuf:"bytes,3,opt,name=Country,proto3" json:"Country,omitempty"` Continent string `protobuf:"bytes,4,opt,name=Continent,proto3" json:"Continent,omitempty"` ASN string `protobuf:"bytes,5,opt,name=ASN,proto3" json:"ASN,omitempty"` Warnings []string `protobuf:"bytes,6,rep,name=Warnings,proto3" json:"Warnings,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AddMirrorReply) Reset() { *m = AddMirrorReply{} } func (m *AddMirrorReply) String() string { return proto.CompactTextString(m) } func (*AddMirrorReply) ProtoMessage() {} func (*AddMirrorReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{8} } func (m *AddMirrorReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AddMirrorReply.Unmarshal(m, b) } func (m *AddMirrorReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_AddMirrorReply.Marshal(b, m, deterministic) } func (m *AddMirrorReply) XXX_Merge(src proto.Message) { xxx_messageInfo_AddMirrorReply.Merge(m, src) } func (m *AddMirrorReply) XXX_Size() int { return xxx_messageInfo_AddMirrorReply.Size(m) } func (m *AddMirrorReply) XXX_DiscardUnknown() { xxx_messageInfo_AddMirrorReply.DiscardUnknown(m) } var xxx_messageInfo_AddMirrorReply proto.InternalMessageInfo func (m *AddMirrorReply) GetLatitude() float32 { if m != nil { return m.Latitude } return 0 } func (m *AddMirrorReply) GetLongitude() float32 { if m != nil { return m.Longitude } return 0 } func (m *AddMirrorReply) GetCountry() string { if m != nil { return m.Country } return "" } func (m *AddMirrorReply) GetContinent() string { if m != nil { return m.Continent } return "" } func (m *AddMirrorReply) GetASN() string { if m != nil { return m.ASN } return "" } func (m *AddMirrorReply) GetWarnings() []string { if m != nil { return m.Warnings } return nil } type UpdateMirrorReply struct { Diff string `protobuf:"bytes,1,opt,name=Diff,proto3" json:"Diff,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *UpdateMirrorReply) Reset() { *m = UpdateMirrorReply{} } func (m *UpdateMirrorReply) String() string { return proto.CompactTextString(m) } func (*UpdateMirrorReply) ProtoMessage() {} func (*UpdateMirrorReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{9} } func (m *UpdateMirrorReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UpdateMirrorReply.Unmarshal(m, b) } func (m *UpdateMirrorReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UpdateMirrorReply.Marshal(b, m, deterministic) } func (m *UpdateMirrorReply) XXX_Merge(src proto.Message) { xxx_messageInfo_UpdateMirrorReply.Merge(m, src) } func (m *UpdateMirrorReply) XXX_Size() int { return xxx_messageInfo_UpdateMirrorReply.Size(m) } func (m *UpdateMirrorReply) XXX_DiscardUnknown() { xxx_messageInfo_UpdateMirrorReply.DiscardUnknown(m) } var xxx_messageInfo_UpdateMirrorReply proto.InternalMessageInfo func (m *UpdateMirrorReply) GetDiff() string { if m != nil { return m.Diff } return "" } type GeoUpdateMirrorReply struct { Mirror *Mirror `protobuf:"bytes,1,opt,name=Mirror,proto3" json:"Mirror,omitempty"` Diff string `protobuf:"bytes,2,opt,name=Diff,proto3" json:"Diff,omitempty"` Warnings []string `protobuf:"bytes,3,rep,name=Warnings,proto3" json:"Warnings,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *GeoUpdateMirrorReply) Reset() { *m = GeoUpdateMirrorReply{} } func (m *GeoUpdateMirrorReply) String() string { return proto.CompactTextString(m) } func (*GeoUpdateMirrorReply) ProtoMessage() {} func (*GeoUpdateMirrorReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{10} } func (m *GeoUpdateMirrorReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GeoUpdateMirrorReply.Unmarshal(m, b) } func (m *GeoUpdateMirrorReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GeoUpdateMirrorReply.Marshal(b, m, deterministic) } func (m *GeoUpdateMirrorReply) XXX_Merge(src proto.Message) { xxx_messageInfo_GeoUpdateMirrorReply.Merge(m, src) } func (m *GeoUpdateMirrorReply) XXX_Size() int { return xxx_messageInfo_GeoUpdateMirrorReply.Size(m) } func (m *GeoUpdateMirrorReply) XXX_DiscardUnknown() { xxx_messageInfo_GeoUpdateMirrorReply.DiscardUnknown(m) } var xxx_messageInfo_GeoUpdateMirrorReply proto.InternalMessageInfo func (m *GeoUpdateMirrorReply) GetMirror() *Mirror { if m != nil { return m.Mirror } return nil } func (m *GeoUpdateMirrorReply) GetDiff() string { if m != nil { return m.Diff } return "" } func (m *GeoUpdateMirrorReply) GetWarnings() []string { if m != nil { return m.Warnings } return nil } type RefreshRepositoryRequest struct { Rehash bool `protobuf:"varint,1,opt,name=Rehash,proto3" json:"Rehash,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *RefreshRepositoryRequest) Reset() { *m = RefreshRepositoryRequest{} } func (m *RefreshRepositoryRequest) String() string { return proto.CompactTextString(m) } func (*RefreshRepositoryRequest) ProtoMessage() {} func (*RefreshRepositoryRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{11} } func (m *RefreshRepositoryRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RefreshRepositoryRequest.Unmarshal(m, b) } func (m *RefreshRepositoryRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RefreshRepositoryRequest.Marshal(b, m, deterministic) } func (m *RefreshRepositoryRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_RefreshRepositoryRequest.Merge(m, src) } func (m *RefreshRepositoryRequest) XXX_Size() int { return xxx_messageInfo_RefreshRepositoryRequest.Size(m) } func (m *RefreshRepositoryRequest) XXX_DiscardUnknown() { xxx_messageInfo_RefreshRepositoryRequest.DiscardUnknown(m) } var xxx_messageInfo_RefreshRepositoryRequest proto.InternalMessageInfo func (m *RefreshRepositoryRequest) GetRehash() bool { if m != nil { return m.Rehash } return false } type ScanMirrorRequest struct { ID int32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` AutoEnable bool `protobuf:"varint,2,opt,name=AutoEnable,proto3" json:"AutoEnable,omitempty"` Protocol ScanMirrorRequest_Method `protobuf:"varint,3,opt,name=Protocol,proto3,enum=ScanMirrorRequest_Method" json:"Protocol,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ScanMirrorRequest) Reset() { *m = ScanMirrorRequest{} } func (m *ScanMirrorRequest) String() string { return proto.CompactTextString(m) } func (*ScanMirrorRequest) ProtoMessage() {} func (*ScanMirrorRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{12} } func (m *ScanMirrorRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ScanMirrorRequest.Unmarshal(m, b) } func (m *ScanMirrorRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ScanMirrorRequest.Marshal(b, m, deterministic) } func (m *ScanMirrorRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_ScanMirrorRequest.Merge(m, src) } func (m *ScanMirrorRequest) XXX_Size() int { return xxx_messageInfo_ScanMirrorRequest.Size(m) } func (m *ScanMirrorRequest) XXX_DiscardUnknown() { xxx_messageInfo_ScanMirrorRequest.DiscardUnknown(m) } var xxx_messageInfo_ScanMirrorRequest proto.InternalMessageInfo func (m *ScanMirrorRequest) GetID() int32 { if m != nil { return m.ID } return 0 } func (m *ScanMirrorRequest) GetAutoEnable() bool { if m != nil { return m.AutoEnable } return false } func (m *ScanMirrorRequest) GetProtocol() ScanMirrorRequest_Method { if m != nil { return m.Protocol } return ScanMirrorRequest_ALL } type ScanMirrorReply struct { Enabled bool `protobuf:"varint,1,opt,name=Enabled,proto3" json:"Enabled,omitempty"` FilesIndexed int64 `protobuf:"varint,2,opt,name=FilesIndexed,proto3" json:"FilesIndexed,omitempty"` KnownIndexed int64 `protobuf:"varint,3,opt,name=KnownIndexed,proto3" json:"KnownIndexed,omitempty"` Removed int64 `protobuf:"varint,4,opt,name=Removed,proto3" json:"Removed,omitempty"` TZOffsetMs int64 `protobuf:"varint,5,opt,name=TZOffsetMs,proto3" json:"TZOffsetMs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ScanMirrorReply) Reset() { *m = ScanMirrorReply{} } func (m *ScanMirrorReply) String() string { return proto.CompactTextString(m) } func (*ScanMirrorReply) ProtoMessage() {} func (*ScanMirrorReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{13} } func (m *ScanMirrorReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ScanMirrorReply.Unmarshal(m, b) } func (m *ScanMirrorReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ScanMirrorReply.Marshal(b, m, deterministic) } func (m *ScanMirrorReply) XXX_Merge(src proto.Message) { xxx_messageInfo_ScanMirrorReply.Merge(m, src) } func (m *ScanMirrorReply) XXX_Size() int { return xxx_messageInfo_ScanMirrorReply.Size(m) } func (m *ScanMirrorReply) XXX_DiscardUnknown() { xxx_messageInfo_ScanMirrorReply.DiscardUnknown(m) } var xxx_messageInfo_ScanMirrorReply proto.InternalMessageInfo func (m *ScanMirrorReply) GetEnabled() bool { if m != nil { return m.Enabled } return false } func (m *ScanMirrorReply) GetFilesIndexed() int64 { if m != nil { return m.FilesIndexed } return 0 } func (m *ScanMirrorReply) GetKnownIndexed() int64 { if m != nil { return m.KnownIndexed } return 0 } func (m *ScanMirrorReply) GetRemoved() int64 { if m != nil { return m.Removed } return 0 } func (m *ScanMirrorReply) GetTZOffsetMs() int64 { if m != nil { return m.TZOffsetMs } return 0 } type StatsFileRequest struct { Pattern string `protobuf:"bytes,1,opt,name=Pattern,proto3" json:"Pattern,omitempty"` DateStart *timestamp.Timestamp `protobuf:"bytes,2,opt,name=DateStart,proto3" json:"DateStart,omitempty"` DateEnd *timestamp.Timestamp `protobuf:"bytes,3,opt,name=DateEnd,proto3" json:"DateEnd,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *StatsFileRequest) Reset() { *m = StatsFileRequest{} } func (m *StatsFileRequest) String() string { return proto.CompactTextString(m) } func (*StatsFileRequest) ProtoMessage() {} func (*StatsFileRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{14} } func (m *StatsFileRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatsFileRequest.Unmarshal(m, b) } func (m *StatsFileRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StatsFileRequest.Marshal(b, m, deterministic) } func (m *StatsFileRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_StatsFileRequest.Merge(m, src) } func (m *StatsFileRequest) XXX_Size() int { return xxx_messageInfo_StatsFileRequest.Size(m) } func (m *StatsFileRequest) XXX_DiscardUnknown() { xxx_messageInfo_StatsFileRequest.DiscardUnknown(m) } var xxx_messageInfo_StatsFileRequest proto.InternalMessageInfo func (m *StatsFileRequest) GetPattern() string { if m != nil { return m.Pattern } return "" } func (m *StatsFileRequest) GetDateStart() *timestamp.Timestamp { if m != nil { return m.DateStart } return nil } func (m *StatsFileRequest) GetDateEnd() *timestamp.Timestamp { if m != nil { return m.DateEnd } return nil } type StatsFileReply struct { Files map[string]int64 `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *StatsFileReply) Reset() { *m = StatsFileReply{} } func (m *StatsFileReply) String() string { return proto.CompactTextString(m) } func (*StatsFileReply) ProtoMessage() {} func (*StatsFileReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{15} } func (m *StatsFileReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatsFileReply.Unmarshal(m, b) } func (m *StatsFileReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StatsFileReply.Marshal(b, m, deterministic) } func (m *StatsFileReply) XXX_Merge(src proto.Message) { xxx_messageInfo_StatsFileReply.Merge(m, src) } func (m *StatsFileReply) XXX_Size() int { return xxx_messageInfo_StatsFileReply.Size(m) } func (m *StatsFileReply) XXX_DiscardUnknown() { xxx_messageInfo_StatsFileReply.DiscardUnknown(m) } var xxx_messageInfo_StatsFileReply proto.InternalMessageInfo func (m *StatsFileReply) GetFiles() map[string]int64 { if m != nil { return m.Files } return nil } type StatsMirrorRequest struct { ID int32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` DateStart *timestamp.Timestamp `protobuf:"bytes,2,opt,name=DateStart,proto3" json:"DateStart,omitempty"` DateEnd *timestamp.Timestamp `protobuf:"bytes,3,opt,name=DateEnd,proto3" json:"DateEnd,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *StatsMirrorRequest) Reset() { *m = StatsMirrorRequest{} } func (m *StatsMirrorRequest) String() string { return proto.CompactTextString(m) } func (*StatsMirrorRequest) ProtoMessage() {} func (*StatsMirrorRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{16} } func (m *StatsMirrorRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatsMirrorRequest.Unmarshal(m, b) } func (m *StatsMirrorRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StatsMirrorRequest.Marshal(b, m, deterministic) } func (m *StatsMirrorRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_StatsMirrorRequest.Merge(m, src) } func (m *StatsMirrorRequest) XXX_Size() int { return xxx_messageInfo_StatsMirrorRequest.Size(m) } func (m *StatsMirrorRequest) XXX_DiscardUnknown() { xxx_messageInfo_StatsMirrorRequest.DiscardUnknown(m) } var xxx_messageInfo_StatsMirrorRequest proto.InternalMessageInfo func (m *StatsMirrorRequest) GetID() int32 { if m != nil { return m.ID } return 0 } func (m *StatsMirrorRequest) GetDateStart() *timestamp.Timestamp { if m != nil { return m.DateStart } return nil } func (m *StatsMirrorRequest) GetDateEnd() *timestamp.Timestamp { if m != nil { return m.DateEnd } return nil } type StatsMirrorReply struct { Mirror *Mirror `protobuf:"bytes,1,opt,name=Mirror,proto3" json:"Mirror,omitempty"` Requests int64 `protobuf:"varint,2,opt,name=Requests,proto3" json:"Requests,omitempty"` Bytes int64 `protobuf:"varint,3,opt,name=Bytes,proto3" json:"Bytes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *StatsMirrorReply) Reset() { *m = StatsMirrorReply{} } func (m *StatsMirrorReply) String() string { return proto.CompactTextString(m) } func (*StatsMirrorReply) ProtoMessage() {} func (*StatsMirrorReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{17} } func (m *StatsMirrorReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatsMirrorReply.Unmarshal(m, b) } func (m *StatsMirrorReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StatsMirrorReply.Marshal(b, m, deterministic) } func (m *StatsMirrorReply) XXX_Merge(src proto.Message) { xxx_messageInfo_StatsMirrorReply.Merge(m, src) } func (m *StatsMirrorReply) XXX_Size() int { return xxx_messageInfo_StatsMirrorReply.Size(m) } func (m *StatsMirrorReply) XXX_DiscardUnknown() { xxx_messageInfo_StatsMirrorReply.DiscardUnknown(m) } var xxx_messageInfo_StatsMirrorReply proto.InternalMessageInfo func (m *StatsMirrorReply) GetMirror() *Mirror { if m != nil { return m.Mirror } return nil } func (m *StatsMirrorReply) GetRequests() int64 { if m != nil { return m.Requests } return 0 } func (m *StatsMirrorReply) GetBytes() int64 { if m != nil { return m.Bytes } return 0 } type GetMirrorLogsRequest struct { ID int32 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` MaxResults int32 `protobuf:"varint,2,opt,name=MaxResults,proto3" json:"MaxResults,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *GetMirrorLogsRequest) Reset() { *m = GetMirrorLogsRequest{} } func (m *GetMirrorLogsRequest) String() string { return proto.CompactTextString(m) } func (*GetMirrorLogsRequest) ProtoMessage() {} func (*GetMirrorLogsRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{18} } func (m *GetMirrorLogsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetMirrorLogsRequest.Unmarshal(m, b) } func (m *GetMirrorLogsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetMirrorLogsRequest.Marshal(b, m, deterministic) } func (m *GetMirrorLogsRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_GetMirrorLogsRequest.Merge(m, src) } func (m *GetMirrorLogsRequest) XXX_Size() int { return xxx_messageInfo_GetMirrorLogsRequest.Size(m) } func (m *GetMirrorLogsRequest) XXX_DiscardUnknown() { xxx_messageInfo_GetMirrorLogsRequest.DiscardUnknown(m) } var xxx_messageInfo_GetMirrorLogsRequest proto.InternalMessageInfo func (m *GetMirrorLogsRequest) GetID() int32 { if m != nil { return m.ID } return 0 } func (m *GetMirrorLogsRequest) GetMaxResults() int32 { if m != nil { return m.MaxResults } return 0 } type GetMirrorLogsReply struct { Line []string `protobuf:"bytes,1,rep,name=line,proto3" json:"line,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *GetMirrorLogsReply) Reset() { *m = GetMirrorLogsReply{} } func (m *GetMirrorLogsReply) String() string { return proto.CompactTextString(m) } func (*GetMirrorLogsReply) ProtoMessage() {} func (*GetMirrorLogsReply) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{19} } func (m *GetMirrorLogsReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetMirrorLogsReply.Unmarshal(m, b) } func (m *GetMirrorLogsReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetMirrorLogsReply.Marshal(b, m, deterministic) } func (m *GetMirrorLogsReply) XXX_Merge(src proto.Message) { xxx_messageInfo_GetMirrorLogsReply.Merge(m, src) } func (m *GetMirrorLogsReply) XXX_Size() int { return xxx_messageInfo_GetMirrorLogsReply.Size(m) } func (m *GetMirrorLogsReply) XXX_DiscardUnknown() { xxx_messageInfo_GetMirrorLogsReply.DiscardUnknown(m) } var xxx_messageInfo_GetMirrorLogsReply proto.InternalMessageInfo func (m *GetMirrorLogsReply) GetLine() []string { if m != nil { return m.Line } return nil } func init() { proto.RegisterEnum("ScanMirrorRequest_Method", ScanMirrorRequest_Method_name, ScanMirrorRequest_Method_value) proto.RegisterType((*VersionReply)(nil), "VersionReply") proto.RegisterType((*MatchRequest)(nil), "MatchRequest") proto.RegisterType((*Mirror)(nil), "Mirror") proto.RegisterType((*MirrorListReply)(nil), "MirrorListReply") proto.RegisterType((*MirrorID)(nil), "MirrorID") proto.RegisterType((*MatchReply)(nil), "MatchReply") proto.RegisterType((*ChangeStatusRequest)(nil), "ChangeStatusRequest") proto.RegisterType((*MirrorIDRequest)(nil), "MirrorIDRequest") proto.RegisterType((*AddMirrorReply)(nil), "AddMirrorReply") proto.RegisterType((*UpdateMirrorReply)(nil), "UpdateMirrorReply") proto.RegisterType((*GeoUpdateMirrorReply)(nil), "GeoUpdateMirrorReply") proto.RegisterType((*RefreshRepositoryRequest)(nil), "RefreshRepositoryRequest") proto.RegisterType((*ScanMirrorRequest)(nil), "ScanMirrorRequest") proto.RegisterType((*ScanMirrorReply)(nil), "ScanMirrorReply") proto.RegisterType((*StatsFileRequest)(nil), "StatsFileRequest") proto.RegisterType((*StatsFileReply)(nil), "StatsFileReply") proto.RegisterMapType((map[string]int64)(nil), "StatsFileReply.FilesEntry") proto.RegisterType((*StatsMirrorRequest)(nil), "StatsMirrorRequest") proto.RegisterType((*StatsMirrorReply)(nil), "StatsMirrorReply") proto.RegisterType((*GetMirrorLogsRequest)(nil), "GetMirrorLogsRequest") proto.RegisterType((*GetMirrorLogsReply)(nil), "GetMirrorLogsReply") } func init() { proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) } var fileDescriptor_77a6da22d6a3feb1 = []byte{ // 1472 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xdd, 0x72, 0x1b, 0xc5, 0x12, 0xd6, 0x4a, 0xb6, 0x65, 0xb5, 0x64, 0x5b, 0x1e, 0x3b, 0x3e, 0x1b, 0x25, 0x27, 0x51, 0xe6, 0xfc, 0x44, 0xa7, 0x4e, 0x9d, 0xcd, 0x89, 0x49, 0xc0, 0x15, 0x02, 0x94, 0x90, 0x6c, 0xc7, 0x20, 0xc7, 0xae, 0x55, 0x0c, 0x05, 0x77, 0x1b, 0xed, 0x48, 0xde, 0x62, 0xb5, 0x23, 0x76, 0x46, 0x89, 0x55, 0xc5, 0x63, 0x70, 0xc9, 0x05, 0x3c, 0x00, 0x55, 0x5c, 0xf2, 0x40, 0x3c, 0x08, 0xd5, 0x33, 0xb3, 0xd2, 0x6a, 0xe5, 0x1f, 0xc8, 0x05, 0x77, 0xf3, 0x7d, 0xdd, 0x33, 0xdd, 0xd3, 0xd3, 0x3f, 0xbb, 0x50, 0x8a, 0x47, 0x3d, 0x67, 0x14, 0x73, 0xc9, 0x6b, 0x77, 0x06, 0x9c, 0x0f, 0x42, 0xf6, 0x48, 0xa1, 0xd7, 0xe3, 0xfe, 0x23, 0x36, 0x1c, 0xc9, 0x89, 0x11, 0xde, 0xcf, 0x0a, 0x65, 0x30, 0x64, 0x42, 0x7a, 0xc3, 0x91, 0x56, 0xa0, 0x3f, 0x5a, 0x50, 0xf9, 0x82, 0xc5, 0x22, 0xe0, 0x91, 0xcb, 0x46, 0xe1, 0x84, 0xd8, 0x50, 0x34, 0xd8, 0xb6, 0xea, 0x56, 0xa3, 0xe4, 0x26, 0x90, 0x6c, 0xc3, 0xf2, 0xa7, 0xe3, 0x20, 0xf4, 0xed, 0xbc, 0xe2, 0x35, 0x20, 0x77, 0xa1, 0x74, 0xc8, 0x93, 0x1d, 0x05, 0x25, 0x99, 0x11, 0x64, 0x1d, 0xf2, 0x27, 0x5d, 0x7b, 0x49, 0xd1, 0xf9, 0x93, 0x2e, 0x21, 0xb0, 0xd4, 0x8c, 0x7b, 0xe7, 0xf6, 0xb2, 0x62, 0xd4, 0x9a, 0xdc, 0x03, 0x38, 0xe4, 0xc7, 0xde, 0xc5, 0x69, 0xcc, 0x7b, 0xc2, 0x5e, 0xa9, 0x5b, 0x8d, 0x65, 0x37, 0xc5, 0xd0, 0x06, 0x54, 0x8e, 0x3d, 0xd9, 0x3b, 0x77, 0xd9, 0xb7, 0x63, 0x26, 0x24, 0x7a, 0x78, 0xea, 0x49, 0xc9, 0xe2, 0xa9, 0x87, 0x06, 0xd2, 0xdf, 0x56, 0x61, 0xe5, 0x38, 0x88, 0x63, 0x1e, 0xa3, 0xe1, 0xa3, 0xb6, 0x92, 0x2f, 0xbb, 0xf9, 0xa3, 0x36, 0x1a, 0x7e, 0xe9, 0x0d, 0x99, 0xf1, 0x5d, 0xad, 0xf1, 0xa0, 0x17, 0x52, 0x8e, 0xce, 0xdc, 0x8e, 0x71, 0x3c, 0x81, 0xa4, 0x06, 0xab, 0xae, 0x98, 0x44, 0x3d, 0x14, 0x69, 0xe7, 0xa7, 0x98, 0xec, 0xc0, 0xca, 0x81, 0xde, 0xa4, 0x2f, 0x61, 0x10, 0xa9, 0x43, 0xb9, 0x3b, 0xe2, 0x91, 0xe0, 0xb1, 0x32, 0xb4, 0xa2, 0x84, 0x69, 0x0a, 0x2f, 0x6a, 0x20, 0xee, 0x2e, 0x2a, 0x85, 0x14, 0x43, 0xfe, 0x0d, 0xeb, 0x06, 0x75, 0xf8, 0x80, 0xa3, 0xce, 0xaa, 0xd2, 0xc9, 0xb0, 0x18, 0xf2, 0xa6, 0x3f, 0x0c, 0x22, 0x65, 0xa7, 0xa4, 0x43, 0x3e, 0x25, 0xd0, 0x8a, 0x02, 0xfb, 0x43, 0x2f, 0x08, 0x6d, 0xd0, 0x56, 0x66, 0x0c, 0xca, 0x5b, 0x63, 0x21, 0xf9, 0xb0, 0xed, 0x49, 0xcf, 0x2e, 0x6b, 0xf9, 0x8c, 0x21, 0xff, 0x84, 0xb5, 0x16, 0x8f, 0x64, 0x10, 0xb1, 0x48, 0x9e, 0x44, 0xe1, 0xc4, 0xae, 0xd4, 0xad, 0xc6, 0xaa, 0x3b, 0x4f, 0xe2, 0x6d, 0x5b, 0x7c, 0x1c, 0xc9, 0x78, 0xa2, 0x74, 0xd6, 0x94, 0x4e, 0x9a, 0xc2, 0x38, 0x35, 0xbb, 0x4a, 0xb8, 0xae, 0x84, 0x06, 0x61, 0x1a, 0x75, 0x7b, 0x3c, 0x66, 0xf6, 0x86, 0x7a, 0x1c, 0x0d, 0x30, 0xe2, 0x1d, 0x4f, 0x06, 0x72, 0xec, 0x33, 0xbb, 0x5a, 0xb7, 0x1a, 0x79, 0x77, 0x8a, 0xf1, 0xbe, 0x1d, 0x1e, 0x0d, 0xb4, 0x70, 0x53, 0x09, 0x67, 0xc4, 0x9c, 0xbf, 0x2d, 0xee, 0x33, 0x9b, 0xa8, 0x2b, 0xcd, 0x93, 0x84, 0x42, 0xc5, 0x38, 0x87, 0x50, 0xd8, 0x5b, 0x4a, 0x69, 0x8e, 0x23, 0xbb, 0xb0, 0xbd, 0x7f, 0xd1, 0x0b, 0xc7, 0x3e, 0xf3, 0xe7, 0x74, 0xb7, 0x95, 0xee, 0xa5, 0x32, 0xbc, 0x4d, 0x53, 0x44, 0xe3, 0xa1, 0x7d, 0xab, 0x6e, 0x35, 0xd6, 0x5c, 0x0d, 0x30, 0xb3, 0x5a, 0x7c, 0x38, 0x64, 0x91, 0xb4, 0x77, 0x74, 0x66, 0x19, 0x88, 0x92, 0xfd, 0xc8, 0x7b, 0x1d, 0x32, 0xdf, 0xfe, 0x9b, 0x0a, 0x4b, 0x02, 0x31, 0x5e, 0x2a, 0xfd, 0x46, 0xb6, 0xad, 0xe3, 0xa5, 0x11, 0x66, 0x05, 0xae, 0xda, 0xfc, 0x6d, 0xe4, 0x32, 0x4f, 0xf0, 0xc8, 0xbe, 0xad, 0xb3, 0x62, 0x9e, 0x25, 0xcf, 0x00, 0xba, 0xd2, 0x93, 0xac, 0x1b, 0x44, 0x3d, 0x66, 0xd7, 0xea, 0x56, 0xa3, 0xbc, 0x5b, 0x73, 0x74, 0xfd, 0x3b, 0x49, 0xfd, 0x3b, 0xaf, 0x92, 0xfa, 0x77, 0x53, 0xda, 0x68, 0xa3, 0x19, 0x86, 0xfc, 0xad, 0xcb, 0xfc, 0x20, 0x66, 0x3d, 0x29, 0xec, 0x3b, 0xea, 0x71, 0x32, 0x2c, 0x79, 0x1f, 0x5f, 0x49, 0xc8, 0xee, 0x24, 0xea, 0xd9, 0x77, 0x6f, 0xb4, 0x30, 0xd5, 0x25, 0x9f, 0x01, 0x51, 0xeb, 0x71, 0xaf, 0xc7, 0x84, 0xe8, 0x8f, 0x43, 0x75, 0xc2, 0xdf, 0x6f, 0x3c, 0xe1, 0x92, 0x5d, 0xe4, 0x39, 0x94, 0x91, 0x3d, 0xe6, 0x3e, 0xea, 0xd9, 0xf7, 0x6e, 0x3c, 0x24, 0xad, 0x9e, 0xd4, 0xbc, 0x38, 0x1b, 0xd9, 0xf7, 0x75, 0xfc, 0x0d, 0x24, 0x0d, 0xd8, 0x50, 0xcb, 0x54, 0xa0, 0xeb, 0x2a, 0xd0, 0x59, 0x9a, 0x3e, 0x81, 0x0d, 0xdd, 0x65, 0x3a, 0x81, 0x90, 0xba, 0x6b, 0x3e, 0x80, 0xa2, 0xa6, 0x84, 0x6d, 0xd5, 0x0b, 0x8d, 0xf2, 0x6e, 0xd1, 0xd1, 0xd8, 0x4d, 0x78, 0xea, 0xc0, 0xaa, 0x5e, 0x1e, 0xb5, 0xff, 0x48, 0x77, 0xa2, 0x8f, 0x01, 0x4c, 0xdb, 0x43, 0x03, 0xff, 0xc8, 0x1a, 0x28, 0x39, 0xc9, 0x69, 0x33, 0x13, 0x9f, 0xc0, 0x56, 0xeb, 0xdc, 0x8b, 0x06, 0x0c, 0x9f, 0x76, 0x2c, 0x92, 0x86, 0x99, 0xb5, 0x96, 0xca, 0xc1, 0xfc, 0x5c, 0x0e, 0xd2, 0x07, 0xc9, 0xcd, 0x8e, 0xda, 0x57, 0x6c, 0xa6, 0xbf, 0x58, 0xb0, 0xde, 0xf4, 0x7d, 0x73, 0x3b, 0xe5, 0x5b, 0xba, 0x76, 0xad, 0xeb, 0x6a, 0x37, 0x9f, 0xad, 0x5d, 0x55, 0x27, 0xaa, 0x9a, 0x92, 0x0e, 0x6c, 0x20, 0xee, 0x9b, 0x16, 0xb0, 0x69, 0xc1, 0x33, 0x82, 0x54, 0xa1, 0xd0, 0xec, 0xbe, 0x34, 0x0d, 0x18, 0x97, 0xe8, 0xc3, 0x97, 0x5e, 0x1c, 0x05, 0xd1, 0x00, 0x47, 0x48, 0x01, 0x3b, 0x76, 0x82, 0xe9, 0x43, 0xd8, 0x3c, 0x1b, 0xf9, 0x9e, 0x64, 0x69, 0xa7, 0x09, 0x2c, 0xb5, 0x83, 0x7e, 0xdf, 0x8c, 0x10, 0xb5, 0xa6, 0x03, 0xd8, 0x3e, 0x64, 0x7c, 0x51, 0xf7, 0x7e, 0x32, 0x56, 0x94, 0x76, 0xea, 0x71, 0x93, 0x69, 0x93, 0x1c, 0x96, 0x9f, 0x1d, 0x36, 0xe7, 0x51, 0x21, 0xe3, 0xd1, 0x2e, 0xd8, 0x2e, 0xeb, 0xc7, 0x4c, 0xe0, 0xeb, 0x72, 0x11, 0x48, 0x1e, 0x4f, 0x92, 0x80, 0xef, 0xc0, 0x8a, 0xcb, 0xce, 0x3d, 0x71, 0xae, 0x8c, 0xad, 0xba, 0x06, 0xd1, 0x9f, 0x2c, 0xd8, 0xec, 0xf6, 0xbc, 0x28, 0x71, 0xec, 0xf2, 0xb7, 0xc5, 0xee, 0x3f, 0x96, 0x5c, 0x3f, 0xa8, 0x79, 0xde, 0x14, 0x43, 0x9e, 0xc2, 0xea, 0x29, 0x96, 0x48, 0x8f, 0x87, 0x2a, 0xe4, 0xeb, 0xbb, 0xb7, 0x9d, 0x85, 0x53, 0x9d, 0x63, 0x26, 0xcf, 0xb9, 0xef, 0x4e, 0x55, 0xe9, 0xbf, 0x60, 0x45, 0x73, 0xa4, 0x08, 0x85, 0x66, 0xa7, 0x53, 0xcd, 0xe1, 0xe2, 0xe0, 0xd5, 0x69, 0xd5, 0x22, 0x25, 0x58, 0x76, 0xbb, 0x5f, 0xbd, 0x6c, 0x55, 0xf3, 0xf4, 0x67, 0x0b, 0x36, 0xd2, 0xa7, 0x99, 0x0f, 0x8a, 0x24, 0xdb, 0xac, 0xf9, 0x8e, 0x47, 0xa1, 0x72, 0x10, 0x84, 0x4c, 0x1c, 0x45, 0x3e, 0xbb, 0x30, 0xc9, 0x58, 0x70, 0xe7, 0x38, 0xd4, 0xf9, 0x3c, 0xe2, 0x6f, 0xa3, 0x44, 0xa7, 0xa0, 0x75, 0xd2, 0x1c, 0x5a, 0x70, 0xd9, 0x90, 0xbf, 0x61, 0xbe, 0xca, 0x94, 0x82, 0x9b, 0x40, 0x8c, 0xc6, 0xab, 0xaf, 0x4f, 0xfa, 0x7d, 0xc1, 0xe4, 0xb1, 0x50, 0xe9, 0x52, 0x70, 0x53, 0x0c, 0xfd, 0xc1, 0x82, 0x2a, 0xd6, 0x8a, 0x40, 0x9b, 0x37, 0x7e, 0x5f, 0x90, 0x3d, 0x28, 0xb5, 0xb1, 0x67, 0x4a, 0x2f, 0x96, 0xca, 0xdb, 0xeb, 0x1b, 0xcf, 0x4c, 0x99, 0x3c, 0x81, 0x22, 0x82, 0xfd, 0x48, 0xdf, 0xe0, 0xfa, 0x7d, 0x89, 0x2a, 0xfd, 0x0e, 0xd6, 0x53, 0xde, 0x61, 0x30, 0xff, 0x0f, 0xcb, 0x7d, 0x0c, 0x8f, 0x69, 0x02, 0x35, 0x67, 0x5e, 0xee, 0xa8, 0xd8, 0xed, 0x63, 0x05, 0xb9, 0x5a, 0xb1, 0xb6, 0x07, 0x30, 0x23, 0xb1, 0x70, 0xbe, 0x61, 0x13, 0x73, 0x2f, 0x5c, 0xe2, 0x00, 0x7b, 0xe3, 0x85, 0x63, 0x66, 0xa2, 0xaf, 0xc1, 0xb3, 0xfc, 0x9e, 0x45, 0xbf, 0xb7, 0x80, 0xa8, 0xe3, 0xaf, 0xcf, 0xb8, 0xbf, 0x3a, 0x28, 0xcc, 0x3c, 0xd9, 0x9f, 0x2a, 0x50, 0xfc, 0xa0, 0xd3, 0xfe, 0x0b, 0x73, 0xd1, 0x29, 0x56, 0xdf, 0xb5, 0x13, 0xc9, 0x84, 0xc9, 0x2d, 0x0d, 0xe8, 0x01, 0xf6, 0x02, 0x69, 0xfa, 0x3c, 0x1f, 0x88, 0x6b, 0x0a, 0xee, 0xd8, 0xbb, 0x70, 0x99, 0x18, 0x87, 0xe6, 0xec, 0x65, 0x37, 0xc5, 0xd0, 0x06, 0x90, 0xcc, 0x39, 0xa6, 0xfb, 0x84, 0x41, 0xc4, 0xd4, 0x33, 0x96, 0x5c, 0xb5, 0xde, 0xfd, 0xb5, 0x08, 0x85, 0x56, 0xe7, 0x88, 0x3c, 0x05, 0x38, 0x64, 0x32, 0xf9, 0x82, 0xde, 0x59, 0x88, 0xc9, 0x3e, 0x7e, 0xdf, 0xd7, 0xd6, 0x9c, 0xf4, 0x67, 0x3b, 0xcd, 0x91, 0x0f, 0xa1, 0x78, 0x36, 0x1a, 0xc4, 0x9e, 0xcf, 0xae, 0xdc, 0x73, 0x05, 0x4f, 0x73, 0xe4, 0x19, 0x36, 0x9d, 0x90, 0x7b, 0xfe, 0x3b, 0xec, 0xfd, 0x18, 0x2a, 0xe9, 0xa9, 0x43, 0xb6, 0x9d, 0x4b, 0x86, 0xd0, 0x35, 0xfb, 0x77, 0x61, 0x09, 0x07, 0xe9, 0x95, 0x96, 0xab, 0x4e, 0x66, 0xda, 0xd2, 0x1c, 0xf9, 0x0f, 0x80, 0x19, 0x54, 0x51, 0x9f, 0x93, 0xaa, 0x93, 0x99, 0x5a, 0xb5, 0x24, 0x01, 0x68, 0x8e, 0x3c, 0xc4, 0xaf, 0x65, 0x33, 0xaf, 0x48, 0xc2, 0xd7, 0x36, 0x9c, 0xf9, 0x21, 0x46, 0x73, 0xe4, 0x7f, 0x50, 0x49, 0xb7, 0xfe, 0x99, 0x2e, 0x71, 0x16, 0x46, 0x82, 0x0a, 0x59, 0x45, 0xb7, 0x19, 0xa3, 0xbe, 0xe8, 0xc4, 0xd5, 0x57, 0x7e, 0x0e, 0x1b, 0x99, 0x41, 0x73, 0xc9, 0xf6, 0x5b, 0xce, 0x65, 0xc3, 0x88, 0xe6, 0xc8, 0x0b, 0xd8, 0x5c, 0x98, 0x1e, 0xe4, 0xb6, 0x73, 0xd5, 0x44, 0xb9, 0xc6, 0x8f, 0x27, 0x00, 0xb3, 0x76, 0x4d, 0xc8, 0xe2, 0x24, 0xa8, 0x55, 0x9d, 0x4c, 0x3f, 0xa7, 0x39, 0xf2, 0x18, 0x4a, 0xd3, 0xb6, 0x43, 0x36, 0x9d, 0x6c, 0x03, 0xad, 0x6d, 0x64, 0xba, 0x12, 0xcd, 0x91, 0x0f, 0xa0, 0x9c, 0x2a, 0x5a, 0xb2, 0xe5, 0x2c, 0x36, 0x96, 0xda, 0xa6, 0x93, 0xad, 0x6b, 0x9a, 0x23, 0x7b, 0xb0, 0x74, 0x1a, 0x44, 0x83, 0x77, 0x48, 0xcb, 0x8f, 0x60, 0x6d, 0xae, 0xf0, 0x08, 0xc6, 0x73, 0xb1, 0xa0, 0x6b, 0x5b, 0xce, 0x62, 0x7d, 0xd2, 0x1c, 0xf9, 0x2f, 0x94, 0xd5, 0xe7, 0x97, 0xf1, 0x78, 0xcd, 0x49, 0xff, 0x83, 0xd6, 0xca, 0xce, 0xec, 0xdb, 0x8c, 0xe6, 0x5e, 0xaf, 0x28, 0xeb, 0xef, 0xfd, 0x1e, 0x00, 0x00, 0xff, 0xff, 0x8e, 0xac, 0x3b, 0x6f, 0x97, 0x0f, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConnInterface // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. const _ = grpc.SupportPackageIsVersion6 // CLIClient is the client API for CLI service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type CLIClient interface { GetVersion(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*VersionReply, error) Upgrade(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) Reload(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) ChangeStatus(ctx context.Context, in *ChangeStatusRequest, opts ...grpc.CallOption) (*empty.Empty, error) List(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*MirrorListReply, error) MirrorInfo(ctx context.Context, in *MirrorIDRequest, opts ...grpc.CallOption) (*Mirror, error) AddMirror(ctx context.Context, in *Mirror, opts ...grpc.CallOption) (*AddMirrorReply, error) UpdateMirror(ctx context.Context, in *Mirror, opts ...grpc.CallOption) (*UpdateMirrorReply, error) RemoveMirror(ctx context.Context, in *MirrorIDRequest, opts ...grpc.CallOption) (*empty.Empty, error) GeoUpdateMirror(ctx context.Context, in *MirrorIDRequest, opts ...grpc.CallOption) (*GeoUpdateMirrorReply, error) RefreshRepository(ctx context.Context, in *RefreshRepositoryRequest, opts ...grpc.CallOption) (*empty.Empty, error) ScanMirror(ctx context.Context, in *ScanMirrorRequest, opts ...grpc.CallOption) (*ScanMirrorReply, error) StatsFile(ctx context.Context, in *StatsFileRequest, opts ...grpc.CallOption) (*StatsFileReply, error) StatsMirror(ctx context.Context, in *StatsMirrorRequest, opts ...grpc.CallOption) (*StatsMirrorReply, error) Ping(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) GetMirrorLogs(ctx context.Context, in *GetMirrorLogsRequest, opts ...grpc.CallOption) (*GetMirrorLogsReply, error) // Tools MatchMirror(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchReply, error) } type cLIClient struct { cc grpc.ClientConnInterface } func NewCLIClient(cc grpc.ClientConnInterface) CLIClient { return &cLIClient{cc} } func (c *cLIClient) GetVersion(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*VersionReply, error) { out := new(VersionReply) err := c.cc.Invoke(ctx, "/CLI/GetVersion", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) Upgrade(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { out := new(empty.Empty) err := c.cc.Invoke(ctx, "/CLI/Upgrade", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) Reload(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { out := new(empty.Empty) err := c.cc.Invoke(ctx, "/CLI/Reload", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) ChangeStatus(ctx context.Context, in *ChangeStatusRequest, opts ...grpc.CallOption) (*empty.Empty, error) { out := new(empty.Empty) err := c.cc.Invoke(ctx, "/CLI/ChangeStatus", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) List(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*MirrorListReply, error) { out := new(MirrorListReply) err := c.cc.Invoke(ctx, "/CLI/List", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) MirrorInfo(ctx context.Context, in *MirrorIDRequest, opts ...grpc.CallOption) (*Mirror, error) { out := new(Mirror) err := c.cc.Invoke(ctx, "/CLI/MirrorInfo", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) AddMirror(ctx context.Context, in *Mirror, opts ...grpc.CallOption) (*AddMirrorReply, error) { out := new(AddMirrorReply) err := c.cc.Invoke(ctx, "/CLI/AddMirror", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) UpdateMirror(ctx context.Context, in *Mirror, opts ...grpc.CallOption) (*UpdateMirrorReply, error) { out := new(UpdateMirrorReply) err := c.cc.Invoke(ctx, "/CLI/UpdateMirror", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) RemoveMirror(ctx context.Context, in *MirrorIDRequest, opts ...grpc.CallOption) (*empty.Empty, error) { out := new(empty.Empty) err := c.cc.Invoke(ctx, "/CLI/RemoveMirror", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) GeoUpdateMirror(ctx context.Context, in *MirrorIDRequest, opts ...grpc.CallOption) (*GeoUpdateMirrorReply, error) { out := new(GeoUpdateMirrorReply) err := c.cc.Invoke(ctx, "/CLI/GeoUpdateMirror", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) RefreshRepository(ctx context.Context, in *RefreshRepositoryRequest, opts ...grpc.CallOption) (*empty.Empty, error) { out := new(empty.Empty) err := c.cc.Invoke(ctx, "/CLI/RefreshRepository", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) ScanMirror(ctx context.Context, in *ScanMirrorRequest, opts ...grpc.CallOption) (*ScanMirrorReply, error) { out := new(ScanMirrorReply) err := c.cc.Invoke(ctx, "/CLI/ScanMirror", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) StatsFile(ctx context.Context, in *StatsFileRequest, opts ...grpc.CallOption) (*StatsFileReply, error) { out := new(StatsFileReply) err := c.cc.Invoke(ctx, "/CLI/StatsFile", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) StatsMirror(ctx context.Context, in *StatsMirrorRequest, opts ...grpc.CallOption) (*StatsMirrorReply, error) { out := new(StatsMirrorReply) err := c.cc.Invoke(ctx, "/CLI/StatsMirror", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) Ping(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { out := new(empty.Empty) err := c.cc.Invoke(ctx, "/CLI/Ping", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) GetMirrorLogs(ctx context.Context, in *GetMirrorLogsRequest, opts ...grpc.CallOption) (*GetMirrorLogsReply, error) { out := new(GetMirrorLogsReply) err := c.cc.Invoke(ctx, "/CLI/GetMirrorLogs", in, out, opts...) if err != nil { return nil, err } return out, nil } func (c *cLIClient) MatchMirror(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchReply, error) { out := new(MatchReply) err := c.cc.Invoke(ctx, "/CLI/MatchMirror", in, out, opts...) if err != nil { return nil, err } return out, nil } // CLIServer is the server API for CLI service. type CLIServer interface { GetVersion(context.Context, *empty.Empty) (*VersionReply, error) Upgrade(context.Context, *empty.Empty) (*empty.Empty, error) Reload(context.Context, *empty.Empty) (*empty.Empty, error) ChangeStatus(context.Context, *ChangeStatusRequest) (*empty.Empty, error) List(context.Context, *empty.Empty) (*MirrorListReply, error) MirrorInfo(context.Context, *MirrorIDRequest) (*Mirror, error) AddMirror(context.Context, *Mirror) (*AddMirrorReply, error) UpdateMirror(context.Context, *Mirror) (*UpdateMirrorReply, error) RemoveMirror(context.Context, *MirrorIDRequest) (*empty.Empty, error) GeoUpdateMirror(context.Context, *MirrorIDRequest) (*GeoUpdateMirrorReply, error) RefreshRepository(context.Context, *RefreshRepositoryRequest) (*empty.Empty, error) ScanMirror(context.Context, *ScanMirrorRequest) (*ScanMirrorReply, error) StatsFile(context.Context, *StatsFileRequest) (*StatsFileReply, error) StatsMirror(context.Context, *StatsMirrorRequest) (*StatsMirrorReply, error) Ping(context.Context, *empty.Empty) (*empty.Empty, error) GetMirrorLogs(context.Context, *GetMirrorLogsRequest) (*GetMirrorLogsReply, error) // Tools MatchMirror(context.Context, *MatchRequest) (*MatchReply, error) } // UnimplementedCLIServer can be embedded to have forward compatible implementations. type UnimplementedCLIServer struct { } func (*UnimplementedCLIServer) GetVersion(ctx context.Context, req *empty.Empty) (*VersionReply, error) { return nil, status.Errorf(codes.Unimplemented, "method GetVersion not implemented") } func (*UnimplementedCLIServer) Upgrade(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Upgrade not implemented") } func (*UnimplementedCLIServer) Reload(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Reload not implemented") } func (*UnimplementedCLIServer) ChangeStatus(ctx context.Context, req *ChangeStatusRequest) (*empty.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method ChangeStatus not implemented") } func (*UnimplementedCLIServer) List(ctx context.Context, req *empty.Empty) (*MirrorListReply, error) { return nil, status.Errorf(codes.Unimplemented, "method List not implemented") } func (*UnimplementedCLIServer) MirrorInfo(ctx context.Context, req *MirrorIDRequest) (*Mirror, error) { return nil, status.Errorf(codes.Unimplemented, "method MirrorInfo not implemented") } func (*UnimplementedCLIServer) AddMirror(ctx context.Context, req *Mirror) (*AddMirrorReply, error) { return nil, status.Errorf(codes.Unimplemented, "method AddMirror not implemented") } func (*UnimplementedCLIServer) UpdateMirror(ctx context.Context, req *Mirror) (*UpdateMirrorReply, error) { return nil, status.Errorf(codes.Unimplemented, "method UpdateMirror not implemented") } func (*UnimplementedCLIServer) RemoveMirror(ctx context.Context, req *MirrorIDRequest) (*empty.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method RemoveMirror not implemented") } func (*UnimplementedCLIServer) GeoUpdateMirror(ctx context.Context, req *MirrorIDRequest) (*GeoUpdateMirrorReply, error) { return nil, status.Errorf(codes.Unimplemented, "method GeoUpdateMirror not implemented") } func (*UnimplementedCLIServer) RefreshRepository(ctx context.Context, req *RefreshRepositoryRequest) (*empty.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method RefreshRepository not implemented") } func (*UnimplementedCLIServer) ScanMirror(ctx context.Context, req *ScanMirrorRequest) (*ScanMirrorReply, error) { return nil, status.Errorf(codes.Unimplemented, "method ScanMirror not implemented") } func (*UnimplementedCLIServer) StatsFile(ctx context.Context, req *StatsFileRequest) (*StatsFileReply, error) { return nil, status.Errorf(codes.Unimplemented, "method StatsFile not implemented") } func (*UnimplementedCLIServer) StatsMirror(ctx context.Context, req *StatsMirrorRequest) (*StatsMirrorReply, error) { return nil, status.Errorf(codes.Unimplemented, "method StatsMirror not implemented") } func (*UnimplementedCLIServer) Ping(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") } func (*UnimplementedCLIServer) GetMirrorLogs(ctx context.Context, req *GetMirrorLogsRequest) (*GetMirrorLogsReply, error) { return nil, status.Errorf(codes.Unimplemented, "method GetMirrorLogs not implemented") } func (*UnimplementedCLIServer) MatchMirror(ctx context.Context, req *MatchRequest) (*MatchReply, error) { return nil, status.Errorf(codes.Unimplemented, "method MatchMirror not implemented") } func RegisterCLIServer(s *grpc.Server, srv CLIServer) { s.RegisterService(&_CLI_serviceDesc, srv) } func _CLI_GetVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(empty.Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).GetVersion(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/GetVersion", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).GetVersion(ctx, req.(*empty.Empty)) } return interceptor(ctx, in, info, handler) } func _CLI_Upgrade_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(empty.Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).Upgrade(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/Upgrade", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).Upgrade(ctx, req.(*empty.Empty)) } return interceptor(ctx, in, info, handler) } func _CLI_Reload_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(empty.Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).Reload(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/Reload", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).Reload(ctx, req.(*empty.Empty)) } return interceptor(ctx, in, info, handler) } func _CLI_ChangeStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ChangeStatusRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).ChangeStatus(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/ChangeStatus", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).ChangeStatus(ctx, req.(*ChangeStatusRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_List_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(empty.Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).List(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/List", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).List(ctx, req.(*empty.Empty)) } return interceptor(ctx, in, info, handler) } func _CLI_MirrorInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MirrorIDRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).MirrorInfo(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/MirrorInfo", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).MirrorInfo(ctx, req.(*MirrorIDRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_AddMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Mirror) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).AddMirror(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/AddMirror", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).AddMirror(ctx, req.(*Mirror)) } return interceptor(ctx, in, info, handler) } func _CLI_UpdateMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Mirror) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).UpdateMirror(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/UpdateMirror", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).UpdateMirror(ctx, req.(*Mirror)) } return interceptor(ctx, in, info, handler) } func _CLI_RemoveMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MirrorIDRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).RemoveMirror(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/RemoveMirror", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).RemoveMirror(ctx, req.(*MirrorIDRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_GeoUpdateMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MirrorIDRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).GeoUpdateMirror(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/GeoUpdateMirror", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).GeoUpdateMirror(ctx, req.(*MirrorIDRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_RefreshRepository_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RefreshRepositoryRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).RefreshRepository(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/RefreshRepository", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).RefreshRepository(ctx, req.(*RefreshRepositoryRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_ScanMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ScanMirrorRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).ScanMirror(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/ScanMirror", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).ScanMirror(ctx, req.(*ScanMirrorRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_StatsFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(StatsFileRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).StatsFile(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/StatsFile", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).StatsFile(ctx, req.(*StatsFileRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_StatsMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(StatsMirrorRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).StatsMirror(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/StatsMirror", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).StatsMirror(ctx, req.(*StatsMirrorRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(empty.Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).Ping(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/Ping", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).Ping(ctx, req.(*empty.Empty)) } return interceptor(ctx, in, info, handler) } func _CLI_GetMirrorLogs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetMirrorLogsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).GetMirrorLogs(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/GetMirrorLogs", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).GetMirrorLogs(ctx, req.(*GetMirrorLogsRequest)) } return interceptor(ctx, in, info, handler) } func _CLI_MatchMirror_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MatchRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(CLIServer).MatchMirror(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: "/CLI/MatchMirror", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CLIServer).MatchMirror(ctx, req.(*MatchRequest)) } return interceptor(ctx, in, info, handler) } var _CLI_serviceDesc = grpc.ServiceDesc{ ServiceName: "CLI", HandlerType: (*CLIServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "GetVersion", Handler: _CLI_GetVersion_Handler, }, { MethodName: "Upgrade", Handler: _CLI_Upgrade_Handler, }, { MethodName: "Reload", Handler: _CLI_Reload_Handler, }, { MethodName: "ChangeStatus", Handler: _CLI_ChangeStatus_Handler, }, { MethodName: "List", Handler: _CLI_List_Handler, }, { MethodName: "MirrorInfo", Handler: _CLI_MirrorInfo_Handler, }, { MethodName: "AddMirror", Handler: _CLI_AddMirror_Handler, }, { MethodName: "UpdateMirror", Handler: _CLI_UpdateMirror_Handler, }, { MethodName: "RemoveMirror", Handler: _CLI_RemoveMirror_Handler, }, { MethodName: "GeoUpdateMirror", Handler: _CLI_GeoUpdateMirror_Handler, }, { MethodName: "RefreshRepository", Handler: _CLI_RefreshRepository_Handler, }, { MethodName: "ScanMirror", Handler: _CLI_ScanMirror_Handler, }, { MethodName: "StatsFile", Handler: _CLI_StatsFile_Handler, }, { MethodName: "StatsMirror", Handler: _CLI_StatsMirror_Handler, }, { MethodName: "Ping", Handler: _CLI_Ping_Handler, }, { MethodName: "GetMirrorLogs", Handler: _CLI_GetMirrorLogs_Handler, }, { MethodName: "MatchMirror", Handler: _CLI_MatchMirror_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "rpc.proto", } videolabs-mirrorbits-441567e/rpc/rpc.proto000066400000000000000000000077741523530551300206130ustar00rootroot00000000000000syntax = "proto3"; import "google/protobuf/empty.proto"; import "google/protobuf/timestamp.proto"; service CLI { rpc GetVersion (google.protobuf.Empty) returns (VersionReply) {} rpc Upgrade (google.protobuf.Empty) returns (google.protobuf.Empty) {} rpc Reload (google.protobuf.Empty) returns (google.protobuf.Empty) {} rpc ChangeStatus (ChangeStatusRequest) returns (google.protobuf.Empty) {} rpc List (google.protobuf.Empty) returns (MirrorListReply) {} rpc MirrorInfo (MirrorIDRequest) returns (Mirror) {} rpc AddMirror (Mirror) returns (AddMirrorReply) {} rpc UpdateMirror (Mirror) returns (UpdateMirrorReply) {} rpc RemoveMirror (MirrorIDRequest) returns (google.protobuf.Empty) {} rpc GeoUpdateMirror (MirrorIDRequest) returns (GeoUpdateMirrorReply) {} rpc RefreshRepository (RefreshRepositoryRequest) returns (google.protobuf.Empty) {} rpc ScanMirror (ScanMirrorRequest) returns (ScanMirrorReply) {} rpc StatsFile (StatsFileRequest) returns (StatsFileReply) {} rpc StatsMirror (StatsMirrorRequest) returns (StatsMirrorReply) {} rpc Ping (google.protobuf.Empty) returns (google.protobuf.Empty) {} rpc GetMirrorLogs (GetMirrorLogsRequest) returns (GetMirrorLogsReply) {} // Tools rpc MatchMirror (MatchRequest) returns (MatchReply) {} } message VersionReply { string Version = 1; string Build = 2; string GoVersion = 3; string OS = 4; string Arch = 5; int32 GoMaxProcs = 6; } message MatchRequest { string Pattern = 1; } message Mirror { int32 ID = 1; string Name = 2; string HttpURL = 3; string RsyncURL = 4; string FtpURL = 5; string SponsorName = 6; string SponsorURL = 7; string SponsorLogoURL = 8; string AdminName = 9; string AdminEmail = 10; string CustomData = 11; bool ContinentOnly = 12; bool CountryOnly = 13; bool ASOnly = 14; int32 Score = 15; float Latitude = 16; float Longitude = 17; string ContinentCode = 18; string CountryCodes = 19; string ExcludedCountryCodes = 20; uint32 Asnum = 21; string Comment = 22; bool Enabled = 23; bool HttpUp = 24; string HttpDownReason = 25; google.protobuf.Timestamp StateSince = 26; int32 AllowRedirects = 27; google.protobuf.Timestamp LastSync = 28; google.protobuf.Timestamp LastSuccessfulSync = 29; google.protobuf.Timestamp LastModTime = 30; bool HttpsUp = 31; string HttpsDownReason = 32; } message MirrorListReply { repeated Mirror Mirrors = 1; } message MirrorID { int32 ID = 1; string Name = 2; } message MatchReply { repeated MirrorID Mirrors = 1; } message ChangeStatusRequest { int32 ID = 1; bool Enabled = 2; } message MirrorIDRequest { int32 ID = 1; } message AddMirrorReply { float Latitude = 1; float Longitude = 2; string Country = 3; string Continent = 4; string ASN = 5; repeated string Warnings = 6; } message UpdateMirrorReply { string Diff = 1; } message GeoUpdateMirrorReply { Mirror Mirror = 1; string Diff = 2; repeated string Warnings = 3; } message RefreshRepositoryRequest { bool Rehash = 1; } message ScanMirrorRequest { int32 ID = 1; bool AutoEnable = 2; enum Method { ALL = 0; FTP = 1; RSYNC = 2; } Method Protocol = 3; } message ScanMirrorReply { bool Enabled = 1; int64 FilesIndexed = 2; int64 KnownIndexed = 3; int64 Removed = 4; int64 TZOffsetMs = 5; } message StatsFileRequest { string Pattern = 1; google.protobuf.Timestamp DateStart = 2; google.protobuf.Timestamp DateEnd = 3; } message StatsFileReply { map files = 1; } message StatsMirrorRequest { int32 ID = 1; google.protobuf.Timestamp DateStart = 2; google.protobuf.Timestamp DateEnd = 3; } message StatsMirrorReply { Mirror Mirror = 1; int64 Requests = 2; int64 Bytes = 3; } message GetMirrorLogsRequest { int32 ID = 1; int32 MaxResults = 2; } message GetMirrorLogsReply { repeated string line = 1; } videolabs-mirrorbits-441567e/rpc/rpc_test.go000066400000000000000000000037011523530551300210760ustar00rootroot00000000000000// Copyright (c) 2026 Amit Mishra // Licensed under the MIT license package rpc import ( "sort" "testing" ) func names(mirrors []*MirrorID) []string { var out []string for _, m := range mirrors { out = append(out, m.Name) } sort.Strings(out) return out } func TestMatchMirrorsByPattern(t *testing.T) { // Regression test for https://github.com/videolabs/mirrorbits/issues/134 tests := []struct { name string mirrors map[int]string pattern string want []string }{ { name: "exact match takes priority over substring matches", mirrors: map[int]string{ 1: "fcix.net", 2: "mirror.fcix.net", 3: "paducahix.mm.fcix.net", 4: "forksystems.mm.fcix.net", }, pattern: "fcix.net", want: []string{"fcix.net"}, }, { name: "exact match is case-insensitive", mirrors: map[int]string{ 1: "FCIX.net", 2: "mirror.fcix.net", }, pattern: "fcix.net", want: []string{"FCIX.net"}, }, { name: "multiple substring matches returned when no exact match", mirrors: map[int]string{ 1: "mirror.fcix.net", 2: "paducahix.mm.fcix.net", }, pattern: "fcix.net", want: []string{"mirror.fcix.net", "paducahix.mm.fcix.net"}, }, { name: "no match returns empty", mirrors: map[int]string{ 1: "alpha", 2: "beta", }, pattern: "gamma", want: nil, }, { name: "single substring match", mirrors: map[int]string{ 1: "mirror.example.com", 2: "other.example.org", }, pattern: "example.com", want: []string{"mirror.example.com"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { got := names(matchMirrorsByPattern(tc.mirrors, tc.pattern)) if len(got) != len(tc.want) { t.Fatalf("matchMirrorsByPattern(%q) = %v, want %v", tc.pattern, got, tc.want) } for i := range tc.want { if got[i] != tc.want[i] { t.Fatalf("matchMirrorsByPattern(%q) = %v, want %v", tc.pattern, got, tc.want) } } }) } } videolabs-mirrorbits-441567e/rpc/utils.go000066400000000000000000000071541523530551300204210ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package rpc import ( "github.com/etix/mirrorbits/mirrors" "github.com/golang/protobuf/ptypes" ) func MirrorToRPC(m *mirrors.Mirror) (*Mirror, error) { stateSince, err := ptypes.TimestampProto(m.StateSince.Time) if err != nil { return nil, err } lastSync, err := ptypes.TimestampProto(m.LastSync.Time) if err != nil { return nil, err } lastSuccessfulSync, err := ptypes.TimestampProto(m.LastSuccessfulSync.Time) if err != nil { return nil, err } lastModTime, err := ptypes.TimestampProto(m.LastModTime.Time) if err != nil { return nil, err } return &Mirror{ ID: int32(m.ID), Name: m.Name, HttpURL: m.HttpURL, RsyncURL: m.RsyncURL, FtpURL: m.FtpURL, SponsorName: m.SponsorName, SponsorURL: m.SponsorURL, SponsorLogoURL: m.SponsorLogoURL, AdminName: m.AdminName, AdminEmail: m.AdminEmail, CustomData: m.CustomData, ContinentOnly: m.ContinentOnly, CountryOnly: m.CountryOnly, ASOnly: m.ASOnly, Score: int32(m.Score), Latitude: m.Latitude, Longitude: m.Longitude, ContinentCode: m.ContinentCode, CountryCodes: m.CountryCodes, ExcludedCountryCodes: m.ExcludedCountryCodes, Asnum: uint32(m.Asnum), Comment: m.Comment, Enabled: m.Enabled, HttpUp: m.HttpUp, HttpsUp: m.HttpsUp, HttpDownReason: m.HttpDownReason, HttpsDownReason: m.HttpsDownReason, StateSince: stateSince, AllowRedirects: int32(m.AllowRedirects), LastSync: lastSync, LastSuccessfulSync: lastSuccessfulSync, LastModTime: lastModTime, }, nil } func MirrorFromRPC(m *Mirror) (*mirrors.Mirror, error) { stateSince, err := ptypes.Timestamp(m.StateSince) if err != nil { return nil, err } lastSync, err := ptypes.Timestamp(m.LastSync) if err != nil { return nil, err } lastSuccessfulSync, err := ptypes.Timestamp(m.LastSuccessfulSync) if err != nil { return nil, err } lastModTime, err := ptypes.Timestamp(m.LastModTime) if err != nil { return nil, err } return &mirrors.Mirror{ ID: int(m.ID), Name: m.Name, HttpURL: m.HttpURL, RsyncURL: m.RsyncURL, FtpURL: m.FtpURL, SponsorName: m.SponsorName, SponsorURL: m.SponsorURL, SponsorLogoURL: m.SponsorLogoURL, AdminName: m.AdminName, AdminEmail: m.AdminEmail, CustomData: m.CustomData, ContinentOnly: m.ContinentOnly, CountryOnly: m.CountryOnly, ASOnly: m.ASOnly, Score: int(m.Score), Latitude: m.Latitude, Longitude: m.Longitude, ContinentCode: m.ContinentCode, CountryCodes: m.CountryCodes, ExcludedCountryCodes: m.ExcludedCountryCodes, Asnum: uint(m.Asnum), Comment: m.Comment, Enabled: m.Enabled, HttpUp: m.HttpUp, HttpsUp: m.HttpsUp, HttpDownReason: m.HttpDownReason, HttpsDownReason: m.HttpsDownReason, StateSince: mirrors.Time{}.FromTime(stateSince), AllowRedirects: mirrors.Redirects(m.AllowRedirects), LastSync: mirrors.Time{}.FromTime(lastSync), LastSuccessfulSync: mirrors.Time{}.FromTime(lastSuccessfulSync), LastModTime: mirrors.Time{}.FromTime(lastModTime), }, nil } videolabs-mirrorbits-441567e/scan/000077500000000000000000000000001523530551300170635ustar00rootroot00000000000000videolabs-mirrorbits-441567e/scan/ftp.go000066400000000000000000000073231523530551300202100ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package scan import ( "fmt" "net/url" "strings" "time" ftp "github.com/etix/goftp" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/utils" "github.com/gomodule/redigo/redis" ) const ( ftpConnTimeout = 5 * time.Second ftpRWTimeout = 30 * time.Second ) // FTPScanner is the implementation of an ftp scanner type FTPScanner struct { scan *scan featMLST bool featMDTM bool precision core.Precision // Used for truncating time for comparison } // Scan starts an ftp scan of the given mirror func (f *FTPScanner) Scan(scanurl, identifier string, conn redis.Conn, stop <-chan struct{}) (core.Precision, error) { if !strings.HasPrefix(scanurl, "ftp://") { return 0, fmt.Errorf("%s does not start with ftp://", scanurl) } ftpurl, err := url.Parse(scanurl) if err != nil { return 0, err } host := ftpurl.Host if !strings.Contains(host, ":") { host += ":21" } if utils.IsStopped(stop) { return 0, ErrScanAborted } c, err := ftp.DialTimeout(host, ftpConnTimeout, ftpRWTimeout) if err != nil { return 0, err } defer c.Quit() username, password := "anonymous", "anonymous" if ftpurl.User != nil { username = ftpurl.User.Username() pass, hasPassword := ftpurl.User.Password() if hasPassword { password = pass } } err = c.Login(username, password) if err != nil { return 0, err } _, f.featMLST = c.Feature("MLST") _, f.featMDTM = c.Feature("MDTM") if !f.featMLST || !f.featMDTM { log.Warning("This server does not support some of the RFC 3659 extensions, consider using rsync instead.") } log.Infof("[%s] Requesting file list via ftp...", identifier) files := make([]*filedata, 0, 1000) err = c.ChangeDir(ftpurl.Path) if err != nil { return 0, fmt.Errorf("ftp error %s", err.Error()) } _, err = c.CurrentDir() if err != nil { return 0, fmt.Errorf("ftp error %s", err.Error()) } // Remove the trailing slash prefix := strings.TrimRight(ftpurl.Path, "/") files, err = f.walkFtp(c, files, prefix+"/", stop) if err != nil { return 0, fmt.Errorf("ftp error %s", err.Error()) } count := 0 for _, fd := range files { fd.path = strings.TrimPrefix(fd.path, prefix) f.scan.ScannerAddFile(*fd) count++ } return f.precision, nil } // Walk inside an FTP repository func (f *FTPScanner) walkFtp(c *ftp.ServerConn, files []*filedata, path string, stop <-chan struct{}) ([]*filedata, error) { if utils.IsStopped(stop) { return nil, ErrScanAborted } flist, err := c.List(path) if err != nil { return nil, err } for _, e := range flist { if e.Type == ftp.EntryTypeFile { newf := &filedata{} newf.path = path + e.Name newf.size = int64(e.Size) if f.featMDTM { t, _ := c.LastModificationDate(path + e.Name) if !t.IsZero() { newf.modTime = t if f.precision != core.Precision(time.Millisecond) { // We are not yet sure that we can have millisecond precision if newf.modTime.Truncate(time.Second).Equal(newf.modTime) { // The mod time is precise up to the second (for this file) f.precision = core.Precision(time.Second) } else { // The mod time is precise up to the millisecond f.precision = core.Precision(time.Millisecond) } } } } if newf.modTime.IsZero() { if f.featMLST { newf.modTime = e.Time if f.precision == 0 { f.precision = core.Precision(time.Second) } } else { newf.modTime = time.Time{} } } files = append(files, newf) } else if e.Type == ftp.EntryTypeFolder { if e.Name == "." || e.Name == ".." { continue } files, err = f.walkFtp(c, files, path+e.Name+"/", stop) if err != nil { return files, err } } } return files, err } videolabs-mirrorbits-441567e/scan/rsync.go000066400000000000000000000121301523530551300205450ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package scan import ( "bufio" "errors" "fmt" "io" "net/url" "os/exec" "regexp" "strconv" "strings" "time" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/utils" "github.com/gomodule/redigo/redis" ) var ( rsyncOutputLine = regexp.MustCompile(`^.+\s+([0-9,]+)\s+([0-9/]+)\s+([0-9:]+)\s+(.*)$`) ) // RsyncScanner is the implementation of an rsync scanner type RsyncScanner struct { scan *scan } // Scan starts an rsync scan of the given mirror func (r *RsyncScanner) Scan(rsyncURL, identifier string, conn redis.Conn, stop <-chan struct{}) (core.Precision, error) { var env []string var cmdName string var actualURL string // We allow a custom rsyncs:// scheme which uses rsync-ssl instead if strings.HasPrefix(rsyncURL, "rsync://") { cmdName = "rsync" actualURL = rsyncURL } else if strings.HasPrefix(rsyncURL, "rsyncs://") { cmdName = "rsync-ssl" actualURL = "rsync://" + strings.TrimPrefix(rsyncURL, "rsyncs://") } else { return 0, fmt.Errorf("%s does not start with rsync:// or rsyncs://", rsyncURL) } u, err := url.Parse(actualURL) if err != nil { return 0, err } // Extract the credentials if u.User != nil { if u.User.Username() != "" { env = append(env, fmt.Sprintf("USER=%s", u.User.Username())) } if password, ok := u.User.Password(); ok { env = append(env, fmt.Sprintf("RSYNC_PASSWORD=%s", password)) } // Remove the credentials from the URL as we pass them through the environnement u.User = nil } // Don't use the local timezone, use UTC env = append(env, "TZ=UTC") args := []string{"-r", "--no-motd", "--exclude=.~tmp~/"} // rsync-ssl does not support --contimeout, but from what I see --timeout covers both and // triggers if openssl takes too long to connect. So use a longer combined timeout there. if cmdName == "rsync-ssl" { args = append(args, "--timeout=60") } else { args = append(args, "--timeout=30", "--contimeout=30") } args = append(args, u.String()) cmd := exec.Command(cmdName, args...) // Setup the environnement cmd.Env = env stdout, err := cmd.StdoutPipe() if err != nil { return 0, err } stderr, err := cmd.StderrPipe() if err != nil { return 0, err } // Pipe stdout reader := bufio.NewReader(stdout) readerErr := bufio.NewReader(stderr) if utils.IsStopped(stop) { return 0, ErrScanAborted } // Start the process if err := cmd.Start(); err != nil { return 0, err } log.Infof("[%s] Requesting file list via %s...", identifier, cmdName) scanfinished := make(chan bool) go func() { select { case <-stop: cmd.Process.Kill() return case <-scanfinished: return } }() defer close(scanfinished) line, err := readln(reader) for err == nil { var size int64 var f filedata var modTime time.Time var modString string if utils.IsStopped(stop) { return 0, ErrScanAborted } // Parse one line returned by rsync ret := rsyncOutputLine.FindStringSubmatch(line) if ret[0][0] == 'd' || ret[0][0] == 'l' { // Skip directories and links goto cont } // Add the leading slash if ret[4][0] != '/' { ret[4] = "/" + ret[4] } // Parse the mod time modString = ret[2] + " " + ret[3] modTime, err = time.Parse("2006/01/02 15:04:05", modString) if err != nil { log.Errorf("[%s] ScanRsync: Invalid mod time: %s", identifier, modString) goto cont } // Remove the commas in the file size ret[1] = strings.Replace(ret[1], ",", "", -1) // Convert the size to int size, err = strconv.ParseInt(ret[1], 10, 64) if err != nil { log.Errorf("[%s] ScanRsync: Invalid size: %s", identifier, ret[1]) goto cont } // Fill the struct f.size = size f.modTime = modTime f.path = ret[4] r.scan.ScannerAddFile(f) cont: line, err = readln(reader) } rsyncErrors := []string{} for line, err = readln(readerErr); err == nil; line, err = readln(readerErr) { if strings.Contains(line, ": opendir ") { rsyncErrors = append(rsyncErrors, line) } } if err1 := cmd.Wait(); err1 != nil { switch err1.Error() { case "exit status 5": err1 = errors.New("rsync: Error starting client-server protocol") case "exit status 10": err1 = errors.New("rsync: Error in socket I/O") case "exit status 11": err1 = errors.New("rsync: Error in file I/O") case "exit status 23": for _, line := range rsyncErrors { log.Warningf("[%s] %s", identifier, line) } log.Warningf("[%s] rsync: Partial transfer due to error", identifier) err1 = nil case "exit status 30": err1 = errors.New("rsync: Timeout in data send/receive") case "exit status 35": err1 = errors.New("Timeout waiting for daemon connection") default: if utils.IsStopped(stop) { err1 = ErrScanAborted } else { err1 = errors.New("rsync: " + err1.Error()) } } return 0, err1 } if err != io.EOF { return 0, err } return core.Precision(time.Second), nil } func readln(r *bufio.Reader) (string, error) { var ( isPrefix = true err error line, ln []byte ) for isPrefix && err == nil { line, isPrefix, err = r.ReadLine() ln = append(ln, line...) } return string(ln), err } videolabs-mirrorbits-441567e/scan/scan.go000066400000000000000000000323661523530551300203500ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package scan import ( "errors" "fmt" "os" "path/filepath" "strconv" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/filesystem" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/network" "github.com/etix/mirrorbits/utils" "github.com/gomodule/redigo/redis" "github.com/op/go-logging" ) var ( // ErrScanAborted is returned when a scan is aborted by the user ErrScanAborted = errors.New("scan aborted") // ErrScanInProgress is returned when a scan is started while another is already in progress ErrScanInProgress = errors.New("scan already in progress") // ErrNoSyncMethod is returned when no sync protocol is available ErrNoSyncMethod = errors.New("no suitable URL for the scan") log = logging.MustGetLogger("main") ) // Scanner is the interface that all scanners must implement type Scanner interface { Scan(url, identifier string, conn redis.Conn, stop <-chan struct{}) (core.Precision, error) } type filedata struct { path string sha1 string sha256 string md5 string size int64 modTime time.Time } type scan struct { redis *database.Redis cache *mirrors.Cache conn redis.Conn mirrorid int filesTmpKey string count int64 } type ScanResult struct { MirrorID int MirrorName string FilesIndexed int64 KnownIndexed int64 Removed int64 TZOffsetMs int64 } // IsScanning returns true is a scan is already in progress for the given mirror func IsScanning(conn redis.Conn, id int) (bool, error) { return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%d", id))) } // Scan starts a scan of the given mirror func Scan(typ core.ScannerType, r *database.Redis, c *mirrors.Cache, url string, id int, stop <-chan struct{}) (*ScanResult, error) { // Connect to the database conn := r.Get() defer conn.Close() s := &scan{ redis: r, mirrorid: id, conn: conn, cache: c, } var scanner Scanner switch typ { case core.RSYNC: scanner = &RsyncScanner{ scan: s, } case core.FTP: scanner = &FTPScanner{ scan: s, } default: panic(fmt.Sprintf("Unknown scanner")) } // Get the mirror name name, err := redis.String(conn.Do("HGET", "MIRRORS", id)) if err != nil { return nil, err } // Try to acquire a lock so we don't have a scanning race // from different nodes. // Also make the key expire automatically in case our process // gets killed. lock := network.NewClusterLock(s.redis, fmt.Sprintf("SCANNING_%d", id), name) done, err := lock.Get() if err != nil { return nil, err } else if done == nil { return nil, ErrScanInProgress } defer lock.Release() s.setLastSync(conn, id, typ, 0, false) mirrors.PushLog(r, mirrors.NewLogScanStarted(id, typ)) defer func(err *error) { if err != nil && *err != nil { mirrors.PushLog(r, mirrors.NewLogError(id, *err)) } }(&err) conn.Send("MULTI") filesKey := fmt.Sprintf("MIRRORFILES_%d", id) s.filesTmpKey = fmt.Sprintf("MIRRORFILESTMP_%d", id) // Remove any left over conn.Send("DEL", s.filesTmpKey) var precision core.Precision precision, err = scanner.Scan(url, name, conn, stop) if err != nil { // Discard MULTI s.ScannerDiscard() // Remove the temporary key conn.Do("DEL", s.filesTmpKey) log.Errorf("[%s] %s", name, err.Error()) return nil, err } log.Infof("[%s] Indexing the files...", name) // Exec multi s.ScannerCommit() // Get the list of files no more present on this mirror var toremove []any toremove, err = redis.Values(conn.Do("SDIFF", filesKey, s.filesTmpKey)) if err != nil { return nil, err } // Remove this mirror from the given file SET if len(toremove) > 0 { conn.Send("MULTI") for _, e := range toremove { log.Debugf("[%s] Removing %s from mirror", name, e) conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), id) conn.Send("DEL", fmt.Sprintf("FILEINFO_%d_%s", id, e)) // Publish update database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%d %s", id, e)) } _, err = conn.Do("EXEC") if err != nil { return nil, err } } // Finally rename the temporary sets containing the list // of files for this mirror to the production key if s.count > 0 { _, err = conn.Do("RENAME", s.filesTmpKey, filesKey) if err != nil { return nil, err } } sinterKey := fmt.Sprintf("HANDLEDFILES_%d", id) // Count the number of files known on the remote end common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", filesKey)) if err != nil { return nil, err } s.setLastSync(conn, id, typ, precision, true) var tzoffset int64 tzoffset, err = s.adjustTZOffset(name, precision) if err != nil { log.Warningf("Unable to check timezone shifts: %s", err) } log.Infof("[%s] Indexed %d files (%d known), %d removed", name, s.count, common, len(toremove)) res := &ScanResult{ MirrorID: id, MirrorName: name, FilesIndexed: s.count, KnownIndexed: common, Removed: int64(len(toremove)), TZOffsetMs: tzoffset, } mirrors.PushLog(r, mirrors.NewLogScanCompleted( res.MirrorID, res.FilesIndexed, res.KnownIndexed, res.Removed, res.TZOffsetMs)) return res, nil } func (s *scan) ScannerAddFile(f filedata) { s.count++ // Add all the files to a temporary key s.conn.Send("SADD", s.filesTmpKey, f.path) // Mark the file as being supported by this mirror rk := fmt.Sprintf("FILEMIRRORS_%s", f.path) s.conn.Send("SADD", rk, s.mirrorid) // Save the size of the current file found on this mirror ik := fmt.Sprintf("FILEINFO_%d_%s", s.mirrorid, f.path) s.conn.Send("HSET", ik, "size", f.size, "modTime", f.modTime) // Publish update database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%d %s", s.mirrorid, f.path)) } func (s *scan) ScannerDiscard() { s.conn.Do("DISCARD") } func (s *scan) ScannerCommit() error { _, err := s.conn.Do("EXEC") return err } func (s *scan) setLastSync(conn redis.Conn, id int, protocol core.ScannerType, precision core.Precision, successful bool) error { now := time.Now().UTC().Unix() conn.Send("MULTI") // Set the last sync time conn.Send("HSET", fmt.Sprintf("MIRROR_%d", id), "lastSync", now) // Set the last successful sync time if successful { if precision == 0 { precision = core.Precision(time.Second) } conn.Send("HSET", fmt.Sprintf("MIRROR_%d", id), "lastSuccessfulSync", now, "lastSuccessfulSyncProtocol", protocol, "lastSuccessfulSyncPrecision", precision) } _, err := conn.Do("EXEC") // Publish an update on redis database.Publish(conn, database.MIRROR_UPDATE, strconv.Itoa(id)) return err } func (s *scan) adjustTZOffset(name string, precision core.Precision) (ms int64, err error) { type pair struct { local filesystem.FileInfo remote filesystem.FileInfo } var filepaths []string var pairs []pair var offsetmap map[int64]int var commonOffsetFound bool if s.cache == nil { log.Error("Skipping timezone check: missing cache in instance") return } if GetConfig().FixTimezoneOffsets == false { // We need to reset any previous value already // stored in the database. goto finish } // Get 100 random files from the mirror filepaths, err = redis.Strings(s.conn.Do("SRANDMEMBER", fmt.Sprintf("HANDLEDFILES_%d", s.mirrorid), 100)) if err != nil { return } pairs = make([]pair, 0, 100) // Get the metadata of each file for _, path := range filepaths { p := pair{} p.local, err = s.cache.GetFileInfo(path) if err != nil { return } p.remote, err = s.cache.GetFileInfoMirror(s.mirrorid, path) if err != nil { return } if p.remote.ModTime.IsZero() { // Invalid mod time continue } if p.local.Size != p.remote.Size { // File differ: comparing the modfile will fail continue } // Add the file to valid pairs pairs = append(pairs, p) } if len(pairs) < 10 || len(pairs) < len(filepaths)/2 { // Less than half the files we got have a size // match, this is very suspicious. Skip the // check and reset the offset in the db. goto warn } // Compute the diff between local and remote for those files offsetmap = make(map[int64]int) for _, p := range pairs { // Convert to millisecond since unix timestamp truncating to the available precision local := p.local.ModTime.Truncate(precision.Duration()).UnixNano() / int64(time.Millisecond) remote := p.remote.ModTime.Truncate(precision.Duration()).UnixNano() / int64(time.Millisecond) diff := local - remote offsetmap[diff]++ } for k, v := range offsetmap { // Find the common offset (if any) of at least 90% of our subset if v >= int(float64(len(pairs))/100*90) { ms = k commonOffsetFound = true break } } warn: if !commonOffsetFound { log.Warningf("[%s] Unable to guess the timezone offset", name) } finish: // Store the offset in the database key := fmt.Sprintf("MIRROR_%d", s.mirrorid) _, err = s.conn.Do("HSET", key, "tzoffset", ms) if err != nil { return } // Publish update database.Publish(s.conn, database.MIRROR_UPDATE, strconv.Itoa(s.mirrorid)) if ms != 0 { log.Noticef("[%s] Timezone offset detected: applied correction of %dms", name, ms) } return } type sourcescanner struct { } // Walk inside the source/reference repository func (s *sourcescanner) walkSource(conn redis.Conn, path string, f os.FileInfo, rehash bool, err error) (*filedata, error) { if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 { return nil, nil } d := new(filedata) d.path = path[len(GetConfig().Repository):] d.size = f.Size() d.modTime = f.ModTime() // Get the previous file properties properties, err := redis.Strings(conn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5")) if err != nil && err != redis.ErrNil { return nil, err } else if len(properties) < 5 { // This will force a rehash properties = make([]string, 5) } size, _ := strconv.ParseInt(properties[0], 10, 64) modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1]) sha1 := properties[2] sha256 := properties[3] md5 := properties[4] rehash = rehash || (GetConfig().Hashes.SHA1 && len(sha1) == 0) || (GetConfig().Hashes.SHA256 && len(sha256) == 0) || (GetConfig().Hashes.MD5 && len(md5) == 0) if rehash || size != d.size || !modTime.Equal(d.modTime) { h, err := filesystem.HashFile(GetConfig().Repository + d.path) if err != nil { log.Warningf("%s: hashing failed: %s", d.path, err.Error()) } else { d.sha1 = h.Sha1 d.sha256 = h.Sha256 d.md5 = h.Md5 if len(d.sha1) > 0 { log.Infof("%s: SHA1 %s", d.path, d.sha1) } if len(d.sha256) > 0 { log.Infof("%s: SHA256 %s", d.path, d.sha256) } if len(d.md5) > 0 { log.Infof("%s: MD5 %s", d.path, d.md5) } } } else { d.sha1 = sha1 d.sha256 = sha256 d.md5 = md5 } return d, nil } // ScanSource starts a scan of the local repository func ScanSource(r *database.Redis, forceRehash bool, stop <-chan struct{}) (err error) { s := &sourcescanner{} conn := r.Get() defer conn.Close() if conn.Err() != nil { return conn.Err() } sourceFiles := make([]*filedata, 0, 1000) //TODO lock atomically inside redis to avoid two simultaneous scan if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) { return fmt.Errorf("%s: No such file or directory", GetConfig().Repository) } log.Info("[source] Scanning the filesystem...") err = filepath.Walk(GetConfig().Repository, func(path string, f os.FileInfo, err error) error { fd, err := s.walkSource(conn, path, f, forceRehash, err) if err != nil { return err } if fd != nil { sourceFiles = append(sourceFiles, fd) } return nil }) if utils.IsStopped(stop) { return ErrScanAborted } if err != nil { return err } log.Info("[source] Indexing the files...") lock := network.NewClusterLock(r, "SOURCE_REPO_SYNC", "source repository") retry := 10 for { if retry == 0 { return ErrScanInProgress } done, err := lock.Get() if err != nil { return err } else if done != nil { break } time.Sleep(1 * time.Second) retry-- } defer lock.Release() conn.Send("MULTI") // Remove any left over conn.Send("DEL", "FILES_TMP") // Add all the files to a temporary key count := 0 for _, e := range sourceFiles { conn.Send("SADD", "FILES_TMP", e.path) count++ } _, err = conn.Do("EXEC") if err != nil { return err } // Do a diff between the sets to get the removed files toremove, err := redis.Values(conn.Do("SDIFF", "FILES", "FILES_TMP")) if err != nil { return err } // Create/Update the files' hash keys with the fresh infos conn.Send("MULTI") for _, e := range sourceFiles { conn.Send("HSET", fmt.Sprintf("FILE_%s", e.path), "size", e.size, "modTime", e.modTime, "sha1", e.sha1, "sha256", e.sha256, "md5", e.md5) // Publish update database.SendPublish(conn, database.FILE_UPDATE, e.path) } // Remove old keys if len(toremove) > 0 { for _, e := range toremove { conn.Send("DEL", fmt.Sprintf("FILE_%s", e)) // Publish update database.SendPublish(conn, database.FILE_UPDATE, fmt.Sprintf("%s", e)) } } // Finally rename the temporary sets containing the list // of files to the production key conn.Send("RENAME", "FILES_TMP", "FILES") _, err = conn.Do("EXEC") if err != nil { return err } log.Infof("[source] Indexed %d files, %d removed", count, len(toremove)) return nil } videolabs-mirrorbits-441567e/scan/trace.go000066400000000000000000000063231523530551300205140ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package scan import ( "bufio" "context" "errors" "fmt" "net" "net/http" "strconv" "time" . "github.com/etix/mirrorbits/config" "github.com/etix/mirrorbits/core" "github.com/etix/mirrorbits/database" "github.com/etix/mirrorbits/mirrors" "github.com/etix/mirrorbits/utils" ) var ( userAgent = "Mirrorbits/" + core.VERSION + " TRACE" clientTimeout = time.Duration(20 * time.Second) clientDeadline = time.Duration(40 * time.Second) // ErrNoTrace is returned when no trace file is found ErrNoTrace = errors.New("No trace file") ) // Trace is the internal trace handler type Trace struct { redis *database.Redis transport http.Transport httpClient http.Client stop <-chan struct{} } // NewTraceHandler returns a new instance of the trace file handler. // Trace files are used to compute the time offset between a mirror // and the local repository. func NewTraceHandler(redis *database.Redis, stop <-chan struct{}) *Trace { t := &Trace{ redis: redis, stop: stop, } t.transport = http.Transport{ DisableKeepAlives: true, MaxIdleConnsPerHost: 0, Dial: func(network, addr string) (net.Conn, error) { deadline := time.Now().Add(clientDeadline) c, err := net.DialTimeout(network, addr, clientTimeout) if err != nil { return nil, err } c.SetDeadline(deadline) return c, nil }, } t.httpClient = http.Client{ Transport: &t.transport, } return t } // GetLastUpdate connects in HTTP to the mirror to get the latest // trace file and computes the offset of the mirror. func (t *Trace) GetLastUpdate(mirror mirrors.Mirror) error { traceFile := GetConfig().TraceFileLocation if len(traceFile) == 0 { return ErrNoTrace } log.Debugf("Getting latest trace file for %s...", mirror.Name) // Prepare the mirror URL var mirrorURL string if utils.HasAnyPrefix(mirror.HttpURL, "http://", "https://") { mirrorURL = mirror.HttpURL } else if mirror.HttpsUp == true { mirrorURL = "https://" + mirror.HttpURL } else { mirrorURL = "http://" + mirror.HttpURL } // Prepare the HTTP request req, err := http.NewRequest("GET", utils.ConcatURL(mirrorURL, traceFile), nil) req.Header.Set("User-Agent", userAgent) req.Close = true // Prepare contexts ctx, cancel := context.WithTimeout(req.Context(), clientDeadline) ctx = context.WithValue(ctx, core.ContextMirrorID, mirror.ID) req = req.WithContext(ctx) defer cancel() go func() { select { case <-t.stop: cancel() case <-ctx.Done(): } }() resp, err := t.httpClient.Do(req) if err != nil { return err } defer resp.Body.Close() scanner := bufio.NewScanner(bufio.NewReader(resp.Body)) scanner.Split(bufio.ScanWords) scanner.Scan() if err := scanner.Err(); err != nil { return err } timestamp, err := strconv.ParseInt(scanner.Text(), 10, 64) if err != nil { return err } conn := t.redis.Get() defer conn.Close() _, err = conn.Do("HSET", fmt.Sprintf("MIRROR_%d", mirror.ID), "lastModTime", timestamp) if err != nil { return err } // Publish an update on redis database.Publish(conn, database.MIRROR_UPDATE, strconv.Itoa(mirror.ID)) log.Debugf("[%s] trace last sync: %s", mirror.Name, time.Unix(timestamp, 0)) return nil } videolabs-mirrorbits-441567e/templates/000077500000000000000000000000001523530551300201355ustar00rootroot00000000000000videolabs-mirrorbits-441567e/templates/base.html000066400000000000000000000127571523530551300217510ustar00rootroot00000000000000{{define "base"}} {{template "title" .}} {{if not .LocalJSPath}} {{else}} {{end}} {{template "head" .}}
{{template "body" .}}
{{end}} videolabs-mirrorbits-441567e/templates/mirrorlist.html000066400000000000000000000270031523530551300232330ustar00rootroot00000000000000{{define "title"}}Mirrorlist {{.FileInfo.Path}}{{end}} {{define "headline"}}{{.FileInfo.Path}}{{end}} {{define "head"}} {{if not .LocalJSPath}} {{else}} {{end}} {{end}} {{define "body"}}

Client

You are connecting with IP address {{.IP}}, which belongs to autonomous system {{.ClientInfo.ASName}} (ASN{{.ClientInfo.ASNum}}).
{{if .ClientInfo.IsValid}}We believe you are {{if .ClientInfo.City}}near {{.ClientInfo.City}} in {{else}}somewhere in {{end}}{{.ClientInfo.Country}} and have selected mirrors based on this.{{else}}We were not able to use your IP to approximate your location, so have chosen the mirrors at random.{{end}}

File

{{if not (iszero .FileInfo.ModTime)}} The file {{.FileInfo.Path}} has a size of {{sizeof .FileInfo.Size}} ({{.FileInfo.Size}} bytes) and was last modified on {{dateutc .FileInfo.ModTime}}. {{else}} The file {{.FileInfo.Path}} has not been scanned yet, size and modification time are unknown. {{end}}

Known hashes:
MD5{{if .FileInfo.Md5}}{{.FileInfo.Md5}}{{else}}N/A{{end}}
SHA1{{if .FileInfo.Sha1}}{{.FileInfo.Sha1}}{{else}}N/A{{end}}
SHA256{{if .FileInfo.Sha256}}{{.FileInfo.Sha256}}{{else}}N/A{{end}}


Mirrors

{{if .Fallback}}

Warning: file not served by any mirror, fallbacks to the rescue.

{{end}} {{if .MirrorList}} {{range $i, $v := .MirrorList}} {{end}}
RankMirror NameURLCountryContinentDistanceSelection
{{add $i 1}}.{{if $v.SponsorName}}{{$v.SponsorName}}{{else}}{{$v.Name}}{{end}}{{$v.AbsoluteURL}}{{$v.CountryCodes}}{{$v.ContinentCode}}{{printf "%.0f" $v.Distance}} Km{{if $v.Weight}}{{if ge $v.Weight 1.0}}{{printf "%.0f" $v.Weight}}{{else}}<1{{end}}%{{else}}n/a{{end}}
{{else}} No mirrors for this file {{end}} {{if .ExcludedList}}

Excluded Mirrors

{{range $i, $v := .ExcludedList}} {{end}}
Mirror NameURLCountryContinentDistanceExclude Reason
{{if $v.SponsorName}}{{$v.SponsorName}}{{else}}{{$v.Name}}{{end}}{{$v.AbsoluteURL}}{{$v.CountryCodes}}{{$v.ContinentCode}}{{printf "%.0f" $v.Distance}} Km{{$v.ExcludeReason}}
{{end}}
{{end}} videolabs-mirrorbits-441567e/templates/mirrorstats.html000066400000000000000000000220151523530551300234140ustar00rootroot00000000000000{{define "title"}}Mirrorstats{{end}} {{define "headline"}}Mirrorstats{{end}} {{define "head"}} {{if not .LocalJSPath}} {{else}} {{end}} {{end}} {{define "body"}}
{{if .HasTZAdjustement}}{{end}} {{range $i, $v := .List}} {{if $.HasTZAdjustement}}{{end}} {{end}}
Mirror Since 00:00 UTC… Last updateAdjusted TZ
{{$v.Name}}
{{$v.Downloads}}
downloads
 {{if $v.SyncOffset.Valid}}{{$v.SyncOffset.HumanReadable}}{{else}}unknown{{end}}{{if ne $v.TZOffset 0}}{{$v.TZOffset}}{{end}}
{{sizeof $v.Bytes}}
transferred
{{end}} videolabs-mirrorbits-441567e/testing/000077500000000000000000000000001523530551300176145ustar00rootroot00000000000000videolabs-mirrorbits-441567e/testing/redis.go000066400000000000000000000011701523530551300212500ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package testing import ( "github.com/etix/mirrorbits/database" "github.com/gomodule/redigo/redis" "github.com/rafaeljusto/redigomock" ) type redisPoolMock struct { Conn *redigomock.Conn } func (r *redisPoolMock) Get() redis.Conn { return r.Conn } func (r *redisPoolMock) Close() error { return nil } // PrepareRedisTest initialize redis tests func PrepareRedisTest() (*redigomock.Conn, *database.Redis) { mock := redigomock.NewConn() pool := &redisPoolMock{ Conn: mock, } conn := database.NewRedisCustomPool(pool) return mock, conn } videolabs-mirrorbits-441567e/utils/000077500000000000000000000000001523530551300172775ustar00rootroot00000000000000videolabs-mirrorbits-441567e/utils/utils.go000066400000000000000000000131631523530551300207720ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package utils import ( "fmt" "math" "os" "strings" "time" "github.com/etix/mirrorbits/core" ) const ( // DegToRad is a constant to convert degrees to radians DegToRad = 0.017453292519943295769236907684886127134428718885417 // N[Pi/180, 50] // RadToDeg is a constant to convert radians to degrees RadToDeg = 57.295779513082320876798154814105170332405472466564 // N[180/Pi, 50] ) // HasAnyPrefix reports whether the string s begins with any of the prefixes func HasAnyPrefix(s string, prefixes ...string) bool { for _, p := range prefixes { if strings.HasPrefix(s, p) { return true } } return false } // NormalizeURL adds a trailing slash to the URL func NormalizeURL(url string) string { if url != "" && !strings.HasSuffix(url, "/") { url += "/" } return url } // GetDistanceKm returns the distance in km between two coordinates func GetDistanceKm(lat1, lon1, lat2, lon2 float32) float32 { var R float32 = 6371 // radius of the earth in Km dLat := (lat2 - lat1) * float32(DegToRad) dLon := (lon2 - lon1) * float32(DegToRad) a := math.Sin(float64(dLat/2))*math.Sin(float64(dLat/2)) + math.Cos(float64(lat1*DegToRad))*math.Cos(float64(lat2*DegToRad))*math.Sin(float64(dLon/2))*math.Sin(float64(dLon/2)) c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a)) return R * float32(c) } // Min returns the smallest of the two values func Min(v1, v2 int) int { if v1 < v2 { return v1 } return v2 } // Max returns the highest of the two values func Max(v1, v2 int) int { if v1 > v2 { return v1 } return v2 } // Add does a simple addition func Add(x, y int) int { return x + y } // Version returns the version as a string func Version() string { return core.VERSION } // Hostname return the host name as a string func Hostname() string { hostname, _ := os.Hostname() return hostname } // IsInSlice returns true is `a` is contained in `list` // Warning: this is slow, don't use it for long datasets func IsInSlice(a string, list []string) bool { for _, b := range list { if b == a { return true } } return false } // IsStopped returns true if a stop has been requested func IsStopped(stop <-chan struct{}) bool { select { case <-stop: return true default: return false } } // ReadableSize returns a file size in a human readable form func ReadableSize(value int64) string { units := []string{"bytes", "KiB", "MiB", "GiB", "TiB"} v := float64(value) for _, u := range units { if v < 1024 || u == "TiB" { return fmt.Sprintf("%3.1f %s", v, u) } v /= 1024 } return "" } // ElapsedSec returns true if lastTimestamp + elapsed time is in the past func ElapsedSec(lastTimestamp int64, elapsedTime int64) bool { if lastTimestamp+elapsedTime < time.Now().UTC().Unix() { return true } return false } // Plural returns a single 's' if there are more than one value func Plural(value any) string { n, ok := value.(int) if ok && n > 1 || n < -1 { return "s" } return "" } // ConcatURL concatenate the url and path func ConcatURL(url, path string) string { if strings.HasSuffix(url, "/") && strings.HasPrefix(path, "/") { return url[:len(url)-1] + path } if !strings.HasSuffix(url, "/") && !strings.HasPrefix(path, "/") { return url + "/" + path } return url + path } // FormattedDateUTC returns the date formatted as RFC1123 func FormattedDateUTC(t time.Time) string { return t.UTC().Format(time.RFC1123) } // IsZero returns whether the date represents the zero time instant func IsZero(t time.Time) bool { return t.IsZero() } // TimeKeyCoverage returns a slice of strings covering the date range // used in the redis backend. func TimeKeyCoverage(start, end time.Time) (dates []string) { if start.Day() == end.Day() && start.Month() == end.Month() && start.Year() == end.Year() { dates = append(dates, start.Format("2006_01_02")) return } if start.Day() != 1 { month := start.Month() for { if start.Month() != month || start.Equal(end) { break } dates = append(dates, start.Format("2006_01_02")) start = start.AddDate(0, 0, 1) } } for { tmpyear := time.Date(start.Year()+1, 1, 1, 0, 0, 0, 0, start.Location()) tmpmonth := time.Date(start.Year(), start.Month()+1, 1, 0, 0, 0, 0, start.Location()) if start.Day() == 1 && start.Month() == 1 && (tmpyear.Before(end) || tmpyear.Equal(end)) { dates = append(dates, start.Format("2006")) start = tmpyear } else if tmpmonth.Before(end) || tmpmonth.Equal(end) { dates = append(dates, start.Format("2006_01")) start = tmpmonth } else { break } } for { if start.AddDate(0, 0, 1).After(end) { break } dates = append(dates, start.Format("2006_01_02")) start = start.AddDate(0, 0, 1) } return } // FuzzyTimeStr returns the duration as fuzzy time func FuzzyTimeStr(duration time.Duration) string { hours := duration.Hours() minutes := duration.Minutes() if int(minutes) == 0 { return "up-to-date" } if minutes < 0 { return "in the future" } if hours < 1 { return fmt.Sprintf("%d minute%s ago", int(duration.Minutes()), Plural(int(duration.Minutes()))) } if hours/24 < 1 { return fmt.Sprintf("%d hour%s ago", int(hours), Plural(int(hours))) } if hours/24/365 > 1 { return fmt.Sprintf("%d year%s ago", int(hours/24/365), Plural(int(hours/24/365))) } return fmt.Sprintf("%d day%s ago", int(hours/24), Plural(int(hours/24))) } // SanitizeLocationCodes sanitizes the given location codes func SanitizeLocationCodes(input string) string { input = strings.Replace(input, ",", " ", -1) ccodes := strings.Fields(input) output := "" for _, c := range ccodes { output += strings.ToUpper(c) + " " } return strings.TrimRight(output, " ") } videolabs-mirrorbits-441567e/utils/utils_test.go000066400000000000000000000131611523530551300220270ustar00rootroot00000000000000// Copyright (c) 2014-2019 Ludovic Fauvet // Licensed under the MIT license package utils import ( "testing" "time" "github.com/etix/mirrorbits/core" ) func TestHasAnyPrefix(t *testing.T) { var b bool b = HasAnyPrefix("http://test.com", "http://", "https://") if !b { t.Fatal("Expected true, got false") } b = HasAnyPrefix("https://test.com", "http://", "https://") if !b { t.Fatal("Expected true, got false") } b = HasAnyPrefix("test.com", "http://", "https://") if b { t.Fatal("Expected false, got true") } } func TestNormalizeURL(t *testing.T) { s := []string{ "", "", "rsync://test.com", "rsync://test.com/", "rsync://test.com/", "rsync://test.com/", } if len(s)%2 != 0 { t.Fatal("not multiple of 2") } for i := 0; i < len(s); i += 2 { if r := NormalizeURL(s[i]); r != s[i+1] { t.Fatalf("%q: expected %q, got %q", s[i], s[i+1], r) } } } func TestGetDistanceKm(t *testing.T) { if r := GetDistanceKm(48.8567, 2.3508, 40.7127, 74.0059); int(r) != 5514 { t.Fatalf("Expected 5514, got %f", r) } if r := GetDistanceKm(48.8567, 2.3508, 48.8567, 2.3508); int(r) != 0 { t.Fatalf("Expected 0, got %f", r) } } func TestMin(t *testing.T) { if r := Min(-10, 5); r != -10 { t.Fatalf("Expected -10, got %d", r) } } func TestMax(t *testing.T) { if r := Max(-10, 5); r != 5 { t.Fatalf("Expected 5, got %d", r) } } func TestAdd(t *testing.T) { if r := Add(2, 40); r != 42 { t.Fatalf("Expected 42, got %d", r) } } func TestVersion(t *testing.T) { if r := Version(); len(r) == 0 || r != core.VERSION { t.Fatalf("Expected %s, got %s", core.VERSION, r) } } func TestHostname(t *testing.T) { if r := Hostname(); len(r) == 0 { t.Fatalf("Expected a valid hostname") } } func TestIsInSlice(t *testing.T) { var b bool list := []string{"aaa", "bbb", "ccc"} b = IsInSlice("ccc", list) if !b { t.Fatal("Expected true, got false") } b = IsInSlice("b", list) if b { t.Fatal("Expected false, got true") } b = IsInSlice("", list) if b { t.Fatal("Expected false, got true") } } func TestIsStopped(t *testing.T) { stop := make(chan struct{}, 1) if IsStopped(stop) { t.Fatal("Expected false, got true") } close(stop) if !IsStopped(stop) { t.Fatal("Expected true, got false") } } func TestReadableSize(t *testing.T) { ivalues := []int64{0, 1, 1024, 1000000} svalues := []string{"0.0 bytes", "1.0 bytes", "1.0 KiB", "976.6 KiB"} for i := range ivalues { if r := ReadableSize(ivalues[i]); r != svalues[i] { t.Fatalf("Expected %q, got %q", svalues[i], r) } } } func TestElapsedSec(t *testing.T) { now := time.Now().UTC().Unix() lastTimestamp := now - 1000 if ElapsedSec(lastTimestamp, 500) == false { t.Fatalf("Expected true, got false") } if ElapsedSec(lastTimestamp, 5000) == true { t.Fatalf("Expected false, got true") } } func TestPlural(t *testing.T) { if Plural(2) != "s" { t.Fatalf("Expected 's', got ''") } if Plural(10000000) != "s" { t.Fatalf("Expected 's', got ''") } if Plural(-2) != "s" { t.Fatalf("Expected 's', got ''") } if Plural(1) != "" { t.Fatalf("Expected '', got 's'") } if Plural(-1) != "" { t.Fatalf("Expected '', got 's'") } if Plural(0) != "" { t.Fatalf("Expected '', got 's'") } } func TestConcatURL(t *testing.T) { part1 := "http://test.example/somedir/" part2 := "/somefile.bin" result := "http://test.example/somedir/somefile.bin" if r := ConcatURL(part1, part2); r != result { t.Fatalf("Expected %s, got %s", result, r) } part1 = "http://test.example/somedir" part2 = "/somefile.bin" result = "http://test.example/somedir/somefile.bin" if r := ConcatURL(part1, part2); r != result { t.Fatalf("Expected %s, got %s", result, r) } part1 = "http://test.example/somedir" part2 = "somefile.bin" result = "http://test.example/somedir/somefile.bin" if r := ConcatURL(part1, part2); r != result { t.Fatalf("Expected %s, got %s", result, r) } } func TestTimeKeyCoverage(t *testing.T) { date1Start := time.Date(2015, 10, 30, 12, 42, 11, 0, time.UTC) date1End := time.Date(2015, 12, 2, 13, 42, 11, 0, time.UTC) result1 := []string{"2015_10_30", "2015_10_31", "2015_11", "2015_12_01"} result := TimeKeyCoverage(date1Start, date1End) if len(result) != len(result1) { t.Fatalf("Expect %d elements, got %d", len(result1), len(result)) } for i, r := range result { if r != result1[i] { t.Fatalf("Expect %#v, got %#v", result1, result) } } /* */ date2Start := time.Date(2015, 12, 2, 12, 42, 11, 0, time.UTC) date2End := time.Date(2015, 12, 2, 13, 42, 11, 0, time.UTC) result2 := []string{"2015_12_02"} result = TimeKeyCoverage(date2Start, date2End) if len(result) != len(result2) { t.Fatalf("Expect %d elements, got %d", len(result2), len(result)) } for i, r := range result { if r != result2[i] { t.Fatalf("Expect %#v, got %#v", result2, result) } } /* */ date3Start := time.Date(2015, 1, 1, 12, 42, 11, 0, time.UTC) date3End := time.Date(2017, 1, 1, 13, 42, 11, 0, time.UTC) result3 := []string{"2015", "2016"} result = TimeKeyCoverage(date3Start, date3End) if len(result) != len(result3) { t.Fatalf("Expect %d elements, got %d", len(result3), len(result)) } for i, r := range result { if r != result3[i] { t.Fatalf("Expect %#v, got %#v", result3, result) } } /* */ date4Start := time.Date(2015, 12, 31, 12, 42, 11, 0, time.UTC) date4End := time.Date(2016, 1, 2, 13, 42, 11, 0, time.UTC) result4 := []string{"2015_12_31", "2016_01_01"} result = TimeKeyCoverage(date4Start, date4End) if len(result) != len(result4) { t.Fatalf("Expect %d elements, got %d", len(result4), len(result)) } for i, r := range result { if r != result4[i] { t.Fatalf("Expect %#v, got %#v", result4, result) } } }