pax_global_header00006660000000000000000000000064146226672070014526gustar00rootroot0000000000000052 comment=8b465fc5db8fe46d64f3d9313686efd684ebd58a go-downloader-2.2.0/000077500000000000000000000000001462266720700142705ustar00rootroot00000000000000go-downloader-2.2.0/.travis.yml000066400000000000000000000003211462266720700163750ustar00rootroot00000000000000language: go go: - 1.11.x - tip before_install: - go get -t -v ./... script: - go test -race -coverprofile=coverage.txt -covermode=atomic after_success: - bash <(curl -s https://codecov.io/bash) go-downloader-2.2.0/LICENSE000066400000000000000000000027431462266720700153030ustar00rootroot00000000000000 Copyright (c) 2018, Cristian Maglie. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. go-downloader-2.2.0/README.md000066400000000000000000000005061462266720700155500ustar00rootroot00000000000000# go.bug.st/downloader/v2 [![build status](https://api.travis-ci.org/bugst/go-downloader.svg?branch=master)](https://travis-ci.org/bugst/go-downloader) [![codecov](https://codecov.io/gh/bugst/go-downloader/branch/master/graph/badge.svg)](https://codecov.io/gh/bugst/go-downloader) A simple HTTP/S file downloader for golang. go-downloader-2.2.0/downloader.go000066400000000000000000000113321462266720700167550ustar00rootroot00000000000000// // Copyright 2018 Cristian Maglie. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // package downloader import ( "context" "fmt" "io" "net/http" "os" "sync" "time" ) // Downloader is an asynchronous downloader type Downloader struct { URL string Done chan bool Resp *http.Response out *os.File completed int64 completedLock sync.Mutex size int64 err error } // DownloadOptions are optional flags that can be passed to Download function type DownloadOptions int const ( // NoResume will not try to resume a partial download NoResume DownloadOptions = iota ) // Close the download func (d *Downloader) Close() error { err1 := d.out.Close() err2 := d.Resp.Body.Close() if err1 != nil { return fmt.Errorf("closing output file: %s", err1) } if err2 != nil { return fmt.Errorf("closing input stream: %s", err2) } return nil } // Size return the size of the download func (d *Downloader) Size() int64 { return d.size } // RunAndPoll starts the downloader copy-loop and calls the poll function every // interval time to update progress. func (d *Downloader) RunAndPoll(poll func(current int64), interval time.Duration) error { t := time.NewTicker(interval) defer t.Stop() go d.AsyncRun() for { select { case <-t.C: poll(d.Completed()) case <-d.Done: poll(d.Completed()) return d.Error() } } } // AsyncRun starts the downloader copy-loop. This function is supposed to be run // on his own go routine because it sends a confirmation on the Done channel func (d *Downloader) AsyncRun() { in := d.Resp.Body buff := [4096]byte{} for { n, err := in.Read(buff[:]) if n > 0 { _, _ = d.out.Write(buff[:n]) d.completedLock.Lock() d.completed += int64(n) d.completedLock.Unlock() } if err == io.EOF { break } if err != nil { d.err = err break } } _ = d.Close() d.Done <- true } // Run starts the downloader and waits until it completes the download. func (d *Downloader) Run() error { go d.AsyncRun() <-d.Done return d.Error() } // Error returns the error during download or nil if no errors happened func (d *Downloader) Error() error { return d.err } // Completed returns the bytes read so far func (d *Downloader) Completed() int64 { d.completedLock.Lock() res := d.completed d.completedLock.Unlock() return res } // Download returns an asynchronous downloader that will download the specified url // in the specified file. A download resume is tried if a file shorter than the requested // url is already present. func Download(file string, reqURL string, options ...DownloadOptions) (*Downloader, error) { return DownloadWithConfig(file, reqURL, GetDefaultConfig(), options...) } // DownloadWithConfig applies an additional configuration to the http client and // returns an asynchronous downloader that will download the specified url // in the specified file. A download resume is tried if a file shorter than the requested // url is already present. func DownloadWithConfig(file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) { return DownloadWithConfigAndContext(context.Background(), file, reqURL, config, options...) } // DownloadWithConfigAndContext applies an additional configuration to the http client and // returns an asynchronous downloader that will download the specified url // in the specified file. A download resume is tried if a file shorter than the requested // url is already present. The download can be cancelled using the provided context. func DownloadWithConfigAndContext(ctx context.Context, file string, reqURL string, config Config, options ...DownloadOptions) (*Downloader, error) { noResume := false for _, opt := range options { if opt == NoResume { noResume = true } } req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { return nil, fmt.Errorf("setting up HTTP request: %s", err) } var completed int64 if !noResume { if info, err := os.Stat(file); err == nil { completed = info.Size() req.Header.Set("Range", fmt.Sprintf("bytes=%d-", completed)) } } resp, err := config.HttpClient.Do(req) if err != nil { return nil, err } // TODO: if file size == header size return nil, nil flags := os.O_WRONLY if completed == 0 { flags |= os.O_CREATE | os.O_TRUNC } else { flags |= os.O_APPEND } f, err := os.OpenFile(file, flags, 0644) if err != nil { _ = resp.Body.Close() return nil, fmt.Errorf("opening %s for writing: %s", file, err) } d := &Downloader{ URL: reqURL, Done: make(chan bool), Resp: resp, out: f, completed: completed, size: resp.ContentLength + completed, } return d, nil } go-downloader-2.2.0/downloader_config.go000066400000000000000000000016041462266720700203030ustar00rootroot00000000000000// // Copyright 2018 Cristian Maglie. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // package downloader import ( "net/http" "sync" ) // Config contains the configuration for the downloader type Config struct { HttpClient http.Client } var defaultConfig Config = Config{} var defaultConfigLock sync.Mutex // SetDefaultConfig sets the configuration that will be used by the Download // function. func SetDefaultConfig(newConfig Config) { defaultConfigLock.Lock() defer defaultConfigLock.Unlock() defaultConfig = newConfig } // GetDefaultConfig returns a copy of the default configuration. The default // configuration can be changed using the SetDefaultConfig function. func GetDefaultConfig() Config { defaultConfigLock.Lock() defer defaultConfigLock.Unlock() // deep copy struct return defaultConfig } go-downloader-2.2.0/downloader_test.go000066400000000000000000000130041462266720700200120ustar00rootroot00000000000000// // Copyright 2018 Cristian Maglie. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // package downloader import ( "context" "encoding/json" "fmt" "io" "net/http" "os" "testing" "time" "github.com/stretchr/testify/require" ) func makeTmpFile(t *testing.T) string { tmp, err := os.CreateTemp("", "") require.NoError(t, err) require.NoError(t, tmp.Close()) tmpFile := tmp.Name() require.NoError(t, os.Remove(tmpFile)) return tmpFile } func TestDownload(t *testing.T) { tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) d, err := Download(tmpFile, "https://go.bug.st/test.txt") require.NoError(t, err) require.Equal(t, int64(0), d.Completed()) require.Equal(t, int64(8052), d.Size()) require.NoError(t, d.Run()) require.Equal(t, int64(8052), d.Completed()) require.Equal(t, int64(8052), d.Size()) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) file2, err := os.ReadFile(tmpFile) require.NoError(t, err) require.Equal(t, file1, file2) } func TestResume(t *testing.T) { tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) part, err := os.ReadFile("testdata/test.txt.part") require.NoError(t, err) err = os.WriteFile(tmpFile, part, 0644) require.NoError(t, err) d, err := Download(tmpFile, "https://go.bug.st/test.txt") require.Equal(t, int64(3506), d.Completed()) require.Equal(t, int64(8052), d.Size()) require.NoError(t, err) require.NoError(t, d.Run()) require.Equal(t, int64(8052), d.Completed()) require.Equal(t, int64(8052), d.Size()) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) file2, err := os.ReadFile(tmpFile) require.NoError(t, err) require.Equal(t, file1, file2) } func TestNoResume(t *testing.T) { tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) part, err := os.ReadFile("testdata/test.txt.part") require.NoError(t, err) err = os.WriteFile(tmpFile, part, 0644) require.NoError(t, err) d, err := Download(tmpFile, "https://go.bug.st/test.txt", NoResume) require.Equal(t, int64(0), d.Completed()) require.Equal(t, int64(8052), d.Size()) require.NoError(t, err) require.NoError(t, d.Run()) require.Equal(t, int64(8052), d.Completed()) require.Equal(t, int64(8052), d.Size()) file1, err := os.ReadFile("testdata/test.txt") require.NoError(t, err) file2, err := os.ReadFile(tmpFile) require.NoError(t, err) require.Equal(t, file1, file2) } func TestInvalidRequest(t *testing.T) { tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) d, err := Download(tmpFile, "asd://go.bug.st/test.txt") require.Error(t, err) require.Nil(t, d) fmt.Println("ERROR:", err) d, err = Download(tmpFile, "://") require.Error(t, err) require.Nil(t, d) fmt.Println("ERROR:", err) } func TestRunAndPool(t *testing.T) { tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) d, err := Download(tmpFile, "https://downloads.arduino.cc/cores/avr-1.6.20.tar.bz2") require.NoError(t, err) prevCurr := int64(0) callCount := 0 callback := func(curr int64) { require.True(t, prevCurr <= curr) prevCurr = curr callCount++ } _ = d.RunAndPoll(callback, time.Millisecond) fmt.Printf("callback called %d times\n", callCount) require.True(t, callCount > 10) require.Equal(t, int64(4897949), d.Completed()) } func TestErrorOnFileOpening(t *testing.T) { tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) require.NoError(t, os.WriteFile(tmpFile, []byte{}, 0000)) d, err := Download(tmpFile, "http://go.bug.st/test.txt") require.Error(t, err) require.Nil(t, d) } type roundTripper struct { UserAgent string transport http.Transport } func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { req.Header["User-Agent"] = []string{r.UserAgent} return r.transport.RoundTrip(req) } // TestApplyUserAgentHeaderUsingConfig test uses the https://postman-echo.com/ service func TestApplyUserAgentHeaderUsingConfig(t *testing.T) { type echoBody struct { Headers map[string]string } tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) httpClient := http.Client{ Transport: &roundTripper{UserAgent: "go-downloader / 0.0.0-test"}, } config := Config{ HttpClient: httpClient, } d, err := DownloadWithConfig(tmpFile, "https://postman-echo.com/headers", config) require.NoError(t, err) testEchoBody := echoBody{} body, err := io.ReadAll(d.Resp.Body) require.NoError(t, err) err = json.Unmarshal(body, &testEchoBody) require.NoError(t, err) require.Equal(t, "go-downloader / 0.0.0-test", testEchoBody.Headers["user-agent"]) } func TestContextCancelation(t *testing.T) { slowHandler := func(w http.ResponseWriter, r *http.Request) { for i := 0; i < 50; i++ { fmt.Fprintf(w, "Hello %d\n", i) w.(http.Flusher).Flush() time.Sleep(100 * time.Millisecond) } } mux := http.NewServeMux() mux.HandleFunc("/slow", slowHandler) server := &http.Server{Addr: ":8080", Handler: mux} go func() { server.ListenAndServe() fmt.Println("Server stopped") }() // Wait for server start time.Sleep(time.Second) tmpFile := makeTmpFile(t) defer os.Remove(tmpFile) ctx, cancel := context.WithCancel(context.Background()) d, err := DownloadWithConfigAndContext(ctx, tmpFile, "http://127.0.0.1:8080/slow", Config{}) require.NoError(t, err) // Cancel in two seconds go func() { time.Sleep(2 * time.Second) cancel() }() // Run slow download max := int64(0) err = d.RunAndPoll(func(curr int64) { fmt.Println(curr) max = curr }, 100*time.Millisecond) require.EqualError(t, err, "context canceled") require.True(t, max < 400) require.NoError(t, server.Shutdown(ctx)) } go-downloader-2.2.0/go.mod000066400000000000000000000001241462266720700153730ustar00rootroot00000000000000module go.bug.st/downloader/v2 go 1.12 require github.com/stretchr/testify v1.3.0 go-downloader-2.2.0/go.sum000066400000000000000000000011401462266720700154170ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= go-downloader-2.2.0/testdata/000077500000000000000000000000001462266720700161015ustar00rootroot00000000000000go-downloader-2.2.0/testdata/test.txt000066400000000000000000000175641462266720700176360ustar00rootroot00000000000000Hey! You've found the test file! `...` ` ```..-:-.`````` `.`` ````.....-:/:......` `-::-` .-.....-:::///-.....`` `---:--` .://:///:::/++///:--.` `---.---. `..-/o+ooo+///+oo++/:-``` --...---.` `....--/osoosoo++oooy+:-.--` .......---. `:+:-::/+s+//:-.:/oss+/:--. `-..-....--.` ```` .//:///+o+:-...../oo+/:-:` `-..--:------.` `..----.` .--///++/-.::--../oo///- `.----:-:::////::.` `.......---. ./+o++++:-++//::++///:. `-::/:///::-/o/////:---..`` ``.......-----. .yhysss+/+//:-:/+o+/-. `-:::---/:.``::///////o++//::-.```............-.---:. /dddddso+///--:+ss+-` `.-:+oss+-.```..---...`.-++///++//::--............-----` /yddddhys+//:-://+//- .:/::/hNMMMm: .--..`` -///:://+++/:::----.--......---.` `:+syyssoo+//:----:--` `:osy:`:dMmNoy-..-.`` `.....-////////:::---............` `/+ooso+os///:..---.` .++syys/--/os+:://-. `-+yyyo:--:////////::---------...`.-:-.` .:oyhhyyys+/+/://+: .ohhs+/-:--o+/://oo+/:/:/shNNNdmh+---:://////::::---..--.``:ooo+:.` ./oyhy/:os//:/++//- .+yys+-``` `yo/+yss///+yo:ydmNd+hhh:-----:////:///::--..```-+osoo+:--` .:+syho//+//::////. `+hhys:.`````/s+:::-``.:o/..:ossss+-`.::---:////:::::-....:+sssooo+++/-.` .:///oo/:/-::///+:``:yhhhy+-----:os:.` `.::/-.-.........-++:-::::::///-...-:+syssooo+o++/:.` .+/+///::://+++//-``+hdddho//+o+oos/-.` `....---:/+++oooo+++ss+////::::-..--:/oyyysssoo++oo+:-` :++osso+/+o////::::..ohhhhyo++-..-/+/-......``..-/ssssso++++osyo++o+/-...:/+++sssssssooooooo+:. `+sssyys++/:--:-::+s:.:sdhyyo/--....-:/:---..-..`.-+sso+/::---/syso+++/:..:/+/+ssssssssoooo++/:. -/ossoo+++::-:-:/oyhs+ymmddhs+::--....-//-:-:-..```-/osyys+:--:+sysooo+/:-.:/+ossssossssoo+++/:` :/+++/////:---:++shdhdNMNmdhyo+/::------////o+:....-:ososyyo/:-:/oysoooo+:--:+sssssosssooo++++/-` :+oo+/:--::/::///shddNMMMNdhhyso+/:::::------::-----:/::/+sso+::/osso+//++::+osssssssooo++o+++:.` `:+ooo++++++/-../hmmmNNNMMMmdhhysso++/://:::--:-:://::////+osyo+::+syso/::/:/+syyssssoooo+++++/:` `/+ooooo+/:-/osshdddNNNNNMMMmhhyyyyso++++oo++oo+o+ooso++o++osyyso+oosys+/://+syysssoooooo++++//:` `/+++++osyyyddddhhyhmNNNMMMMNmdhhmdhyssyssssoossooooossoossooossooosssso///+osssssssoooo++++++/-` /yhyhdddmmmmmddhsohNNNMMMMMNNmmdmddhhyssssso+osossoooosossssso++++ossssso+oosyyyssoo+++++++++/. :shdddddhddhhyyo+sdNNNMNNNNNNmmmmmmhhddhyhhoo+sssss++//+osssoo++++oosyysoooosyyssoo++o++o++++-` ./osyyyyyysssoo++hNNNNMNNNNNNmmdmmmdmmmddddhysyho/oo:/+osyyyhhhyysyysyyysossyssssoo+++//+++/:. `-/+ooo+oo++o+++sdNmNNNNmmmmmmddmNNmdddddmmNmNNNNmmmdhdhmmdmmddddhyyysssssyyyyssoo++//+++//:-` `-/+++oo//+++++ohmmNmmmmdddddmmNNmdhhhhhddhddmmNmNNmmdhddhhhyyyyysssosssyyyyssoo+++/+++++:.` `.:++o++o+//+oohmmmddddddddmddNmdhyyyyhhhyshhdhhdddhyyyyhyyyysssssssssyyyyyyssso+++++//:.` `-/+ooo++ssssyddhhyyhhhhddhddhyysss+oosssyyyhhsssossssosssssssssyhhhhhhyyyssoo+++oo+/-` `.:+sooossysyhhyhhyyyhhhhyyysso//:----::+oyyys++++ossooooossssyhhhhhhyyssssoooo+++/-` `:oshysyyyyyyyhhhyhhyhyysoo///::--.....--+syys+///+ooo+oossyyhhhhhhyyssssoooooo+/:.` `:shddhhhhyyyyyyyyssso+++++osso++/:-:-...-:+sso/://++osooyhhhhhhhyyssssooo++/////-. `-ohdmdddyysoo+oooo/:::---:/oosssoo+///:---:oso/::+++oyyyhhddhhhhysoossso++++/:::-` `./yhyyyso++////+o++++++/////++ossooso+++/--/sso//+oosyhddddhhhyyysooooo//::/-..--. ``-:++++++ooo+++++++///osso//+o+/+oss+oo++oo:-+so+/oyyhdddmddhhysssooo+++::-../...-. ``.--:::::///++++++/+//::+osso/+oo+/+oso////+++::ooooyhddmmmmddhyso+oooo+/----:.//-...` ``.--:::///:::///////////:::/+ys/:+s+/+oss/--:/oo//osyyhmmmmdddhhyso++++ssso+:/+/:++:..-` ```.-----::/+so/::://://::::::::+s+-:++//oos+-.-:++//oyyyhyhhhyysoo+//::----::::::-:-:-..` ````...---..:/:++:....::/:-...---:+o//://:-:///:-..//::////////:::---....`.```.`````````` `.-...-::+:..--.-::--...-.---.``.-/oo:-----......``````````..`````` ` `---/-:/.`.-.`-/:-/.`..``..:--.-//:.````` .:/::oo-`..:..-+/+:```````.::-.````` ``..-/:...--.`--::.```````..``` ``````` ``````` MEAOOOWWWWWWWW go-downloader-2.2.0/testdata/test.txt.part000066400000000000000000000066621462266720700206000ustar00rootroot00000000000000Hey! You've found the test file! `...` ` ```..-:-.`````` `.`` ````.....-:/:......` `-::-` .-.....-:::///-.....`` `---:--` .://:///:::/++///:--.` `---.---. `..-/o+ooo+///+oo++/:-``` --...---.` `....--/osoosoo++oooy+:-.--` .......---. `:+:-::/+s+//:-.:/oss+/:--. `-..-....--.` ```` .//:///+o+:-...../oo+/:-:` `-..--:------.` `..----.` .--///++/-.::--../oo///- `.----:-:::////::.` `.......---. ./+o++++:-++//::++///:. `-::/:///::-/o/////:---..`` ``.......-----. .yhysss+/+//:-:/+o+/-. `-:::---/:.``::///////o++//::-.```............-.---:. /dddddso+///--:+ss+-` `.-:+oss+-.```..---...`.-++///++//::--............-----` /yddddhys+//:-://+//- .:/::/hNMMMm: .--..`` -///:://+++/:::----.--......---.` `:+syyssoo+//:----:--` `:osy:`:dMmNoy-..-.`` `.....-////////:::---............` `/+ooso+os///:..---.` .++syys/--/os+:://-. `-+yyyo:--:////////::---------...`.-:-.` .:oyhhyyys+/+/://+: .ohhs+/-:--o+/://oo+/:/:/shNNNdmh+---:://////::::---..--.``:ooo+:.` ./oyhy/:os//:/++//- .+yys+-``` `yo/+yss///+yo:ydmNd+hhh:-----:////:///::--..```-+osoo+:--` .:+syho//+//::////. `+hhys:.`````/s+:::-``.:o/..:ossss+-`.::---:////:::::-....:+sssooo+++/-.` .:///oo/:/-::///+:``:yhhhy+-----:os:.` `.::/-.-.........-++:-::::::///-...-:+syssooo+o++/:.` .+/+///::://+++//-``+hdddho//+o+oos/-.` `....---:/+++oooo+++ss+////::::-..--:/oyyysssoo++oo+:-` :++osso+/+o////::::..ohhhhyo++-..-/+/-......``..-/ssssso++++osyo++o+/-...:/+++sssssssooooooo+:.