Verified Commit 1fbeb6c0 authored by Vladimir Hodakov's avatar Vladimir Hodakov 🔥
Browse files

Add actual file upload to Yandex with progress bar

parent ff258dde
......@@ -2,7 +2,7 @@
## English
This app is useful when you want to upload single file to Yandex.Disk but don't want to fiddle with sync official client or WebDAV. It is especially useful for automating backups (e. g. in conjuction with Proxmox's ``vzdump``.)
This app is useful when you want to upload single file to Yandex.Disk but don't want to fiddle with sync official client or WebDAV. It is especially useful for automating backups (e. g. in conjuction with Proxmox's ``vzdump``).
## Russian
......
......@@ -12,6 +12,7 @@ import (
const YANDEX_APPID = "7d8a0561fdc44c05bb6695b464403f9c"
const YANDEX_APPPW = "56e12e4ed0d64738bf441a47f68c7146"
const DEVICE_NAME = "yapusher-cli"
const MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024 // 10 gigabytes
var (
c *context.Context
......@@ -34,16 +35,23 @@ func New(cc *context.Context) {
Name: "uploadPath",
Description: "Path to upload your file on Yandex.Disk. Must exist before uploading.",
Type: "string",
DefaultValue: "/",
DefaultValue: "",
})
_ = c.Flagger.AddFlag(&flagger.Flag{
Name: "file",
Description: "Path to file that will be uploaded. Max upload size - 50 GB",
Description: "Path to file that will be uploaded. Max upload size - 10 GB",
Type: "string",
DefaultValue: "",
})
_ = c.Flagger.AddFlag(&flagger.Flag{
Name: "force",
Description: "Force file to be uploaded even if destination file on Yandex.Disk already exists.",
Type: "bool",
DefaultValue: false,
})
dlog.Info().Msg("Domain initialized")
}
......@@ -56,7 +64,9 @@ func Process() {
filePath, _ := c.Flagger.GetStringValue("file")
if filePath != "" {
uploadFile()
uploadPath, _ := c.Flagger.GetStringValue("uploadPath")
forceUpload, _ := c.Flagger.GetBoolValue("force")
uploadFile(uploadPath, filePath, forceUpload)
}
if !checkAuth() {
......
......@@ -5,13 +5,18 @@ package yandexv1
import (
"encoding/json"
"github.com/schollz/progressbar/v2"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
)
func reportFileProgress(r int64) {
}
func sendCode(code int) {
baseURL := "https://oauth.yandex.ru/token"
......@@ -51,11 +56,108 @@ func sendCode(code int) {
dlog.Error().Err(err).Msg("Failed to decode response")
}
dlog.Error().Interface("response", errorData).Msg("Got error from Yandex, not authorized. Please retry authorization")
dlog.Error().Str("error", errorData.Error).Str("description", errorData.ErrorDescription).Msg("Got error from Yandex, not authorized. Please retry authorization")
authorize()
}
os.Exit(0)
}
func uploadFile() {}
func uploadFile(uploadPath string, filePath string, overwriteFile bool) {
uploadRequestURL := "https://cloud-api.yandex.net/v1/disk/resources/upload"
// Checking file existence before requesting
normalizedFilePath, _ := filepath.Abs(filePath)
fileInfo, err := os.Stat(normalizedFilePath)
if err != nil {
if os.IsNotExist(err) {
dlog.Fatal().Err(err).Msg("File for uploading not found")
} else {
dlog.Fatal().Err(err).Msg("Failed to stat uploading file")
}
}
if fileInfo.Size() > (MAX_FILE_SIZE - 1) {
dlog.Fatal().Msg("Requested file is too big")
}
if !fileInfo.Mode().IsRegular() {
dlog.Fatal().Msg("Only regular files uploading is supported right now")
}
client := http.Client{}
uploadInfo := UploadInfo{}
// The first request will get from Yandex upload URL
req, _ := http.NewRequest("GET", uploadRequestURL, nil)
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", "OAuth "+c.Config.Token.AccessToken)
query := url.Values{}
query.Add("path", "disk:/"+uploadPath+"/"+fileInfo.Name())
query.Add("overwrite", strconv.FormatBool(overwriteFile))
req.URL.RawQuery = query.Encode()
resp, err := client.Do(req)
if err != nil {
dlog.Fatal().Err(err).Msg("Failed to send request")
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
err = json.NewDecoder(resp.Body).Decode(&uploadInfo)
if err != nil {
dlog.Error().Err(err).Msg("Failed to decode response")
}
} else {
errorData := UploadError{}
err = json.NewDecoder(resp.Body).Decode(&errorData)
if err != nil {
dlog.Error().Err(err).Msg("Failed to decode response")
}
dlog.Info().Str("error", errorData.Error).Str("description", errorData.Description).Msg("Failed to upload file")
os.Exit(1)
}
file, _ := os.Open(normalizedFilePath)
bar := progressbar.NewOptions(
int(fileInfo.Size()),
progressbar.OptionSetBytes(int(fileInfo.Size())),
progressbar.OptionSetRenderBlankState(true),
)
progressReader := &progressReader{
r: file,
progressbar: bar,
}
if uploadInfo.URL == "" {
dlog.Fatal().Msg("Got empty upload URL. Report a bug at https://github.com/fat0troll/yapusher/issues because this situation is impossible.")
}
uploadReq, _ := http.NewRequest("PUT", uploadInfo.URL, progressReader)
uploadResp, err := client.Do(uploadReq)
if err != nil {
dlog.Fatal().Err(err).Msg("Failed to send upload request")
}
defer uploadResp.Body.Close()
switch uploadResp.StatusCode {
case http.StatusCreated:
dlog.Info().Msg("File uploaded successfully")
case http.StatusAccepted:
dlog.Info().Msg("File uploaded successfully, but it will take time for Yandex.Disk to handle it internally. Be patient and don't try to upload single file many times")
case http.StatusRequestEntityTooLarge:
dlog.Fatal().Msg("File upload is too large. Report a bug at https://github.com/fat0troll/yapusher/issues because this situation should be handled before upload attempt.")
case http.StatusInsufficientStorage:
dlog.Fatal().Msg("There is no space left on your Yandex.Disk.")
default:
dlog.Fatal().Msg("Failed to upload file (error on Yandex's side). Try again later.")
}
os.Exit(0)
}
......@@ -3,7 +3,46 @@
package yandexv1
import (
"fmt"
"github.com/schollz/progressbar/v2"
"io"
)
type progressReader struct {
r io.Reader
atEOF bool
progressbar *progressbar.ProgressBar
}
func (pr *progressReader) Read(p []byte) (int, error) {
n, err := pr.r.Read(p)
if err == io.EOF {
pr.atEOF = true
}
pr.report(int64(n))
return n, err
}
func (pr *progressReader) report(n int64) {
_ = pr.progressbar.Add64(n)
if pr.atEOF {
fmt.Print("\n\n")
}
}
type TokenError struct {
Error string `json:"error"`
ErrorDescription string `json:"error_description"`
}
type UploadError struct {
Error string `json:"error"`
Description string `json:"description"`
}
type UploadInfo struct {
URL string `json:"href"`
Method string `json:"method"`
URLIsTemplated bool `json:"templated"`
}
......@@ -5,7 +5,11 @@ go 1.12
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/rs/zerolog v1.13.0
github.com/schollz/progressbar v1.0.0 // indirect
github.com/schollz/progressbar/v2 v2.10.0
github.com/stretchr/objx v0.1.1 // indirect
github.com/stretchr/testify v1.3.0 // indirect
gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9
gitlab.com/pztrn/go-uuid v0.0.0-20190208164458-d6cc46783d2b
......
......@@ -3,11 +3,19 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f h1:dKccXx7xA56UNqOcFIbuqFjAWPVtP688j5QMgmo6OHU=
github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f/go.mod h1:4rEELDSfUAlBSyUjPG0JnaNGjf13JySHFeRdD/3dLP0=
github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM=
github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs=
github.com/schollz/progressbar/v2 v2.10.0 h1:AxYYUjr5fOPlA0Pcqc3R3kDBrqyLFWk2P7LRLdXb3yE=
github.com/schollz/progressbar/v2 v2.10.0/go.mod h1:l6tn6yU6ZdQoF8lwX/VoAUQ3FjhCbrcZDnl9xeWZzYw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
gitlab.com/pztrn/flagger v0.0.0-20190122123836-d429d7149cc9 h1:qLlIZybSEr3MSCaE+cJZpT6O/1q3IsBPV7GcZJc05vs=
......
language: go
go:
- 1.0
- 1.1
- 1.2
- 1.3
- tip
script:
- go test
matrix:
allow_failures:
- go: tip
The MIT License (MIT)
Copyright (c) 2014 Mitchell Hashimoto
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
# colorstring [![Build Status](https://travis-ci.org/mitchellh/colorstring.svg)](https://travis-ci.org/mitchellh/colorstring)
colorstring is a [Go](http://www.golang.org) library for outputting colored
strings to a console using a simple inline syntax in your string to specify
the color to print as.
For example, the string `[blue]hello [red]world` would output the text
"hello world" in two colors. The API of colorstring allows for easily disabling
colors, adding aliases, etc.
## Installation
Standard `go get`:
```
$ go get github.com/mitchellh/colorstring
```
## Usage & Example
For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/colorstring).
Usage is easy enough:
```go
colorstring.Println("[blue]Hello [red]World!")
```
Additionally, the `Colorize` struct can be used to set options such as
custom colors, color disabling, etc.
// colorstring provides functions for colorizing strings for terminal
// output.
package colorstring
import (
"bytes"
"fmt"
"io"
"regexp"
"strings"
)
// Color colorizes your strings using the default settings.
//
// Strings given to Color should use the syntax `[color]` to specify the
// color for text following. For example: `[blue]Hello` will return "Hello"
// in blue. See DefaultColors for all the supported colors and attributes.
//
// If an unrecognized color is given, it is ignored and assumed to be part
// of the string. For example: `[hi]world` will result in "[hi]world".
//
// A color reset is appended to the end of every string. This will reset
// the color of following strings when you output this text to the same
// terminal session.
//
// If you want to customize any of this behavior, use the Colorize struct.
func Color(v string) string {
return def.Color(v)
}
// ColorPrefix returns the color sequence that prefixes the given text.
//
// This is useful when wrapping text if you want to inherit the color
// of the wrapped text. For example, "[green]foo" will return "[green]".
// If there is no color sequence, then this will return "".
func ColorPrefix(v string) string {
return def.ColorPrefix(v)
}
// Colorize colorizes your strings, giving you the ability to customize
// some of the colorization process.
//
// The options in Colorize can be set to customize colorization. If you're
// only interested in the defaults, just use the top Color function directly,
// which creates a default Colorize.
type Colorize struct {
// Colors maps a color string to the code for that color. The code
// is a string so that you can use more complex colors to set foreground,
// background, attributes, etc. For example, "boldblue" might be
// "1;34"
Colors map[string]string
// If true, color attributes will be ignored. This is useful if you're
// outputting to a location that doesn't support colors and you just
// want the strings returned.
Disable bool
// Reset, if true, will reset the color after each colorization by
// adding a reset code at the end.
Reset bool
}
// Color colorizes a string according to the settings setup in the struct.
//
// For more details on the syntax, see the top-level Color function.
func (c *Colorize) Color(v string) string {
matches := parseRe.FindAllStringIndex(v, -1)
if len(matches) == 0 {
return v
}
result := new(bytes.Buffer)
colored := false
m := []int{0, 0}
for _, nm := range matches {
// Write the text in between this match and the last
result.WriteString(v[m[1]:nm[0]])
m = nm
var replace string
if code, ok := c.Colors[v[m[0]+1:m[1]-1]]; ok {
colored = true
if !c.Disable {
replace = fmt.Sprintf("\033[%sm", code)
}
} else {
replace = v[m[0]:m[1]]
}
result.WriteString(replace)
}
result.WriteString(v[m[1]:])
if colored && c.Reset && !c.Disable {
// Write the clear byte at the end
result.WriteString("\033[0m")
}
return result.String()
}
// ColorPrefix returns the first color sequence that exists in this string.
//
// For example: "[green]foo" would return "[green]". If no color sequence
// exists, then "" is returned. This is especially useful when wrapping
// colored texts to inherit the color of the wrapped text.
func (c *Colorize) ColorPrefix(v string) string {
return prefixRe.FindString(strings.TrimSpace(v))
}
// DefaultColors are the default colors used when colorizing.
//
// If the color is surrounded in underscores, such as "_blue_", then that
// color will be used for the background color.
var DefaultColors map[string]string
func init() {
DefaultColors = map[string]string{
// Default foreground/background colors
"default": "39",
"_default_": "49",
// Foreground colors
"black": "30",
"red": "31",
"green": "32",
"yellow": "33",
"blue": "34",
"magenta": "35",
"cyan": "36",
"light_gray": "37",
"dark_gray": "90",
"light_red": "91",
"light_green": "92",
"light_yellow": "93",
"light_blue": "94",
"light_magenta": "95",
"light_cyan": "96",
"white": "97",
// Background colors
"_black_": "40",
"_red_": "41",
"_green_": "42",
"_yellow_": "43",
"_blue_": "44",
"_magenta_": "45",
"_cyan_": "46",
"_light_gray_": "47",
"_dark_gray_": "100",
"_light_red_": "101",
"_light_green_": "102",
"_light_yellow_": "103",
"_light_blue_": "104",
"_light_magenta_": "105",
"_light_cyan_": "106",
"_white_": "107",
// Attributes
"bold": "1",
"dim": "2",
"underline": "4",
"blink_slow": "5",
"blink_fast": "6",
"invert": "7",
"hidden": "8",
// Reset to reset everything to their defaults
"reset": "0",
"reset_bold": "21",
}
def = Colorize{
Colors: DefaultColors,
Reset: true,
}
}
var def Colorize
var parseReRaw = `\[[a-z0-9_-]+\]`
var parseRe = regexp.MustCompile(`(?i)` + parseReRaw)
var prefixRe = regexp.MustCompile(`^(?i)(` + parseReRaw + `)+`)
// Print is a convenience wrapper for fmt.Print with support for color codes.
//
// Print formats using the default formats for its operands and writes to
// standard output with support for color codes. Spaces are added between
// operands when neither is a string. It returns the number of bytes written
// and any write error encountered.
func Print(a string) (n int, err error) {
return fmt.Print(Color(a))
}
// Println is a convenience wrapper for fmt.Println with support for color
// codes.
//
// Println formats using the default formats for its operands and writes to
// standard output with support for color codes. Spaces are always added
// between operands and a newline is appended. It returns the number of bytes
// written and any write error encountered.
func Println(a string) (n int, err error) {
return fmt.Println(Color(a))
}
// Printf is a convenience wrapper for fmt.Printf with support for color codes.
//
// Printf formats according to a format specifier and writes to standard output
// with support for color codes. It returns the number of bytes written and any
// write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Printf(Color(format), a...)
}
// Fprint is a convenience wrapper for fmt.Fprint with support for color codes.
//
// Fprint formats using the default formats for its operands and writes to w
// with support for color codes. Spaces are added between operands when neither
// is a string. It returns the number of bytes written and any write error
// encountered.
func Fprint(w io.Writer, a string) (n int, err error) {
return fmt.Fprint(w, Color(a))
}
// Fprintln is a convenience wrapper for fmt.Fprintln with support for color
// codes.
//
// Fprintln formats using the default formats for its operands and writes to w
// with support for color codes. Spaces are always added between operands and a
// newline is appended. It returns the number of bytes written and any write
// error encountered.
func Fprintln(w io.Writer, a string) (n int, err error) {
return fmt.Fprintln(w, Color(a))
}
// Fprintf is a convenience wrapper for fmt.Fprintf with support for color
// codes.
//
// Fprintf formats according to a format specifier and writes to w with support
// for color codes. It returns the number of bytes written and any write error
// encountered.
func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(w, Color(format), a...)
}
module github.com/mitchellh/colorstring
language: go
go:
- tip
\ No newline at end of file
MIT License
Copyright (c) 2017 Zack
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR