如何处理大文件上传到Google Bucket?

5fjcxozz  于 2023-04-09  发布在  Go
关注(0)|答案(1)|浏览(123)

我有以下Golang代码来上传文件到Google bucket:

package main

import (
    "context"
    "fmt"
    "io"
    "net/http"
    "os"
    "time"

    "cloud.google.com/go/storage"
)

var serverPort = ":8444"
var googleCredential = "./credential.json"
var googleBucket = "g1-BUCKET-001"
var googleFolder = "test_folder/"

func uploadToBucket(w http.ResponseWriter, r *http.Request) {
    t1 := time.Now()
    fmt.Println("File Upload Endpoint Hit")

    // Parse our multipart form, 10 << 20 specifies a maximum
    // upload of 10 MB files.
    r.ParseMultipartForm(10 << 20)
    file, handler, err := r.FormFile("myFile")
    if err != nil {
        fmt.Fprintf(w, fmt.Sprintf("Error uploading file: %v", err))
        return
    }
    defer file.Close()
    fmt.Printf("Uploaded File: %+v\n", handler.Filename)
    fmt.Printf("File Size: %+v\n", handler.Size)
    fmt.Printf("MIME Header: %+v\n", handler.Header)

    // Upload file to bucket
    os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", googleCredential)
    ctx := context.Background()
    client, err := storage.NewClient(ctx)
    if err != nil {
        fmt.Fprintf(w, fmt.Sprintf("Error creating storage.NewClient: %v", err))
        return
    }
    defer client.Close()
    fmt.Println("Bucket client created.")

    // Set timeout
    ctx, cancel := context.WithTimeout(ctx, time.Second*7200)
    defer cancel()

    // Create bucket object for stream copy
    destFilePath := googleFolder + handler.Filename
    fmt.Printf("Target bucket: gs://" + googleBucket + "/.\n")
    fmt.Printf("Destination file path: " + destFilePath + ".\n")
    o := client.Bucket(googleBucket).Object(destFilePath)
    o = o.If(storage.Conditions{DoesNotExist: true})
    
    // Upload an object with storage.Writer.
    wc := o.NewWriter(ctx)
    if _, err = io.Copy(wc, file); err != nil {
        fmt.Fprintf(w, fmt.Sprintf("io.Copy error: %v", err))
        return
    }
    if err := wc.Close(); err != nil {
        fmt.Fprintf(w, fmt.Sprintf("Writer.Close() error: %v", err))
        return
    }
    fmt.Printf("%s uploaded to gs://%s/%s.", handler.Filename, googleBucket, googleFolder)

    t2 := time.Now()
    diff := t2.Sub(t1)
    fmt.Printf("Time start: %+v\n", t1)
    fmt.Printf("Time end: %+v\n", t2)
    fmt.Printf("Time diff: %+v\n", diff)

    // Return that we have successfully uploaded our file!
    fmt.Fprintf(w, "Successfully Uploaded File\n")
}

func setupRoutes() {
    http.HandleFunc("/upload", uploadToBucket)
    http.ListenAndServe(serverPort, nil)
}

func main() {
    setupRoutes()
}

它在100MB左右的文件中工作正常。但是当它达到1GB+时,等待时间太长了。用户可能会认为代码停止工作并在完成之前退出。现在,他们得到的只是最后的这行:

fmt.Fprintf(w, "Successfully Uploaded File\n")

我怎样才能实现一种方式来给予用户一些反馈,比如完成栏?

gojuced7

gojuced71#

这里有一个选项可以将进度跟踪添加到任意io.Writer,例如您从o.NewWriter(ctx)获得的。使用io.MultiWriter可以将写入复制到多个writer。一个可以是您的o.NewWriter(ctx),另一个可以是io.Writer的实现,它只计算字节数并可以与总大小进行比较。
然后,您可以运行一个goroutine,它可以通过写入stdout(或其他东西)来定期更新进度。
Take a look at this example使用This Proof of concept implementation

package main

import (
    "fmt"
    "io"
    "os"

    "github.com/farrellit/writeprogress"
)

func main() {
    in, err := os.Open("/dev/urandom")
    if err != nil {
        panic(err)
    }
    out, err := os.OpenFile("/dev/null", os.O_WRONLY, 0)
    if err != nil {
        panic(err)
    }
    defer in.Close()
    defer out.Close()

    length := int64(1e6)
    wp := writeprogress.NewProgressWriter(uint64(length))
    d, _ := wp.Watch(func(p float64) { fmt.Printf("\r%2.0f%%", p*100) })

    if b, err := io.Copy(
        io.MultiWriter(out, wp),
        &io.LimitedReader{R: in, N: length},
    ); err != nil {
        panic(err)
    } else {
        <-d
        fmt.Printf("\n%d/%d %2.0f%%\n", b, length, wp.GetProgress()*100)
    }

}

你可以用你的o.NewWriter(ctx)和一个由stat决定的文件长度来做同样的事情。如果你想要一些漂亮的东西,你也可以把它和https://github.com/schollz/progressbar结合使用。

相关问题