diff --git a/backend/ratings/ratings.go b/backend/ratings/ratings.go index e41102c..c05479e 100644 --- a/backend/ratings/ratings.go +++ b/backend/ratings/ratings.go @@ -2,7 +2,9 @@ package ratings import ( "bufio" + "bytes" "compress/gzip" + "io/ioutil" "net/http" "strconv" "strings" @@ -32,8 +34,17 @@ func Refresh(env *web.Env) error { } defer resp.Body.Close() + // Read all the file (~5MB) in memory + // We do that because the ~1 000 000 upserts take too long, and the IMDB + // server will cut our connection after ~2h + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + readerContent := bytes.NewReader(content) // Unzip it - r, err := gzip.NewReader(resp.Body) + r, err := gzip.NewReader(readerContent) if err != nil { return err } @@ -43,18 +54,18 @@ func Refresh(env *web.Env) error { for scanner.Scan() { elmts := strings.Split(scanner.Text(), "\t") if len(elmts) != 3 { - log.Debugf("got %d elements weird\n", len(elmts)) + log.Debugf("got %d elements weird", len(elmts)) continue } rating, err := strconv.ParseFloat(elmts[1], 64) if err != nil { - log.Debugf("failed to parse rating %s\n", elmts[1]) + log.Debugf("failed to parse rating %s", elmts[1]) continue } numVote, err := strconv.ParseInt(elmts[2], 10, 64) if err != nil { - log.Debugf("failed to parse numVote %q\n", elmts[2]) + log.Debugf("failed to parse numVote %q", elmts[2]) continue }