package ratings import ( "bufio" "compress/gzip" "net/http" "strconv" "strings" "time" "github.com/sirupsen/logrus" "git.quimbo.fr/odwrtw/canape/backend/backend" "git.quimbo.fr/odwrtw/canape/backend/web" ) const imdbRatingsURL = "https://datasets.imdbws.com/title.ratings.tsv.gz" // Refresh will refresh the ImdbRatings func Refresh(env *web.Env) error { log := env.Log.WithFields(logrus.Fields{ "function": "imdbRating.Refresh", }) // Download the data var httpClient = &http.Client{ Timeout: time.Second * 10, } resp, err := httpClient.Get(imdbRatingsURL) if err != nil { return err } defer resp.Body.Close() // Unzip it r, err := gzip.NewReader(resp.Body) if err != nil { return err } // Read it scanner := bufio.NewScanner(r) for scanner.Scan() { elmts := strings.Split(scanner.Text(), "\t") if len(elmts) != 3 { log.Debugf("got %d elements weird\n", len(elmts)) continue } rating, err := strconv.ParseFloat(elmts[1], 64) if err != nil { log.Debugf("failed to parse rating %s\n", elmts[1]) continue } numVote, err := strconv.ParseInt(elmts[2], 10, 64) if err != nil { log.Debugf("failed to parse numVote %q\n", elmts[2]) continue } movie := &backend.ImdbRating{ ImdbID: elmts[0], Rating: float32(rating), Votes: int(numVote), } err = backend.UpsertImdbRating(env.Database, movie) if err != nil { return err } } return nil }