Add IMDB ratings in database

Add method to fetch ratings from imdb every day
Create a new table and a new view to fetch directly the movies and shows
with their imdb ratings

backend/backend/imdb_ratings.go

@@ -0,0 +1,37 @@
+package backend
+
+import (
+	"time"
+
+	"github.com/jmoiron/sqlx"
+)
+
+const (
+	getRatingQueryByImdbID = ` SELECT * FROM imdb_ratings WHERE imdb_id=$1;`
+
+	upsertRatingQuery = `INSERT INTO imdb_ratings (imdb_id, rating, votes) VALUES (:imdb_id, :rating, :votes)
+	ON CONFLICT (imdb_id)
+	DO UPDATE
+	SET imdb_id=:imdb_id, rating=:rating, votes=:votes
+	RETURNING imdb_id;`
+)
+
+// ImdbRating represents the ImdbRating in the DB
+type ImdbRating struct {
+	ImdbID  string    `db:"imdb_id"`
+	Rating  float32   `db:"rating"`
+	Votes   int       `db:"votes"`
+	Created time.Time `db:"created_at"`
+	Updated time.Time `db:"updated_at"`
+}
+
+// UpsertImdbRating upsert a ImdbRating in the database
+func UpsertImdbRating(db *sqlx.DB, rating *ImdbRating) error {
+	r, err := db.NamedQuery(upsertRatingQuery, rating)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	return nil
+}

backend/backend/movies.go

@@ -25,7 +25,7 @@ const (
 
 	getMovieQueryByImdbID = `
 	SELECT *
-	FROM movies
+	FROM movies_with_rating
 	WHERE imdb_id=$1;`
 )
 

backend/backend/shows.go

@@ -20,7 +20,7 @@ const (
 
 	getShowQueryByImdbID = `
 	SELECT *
-	FROM shows WHERE imdb_id=$1;`
+	FROM shows_with_rating WHERE imdb_id=$1;`
 )
 
 // showDB represents the Show in the DB
@@ -32,6 +32,7 @@ type showDB struct {
 	TrackedEpisode *int      `db:"episode"`
 	Title          string    `db:"title"`
 	Rating         float32   `db:"rating"`
+	Votes          int       `db:"votes"`
 	Plot           string    `db:"plot"`
 	Year           int       `db:"year"`
 	FirstAired     time.Time `db:"first_aired"`

backend/main.go

@@ -9,6 +9,7 @@ import (
 	"git.quimbo.fr/odwrtw/canape/backend/backend"
 	"git.quimbo.fr/odwrtw/canape/backend/config"
 	extmedias "git.quimbo.fr/odwrtw/canape/backend/external_medias"
+	"git.quimbo.fr/odwrtw/canape/backend/ratings"
 	"git.quimbo.fr/odwrtw/canape/backend/web"
 
 	"github.com/jmoiron/sqlx"
@@ -80,6 +81,12 @@ func main() {
 			env.Log.Infof("Running refresh cron!")
 			extmedias.Refresh(env)
 		})
+
+		env.Log.Debugf("Running Imdb refresh cron every 24h")
+		c.AddFunc(fmt.Sprintf("@every 24h"), func() {
+			env.Log.Infof("Running IMDB refresh cron!")
+			ratings.Refresh(env)
+		})
 	}
 
 	// Start the cron

backend/movies/movies.go

@@ -7,13 +7,13 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/odwrtw/papi"
-	"github.com/odwrtw/polochon/lib"
-	"github.com/sirupsen/logrus"
 	"git.quimbo.fr/odwrtw/canape/backend/backend"
 	"git.quimbo.fr/odwrtw/canape/backend/subtitles"
 	"git.quimbo.fr/odwrtw/canape/backend/users"
 	"git.quimbo.fr/odwrtw/canape/backend/web"
+	"github.com/odwrtw/papi"
+	"github.com/odwrtw/polochon/lib"
+	"github.com/sirupsen/logrus"
 )
 
 // Movie represents a movie
@@ -124,6 +124,11 @@ func (m *Movie) GetAndFetch(env *web.Env, before []polochon.Detailer, after []po
 // Refresh retrieves details for the movie with the given detailers
 // and update them in database
 func (m *Movie) Refresh(env *web.Env, detailers []polochon.Detailer) error {
+	log := env.Log.WithFields(logrus.Fields{
+		"imdb_id":  m.ImdbID,
+		"function": "movies.Refresh",
+	})
+
 	// Refresh
 	err := m.GetDetails(env, detailers)
 	if err != nil {
@@ -133,11 +138,9 @@ func (m *Movie) Refresh(env *web.Env, detailers []polochon.Detailer) error {
 	// Download poster
 	err = web.Download(m.Thumb, m.imgFile())
 	if err != nil {
-		return err
+		log.Errorf("got error trying to download the poster %q", err)
 	}
 
-	env.Log.Debug("poster downloaded")
-
 	// If found, update in database
 	return backend.UpsertMovie(env.Database, m.Movie)
 }

backend/ratings/handlers.go

@@ -0,0 +1,24 @@
+package ratings
+
+import (
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+
+	"git.quimbo.fr/odwrtw/canape/backend/web"
+)
+
+// RefreshHandler refresh the imdb ratings
+func RefreshHandler(env *web.Env, w http.ResponseWriter, r *http.Request) error {
+	log := env.Log.WithFields(logrus.Fields{
+		"function": "ratings.RefreshHandler",
+	})
+	log.Debugf("refreshing imdb ratings")
+	err := Refresh(env)
+	if err != nil {
+		return env.RenderError(w, err)
+	}
+
+	log.Debugf("done refreshing imdb ratings")
+	return env.RenderOK(w, "Ratings Refreshed")
+}

backend/ratings/ratings.go

@@ -0,0 +1,73 @@
+package ratings
+
+import (
+	"bufio"
+	"compress/gzip"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"git.quimbo.fr/odwrtw/canape/backend/backend"
+	"git.quimbo.fr/odwrtw/canape/backend/web"
+)
+
+const imdbRatingsURL = "https://datasets.imdbws.com/title.ratings.tsv.gz"
+
+// Refresh will refresh the ImdbRatings
+func Refresh(env *web.Env) error {
+	log := env.Log.WithFields(logrus.Fields{
+		"function": "imdbRating.Refresh",
+	})
+
+	// Download the data
+	var httpClient = &http.Client{
+		Timeout: time.Second * 10,
+	}
+	resp, err := httpClient.Get(imdbRatingsURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Unzip it
+	r, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// Read it
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		elmts := strings.Split(scanner.Text(), "\t")
+		if len(elmts) != 3 {
+			log.Debugf("got %d elements weird\n", len(elmts))
+			continue
+		}
+
+		rating, err := strconv.ParseFloat(elmts[1], 64)
+		if err != nil {
+			log.Debugf("failed to parse rating %s\n", elmts[1])
+			continue
+		}
+		numVote, err := strconv.ParseInt(elmts[2], 10, 64)
+		if err != nil {
+			log.Debugf("failed to parse numVote %q\n", elmts[2])
+			continue
+		}
+
+		movie := &backend.ImdbRating{
+			ImdbID: elmts[0],
+			Rating: float32(rating),
+			Votes:  int(numVote),
+		}
+		err = backend.UpsertImdbRating(env.Database, movie)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

backend/routes.go

@@ -4,6 +4,7 @@ import (
 	admin "git.quimbo.fr/odwrtw/canape/backend/admins"
 	extmedias "git.quimbo.fr/odwrtw/canape/backend/external_medias"
 	"git.quimbo.fr/odwrtw/canape/backend/movies"
+	"git.quimbo.fr/odwrtw/canape/backend/ratings"
 	"git.quimbo.fr/odwrtw/canape/backend/shows"
 	"git.quimbo.fr/odwrtw/canape/backend/torrents"
 	"git.quimbo.fr/odwrtw/canape/backend/users"
@@ -44,6 +45,8 @@ func setupRoutes(env *web.Env) {
 	env.Handle("/shows/{id:tt[0-9]+}/seasons/{season:[0-9]+}/episodes/{episode:[0-9]+}/subtitles/{lang}", shows.DownloadVVTSubtitle).WithRole(users.UserRole).Methods("GET")
 	env.Handle("/shows/refresh", extmedias.RefreshShowsHandler).WithRole(users.AdminRole).Methods("POST")
 
+	env.Handle("/ratings/refresh", ratings.RefreshHandler).WithRole(users.AdminRole).Methods("POST")
+
 	// Wishlist routes for shows
 	env.Handle("/wishlist/shows", shows.GetWishlistHandler).WithRole(users.UserRole).Methods("GET")
 	env.Handle("/wishlist/shows/{id:tt[0-9]+}", shows.AddToWishlist).WithRole(users.UserRole).Methods("POST")

migrations/0007_imdb_ratings.down.sql

@@ -0,0 +1,4 @@
+DROP TABLE IF EXISTS imdb_ratings CASCADE;
+
+DROP VIEW IF EXISTS shows_with_rating;
+DROP VIEW IF EXISTS movies_with_rating;

migrations/0007_imdb_ratings.up.sql

@@ -0,0 +1,14 @@
+DROP TABLE IF EXISTS imdb_ratings CASCADE;
+CREATE TABLE imdb_ratings (
+    imdb_id text PRIMARY KEY NOT NULL,
+    rating real NOT NULL DEFAULT 0,
+    votes int NOT NULL DEFAULT 1,
+    LIKE base INCLUDING DEFAULTS
+);
+
+CREATE INDEX ON imdb_ratings (imdb_id);
+CREATE TRIGGER update_imdb_ratings_updated_at BEFORE UPDATE ON imdb_ratings FOR EACH ROW EXECUTE PROCEDURE  update_updated_at_column();
+
+CREATE OR REPLACE VIEW movies_with_rating AS SELECT m.id, m.imdb_id, m.title, m.plot, m.tmdb_id, m.year, m.original_title, m.runtime, m.sort_title, m.tagline, m.genres, r.rating, r.votes, m.updated_at, m.created_at FROM movies m JOIN imdb_ratings r ON m.imdb_id = r.imdb_id;
+CREATE OR REPLACE VIEW shows_with_rating AS SELECT s.id, s.imdb_id, s.title, s.plot, s.tvdb_id, s.year, s.first_aired, r.rating, r.votes, s.updated_at, s.created_at FROM shows s JOIN imdb_ratings r ON s.imdb_id = r.imdb_id;
+