diff --git a/backend/backend/imdb_ratings.go b/backend/backend/imdb_ratings.go new file mode 100644 index 0000000..12d1856 --- /dev/null +++ b/backend/backend/imdb_ratings.go @@ -0,0 +1,37 @@ +package backend + +import ( + "time" + + "github.com/jmoiron/sqlx" +) + +const ( + getRatingQueryByImdbID = ` SELECT * FROM imdb_ratings WHERE imdb_id=$1;` + + upsertRatingQuery = `INSERT INTO imdb_ratings (imdb_id, rating, votes) VALUES (:imdb_id, :rating, :votes) + ON CONFLICT (imdb_id) + DO UPDATE + SET imdb_id=:imdb_id, rating=:rating, votes=:votes + RETURNING imdb_id;` +) + +// ImdbRating represents the ImdbRating in the DB +type ImdbRating struct { + ImdbID string `db:"imdb_id"` + Rating float32 `db:"rating"` + Votes int `db:"votes"` + Created time.Time `db:"created_at"` + Updated time.Time `db:"updated_at"` +} + +// UpsertImdbRating upsert a ImdbRating in the database +func UpsertImdbRating(db *sqlx.DB, rating *ImdbRating) error { + r, err := db.NamedQuery(upsertRatingQuery, rating) + if err != nil { + return err + } + defer r.Close() + + return nil +} diff --git a/backend/backend/movies.go b/backend/backend/movies.go index ef5854d..2bacfe9 100644 --- a/backend/backend/movies.go +++ b/backend/backend/movies.go @@ -25,7 +25,7 @@ const ( getMovieQueryByImdbID = ` SELECT * - FROM movies + FROM movies_with_rating WHERE imdb_id=$1;` ) diff --git a/backend/backend/shows.go b/backend/backend/shows.go index 0abe958..24ddf4d 100644 --- a/backend/backend/shows.go +++ b/backend/backend/shows.go @@ -20,7 +20,7 @@ const ( getShowQueryByImdbID = ` SELECT * - FROM shows WHERE imdb_id=$1;` + FROM shows_with_rating WHERE imdb_id=$1;` ) // showDB represents the Show in the DB @@ -32,6 +32,7 @@ type showDB struct { TrackedEpisode *int `db:"episode"` Title string `db:"title"` Rating float32 `db:"rating"` + Votes int `db:"votes"` Plot string `db:"plot"` Year int `db:"year"` FirstAired time.Time `db:"first_aired"` diff --git a/backend/main.go b/backend/main.go index 6428511..33ab3a7 100644 --- a/backend/main.go +++ b/backend/main.go @@ -9,6 +9,7 @@ import ( "git.quimbo.fr/odwrtw/canape/backend/backend" "git.quimbo.fr/odwrtw/canape/backend/config" extmedias "git.quimbo.fr/odwrtw/canape/backend/external_medias" + "git.quimbo.fr/odwrtw/canape/backend/ratings" "git.quimbo.fr/odwrtw/canape/backend/web" "github.com/jmoiron/sqlx" @@ -80,6 +81,12 @@ func main() { env.Log.Infof("Running refresh cron!") extmedias.Refresh(env) }) + + env.Log.Debugf("Running Imdb refresh cron every 24h") + c.AddFunc(fmt.Sprintf("@every 24h"), func() { + env.Log.Infof("Running IMDB refresh cron!") + ratings.Refresh(env) + }) } // Start the cron diff --git a/backend/movies/movies.go b/backend/movies/movies.go index 56d62c3..2252d2f 100644 --- a/backend/movies/movies.go +++ b/backend/movies/movies.go @@ -7,13 +7,13 @@ import ( "os" "path/filepath" - "github.com/odwrtw/papi" - "github.com/odwrtw/polochon/lib" - "github.com/sirupsen/logrus" "git.quimbo.fr/odwrtw/canape/backend/backend" "git.quimbo.fr/odwrtw/canape/backend/subtitles" "git.quimbo.fr/odwrtw/canape/backend/users" "git.quimbo.fr/odwrtw/canape/backend/web" + "github.com/odwrtw/papi" + "github.com/odwrtw/polochon/lib" + "github.com/sirupsen/logrus" ) // Movie represents a movie @@ -124,6 +124,11 @@ func (m *Movie) GetAndFetch(env *web.Env, before []polochon.Detailer, after []po // Refresh retrieves details for the movie with the given detailers // and update them in database func (m *Movie) Refresh(env *web.Env, detailers []polochon.Detailer) error { + log := env.Log.WithFields(logrus.Fields{ + "imdb_id": m.ImdbID, + "function": "movies.Refresh", + }) + // Refresh err := m.GetDetails(env, detailers) if err != nil { @@ -133,11 +138,9 @@ func (m *Movie) Refresh(env *web.Env, detailers []polochon.Detailer) error { // Download poster err = web.Download(m.Thumb, m.imgFile()) if err != nil { - return err + log.Errorf("got error trying to download the poster %q", err) } - env.Log.Debug("poster downloaded") - // If found, update in database return backend.UpsertMovie(env.Database, m.Movie) } diff --git a/backend/ratings/handlers.go b/backend/ratings/handlers.go new file mode 100644 index 0000000..58e83a1 --- /dev/null +++ b/backend/ratings/handlers.go @@ -0,0 +1,24 @@ +package ratings + +import ( + "net/http" + + "github.com/sirupsen/logrus" + + "git.quimbo.fr/odwrtw/canape/backend/web" +) + +// RefreshHandler refresh the imdb ratings +func RefreshHandler(env *web.Env, w http.ResponseWriter, r *http.Request) error { + log := env.Log.WithFields(logrus.Fields{ + "function": "ratings.RefreshHandler", + }) + log.Debugf("refreshing imdb ratings") + err := Refresh(env) + if err != nil { + return env.RenderError(w, err) + } + + log.Debugf("done refreshing imdb ratings") + return env.RenderOK(w, "Ratings Refreshed") +} diff --git a/backend/ratings/ratings.go b/backend/ratings/ratings.go new file mode 100644 index 0000000..e41102c --- /dev/null +++ b/backend/ratings/ratings.go @@ -0,0 +1,73 @@ +package ratings + +import ( + "bufio" + "compress/gzip" + "net/http" + "strconv" + "strings" + "time" + + "github.com/sirupsen/logrus" + + "git.quimbo.fr/odwrtw/canape/backend/backend" + "git.quimbo.fr/odwrtw/canape/backend/web" +) + +const imdbRatingsURL = "https://datasets.imdbws.com/title.ratings.tsv.gz" + +// Refresh will refresh the ImdbRatings +func Refresh(env *web.Env) error { + log := env.Log.WithFields(logrus.Fields{ + "function": "imdbRating.Refresh", + }) + + // Download the data + var httpClient = &http.Client{ + Timeout: time.Second * 10, + } + resp, err := httpClient.Get(imdbRatingsURL) + if err != nil { + return err + } + defer resp.Body.Close() + + // Unzip it + r, err := gzip.NewReader(resp.Body) + if err != nil { + return err + } + + // Read it + scanner := bufio.NewScanner(r) + for scanner.Scan() { + elmts := strings.Split(scanner.Text(), "\t") + if len(elmts) != 3 { + log.Debugf("got %d elements weird\n", len(elmts)) + continue + } + + rating, err := strconv.ParseFloat(elmts[1], 64) + if err != nil { + log.Debugf("failed to parse rating %s\n", elmts[1]) + continue + } + numVote, err := strconv.ParseInt(elmts[2], 10, 64) + if err != nil { + log.Debugf("failed to parse numVote %q\n", elmts[2]) + continue + } + + movie := &backend.ImdbRating{ + ImdbID: elmts[0], + Rating: float32(rating), + Votes: int(numVote), + } + err = backend.UpsertImdbRating(env.Database, movie) + if err != nil { + return err + } + } + + return nil +} diff --git a/backend/routes.go b/backend/routes.go index 2968205..8ca124a 100644 --- a/backend/routes.go +++ b/backend/routes.go @@ -4,6 +4,7 @@ import ( admin "git.quimbo.fr/odwrtw/canape/backend/admins" extmedias "git.quimbo.fr/odwrtw/canape/backend/external_medias" "git.quimbo.fr/odwrtw/canape/backend/movies" + "git.quimbo.fr/odwrtw/canape/backend/ratings" "git.quimbo.fr/odwrtw/canape/backend/shows" "git.quimbo.fr/odwrtw/canape/backend/torrents" "git.quimbo.fr/odwrtw/canape/backend/users" @@ -44,6 +45,8 @@ func setupRoutes(env *web.Env) { env.Handle("/shows/{id:tt[0-9]+}/seasons/{season:[0-9]+}/episodes/{episode:[0-9]+}/subtitles/{lang}", shows.DownloadVVTSubtitle).WithRole(users.UserRole).Methods("GET") env.Handle("/shows/refresh", extmedias.RefreshShowsHandler).WithRole(users.AdminRole).Methods("POST") + env.Handle("/ratings/refresh", ratings.RefreshHandler).WithRole(users.AdminRole).Methods("POST") + // Wishlist routes for shows env.Handle("/wishlist/shows", shows.GetWishlistHandler).WithRole(users.UserRole).Methods("GET") env.Handle("/wishlist/shows/{id:tt[0-9]+}", shows.AddToWishlist).WithRole(users.UserRole).Methods("POST") diff --git a/migrations/0007_imdb_ratings.down.sql b/migrations/0007_imdb_ratings.down.sql new file mode 100644 index 0000000..185e111 --- /dev/null +++ b/migrations/0007_imdb_ratings.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS imdb_ratings CASCADE; + +DROP VIEW IF EXISTS shows_with_rating; +DROP VIEW IF EXISTS movies_with_rating; diff --git a/migrations/0007_imdb_ratings.up.sql b/migrations/0007_imdb_ratings.up.sql new file mode 100644 index 0000000..945a049 --- /dev/null +++ b/migrations/0007_imdb_ratings.up.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS imdb_ratings CASCADE; +CREATE TABLE imdb_ratings ( + imdb_id text PRIMARY KEY NOT NULL, + rating real NOT NULL DEFAULT 0, + votes int NOT NULL DEFAULT 1, + LIKE base INCLUDING DEFAULTS +); + +CREATE INDEX ON imdb_ratings (imdb_id); +CREATE TRIGGER update_imdb_ratings_updated_at BEFORE UPDATE ON imdb_ratings FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +CREATE OR REPLACE VIEW movies_with_rating AS SELECT m.id, m.imdb_id, m.title, m.plot, m.tmdb_id, m.year, m.original_title, m.runtime, m.sort_title, m.tagline, m.genres, r.rating, r.votes, m.updated_at, m.created_at FROM movies m JOIN imdb_ratings r ON m.imdb_id = r.imdb_id; +CREATE OR REPLACE VIEW shows_with_rating AS SELECT s.id, s.imdb_id, s.title, s.plot, s.tvdb_id, s.year, s.first_aired, r.rating, r.votes, s.updated_at, s.created_at FROM shows s JOIN imdb_ratings r ON s.imdb_id = r.imdb_id; +