From 226fd89f6a6bd5dc29f0a726d1df1ed36a65c657 Mon Sep 17 00:00:00 2001 From: Lucas BEE Date: Sun, 13 Nov 2016 18:41:03 +0000 Subject: [PATCH 1/4] Fix new user config --- src/internal/users/users.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/users/users.go b/src/internal/users/users.go index 30b7d25..80aa56b 100644 --- a/src/internal/users/users.go +++ b/src/internal/users/users.go @@ -80,7 +80,7 @@ func (u *User) SetConfig(key string, v interface{}) error { // NewConfig creates a new empty config func (u *User) NewConfig() error { - var configMap map[string]*json.RawMessage + configMap := make(map[string]*json.RawMessage) b, err := json.Marshal(configMap) if err != nil { return err From a9615b0abae444361e3acd2785ac879cbc147f83 Mon Sep 17 00:00:00 2001 From: Lucas BEE Date: Tue, 6 Dec 2016 12:23:26 +0000 Subject: [PATCH 2/4] Bigger sleep to make 'make dev' work --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 08c0bdd..1c1038f 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ watch-go: build-prepare CONFIG_FILE="./config.yml" fresh -c fresh.conf docker: $(DOCKER_COMPOSE) up -d - sleep 4 + sleep 8 migration-schema: docker $(MIGRATION) -path $(MIGRATION_SCHEMA) up From ab7503997f9f2dc633c45c6325b83c49fc694e48 Mon Sep 17 00:00:00 2001 From: Lucas BEE Date: Wed, 7 Dec 2016 09:32:20 +0000 Subject: [PATCH 3/4] Add Torrents Change the DB to store torrents in database Add Torrenters to the party Add raw links to movie Torrents in the web interface Change the way explore works with multiple source and categories with external_medias Delete StringSlice and use pq StringArray type to avoid problems --- sql/migration/0001_initial.up.sql | 1 + sql/migration/0002_external_medias.up.sql | 5 +- sql/migration/0003_torrents.down.sql | 3 + sql/migration/0003_torrents.up.sql | 30 +++ src/internal/config/canape.go | 17 +- .../external_medias/external_medias.go | 47 ++-- src/internal/external_medias/handlers.go | 171 +++++++++++++++ src/internal/movies/explorer.go | 7 - src/internal/movies/handlers.go | 41 +--- src/internal/movies/movies.go | 203 +++++++++++++----- src/internal/sqly/string_slice.go | 58 ----- src/internal/torrents/torrents.go | 180 ++++++++++++++++ src/main.go | 7 +- src/public/js/app.js | 2 +- src/public/js/components/movies/list.js | 7 + 15 files changed, 591 insertions(+), 188 deletions(-) create mode 100644 sql/migration/0003_torrents.down.sql create mode 100644 sql/migration/0003_torrents.up.sql create mode 100644 src/internal/external_medias/handlers.go delete mode 100644 src/internal/movies/explorer.go delete mode 100644 src/internal/sqly/string_slice.go create mode 100644 src/internal/torrents/torrents.go diff --git a/sql/migration/0001_initial.up.sql b/sql/migration/0001_initial.up.sql index 20e1578..4866e47 100644 --- a/sql/migration/0001_initial.up.sql +++ b/sql/migration/0001_initial.up.sql @@ -85,6 +85,7 @@ CREATE TABLE movies ( runtime integer NOT NULL, sort_title text NOT NULL, tagline text NOT NULL, + genres text[] NOT NULL, LIKE base INCLUDING DEFAULTS ); CREATE INDEX ON movies (imdb_id); diff --git a/sql/migration/0002_external_medias.up.sql b/sql/migration/0002_external_medias.up.sql index 6168db5..86a138f 100644 --- a/sql/migration/0002_external_medias.up.sql +++ b/sql/migration/0002_external_medias.up.sql @@ -1,15 +1,16 @@ CREATE TYPE media_type AS ENUM ('movie', 'show'); CREATE TYPE media_category AS ENUM ('trending', 'popular', 'anticipated', 'box_office'); -CREATE TYPE media_source AS ENUM ('trakttv'); +CREATE TYPE media_source AS ENUM ('trakttv', 'yts'); CREATE TABLE external_medias ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), type media_type NOT NULL, source media_source NOT NULL, category media_category NOT NULL, - ids text[][] NOT NULL, + ids text[] NOT NULL, LIKE base INCLUDING DEFAULTS ); CREATE TRIGGER update_external_medias_updated_at BEFORE UPDATE ON external_medias FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +CREATE UNIQUE INDEX ON external_medias (type, source, category); CREATE INDEX ON external_medias (type, source, category); diff --git a/sql/migration/0003_torrents.down.sql b/sql/migration/0003_torrents.down.sql new file mode 100644 index 0000000..ccd21cd --- /dev/null +++ b/sql/migration/0003_torrents.down.sql @@ -0,0 +1,3 @@ +DROP TABLE movie_torrents; +DROP TABLE episode_torrents; +DROP TABLE torrents_abstract; diff --git a/sql/migration/0003_torrents.up.sql b/sql/migration/0003_torrents.up.sql new file mode 100644 index 0000000..70fef4f --- /dev/null +++ b/sql/migration/0003_torrents.up.sql @@ -0,0 +1,30 @@ +CREATE TABLE torrents_abstract ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + imdb_id text NOT NULL, + url text NOT NULL, + source text NOT NULL, + quality text NOT NULL, + upload_user text NOT NULL, + seeders integer NOT NULL, + leechers integer NOT NULL, + LIKE base INCLUDING DEFAULTS +); + +CREATE TABLE movie_torrents ( +) +INHERITS (torrents_abstract); + +CREATE TABLE episode_torrents ( + season integer NOT NULL, + episode integer NOT NULL +) +INHERITS (torrents_abstract); + +CREATE INDEX ON movie_torrents (imdb_id); +CREATE UNIQUE INDEX ON movie_torrents (imdb_id, source, quality); +CREATE INDEX ON episode_torrents (imdb_id); +CREATE INDEX ON episode_torrents (imdb_id, season, episode); +CREATE INDEX ON torrents_abstract (imdb_id); + +CREATE TRIGGER update_movie_torrents_updated_at BEFORE UPDATE ON movie_torrents FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +CREATE TRIGGER update_episode_torrents_updated_at BEFORE UPDATE ON episode_torrents FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); diff --git a/src/internal/config/canape.go b/src/internal/config/canape.go index 43dc397..8a76564 100644 --- a/src/internal/config/canape.go +++ b/src/internal/config/canape.go @@ -6,6 +6,7 @@ import ( polochon "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/tmdb" + "github.com/odwrtw/polochon/modules/yts" "gopkg.in/yaml.v2" ) @@ -17,9 +18,10 @@ type Config struct { TraktTVClientID string `yaml:"trakttv_client_id"` PublicDir string `yaml:"public_dir"` - // TODO improve the detailers configurations - TmdbAPIKey string `yaml:"tmdb_api_key"` - MovieDetailers []polochon.Detailer + // TODO improve the detailers and torrenters configurations + TmdbAPIKey string `yaml:"tmdb_api_key"` + MovieDetailers []polochon.Detailer + MovieTorrenters []polochon.Torrenter } type AuthorizerConfig struct { @@ -47,6 +49,7 @@ func Load(path string) (*Config, error) { return nil, err } + // Default detailers cf.MovieDetailers = []polochon.Detailer{} if cf.TmdbAPIKey != "" { d, err := tmdb.New(&tmdb.Params{cf.TmdbAPIKey}) @@ -56,5 +59,13 @@ func Load(path string) (*Config, error) { cf.MovieDetailers = append(cf.MovieDetailers, d) } + // Default torrenters + cf.MovieTorrenters = []polochon.Torrenter{} + d, err := yts.New() + if err != nil { + return nil, err + } + cf.MovieTorrenters = append(cf.MovieTorrenters, d) + return cf, nil } diff --git a/src/internal/external_medias/external_medias.go b/src/internal/external_medias/external_medias.go index 2cbe7ba..ba94b0a 100644 --- a/src/internal/external_medias/external_medias.go +++ b/src/internal/external_medias/external_medias.go @@ -2,12 +2,18 @@ package extmedias import ( "github.com/jmoiron/sqlx" + "github.com/lib/pq" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" ) const ( - addExternalMediaQuery = `INSERT INTO external_medias (type, source, category, ids) VALUES ($1, $2, $3, $4) RETURNING id;` - updateExternalMediaQuery = `UPDATE external_medias SET type=:type, source=:source, category=:category, ids=:ids WHERE id=:id RETURNING *;` + // addExternalMediaQuery = `INSERT INTO external_medias (type, source, category, ids) VALUES ($1, $2, $3, $4) RETURNING id;` + upsertExternalMediaQuery = ` + INSERT INTO external_medias (type, source, category, ids) + VALUES (:type, :source, :category, :ids) + ON CONFLICT (type, source, category) + DO UPDATE SET type=:type, source=:source, category=:category, ids=:ids + RETURNING id;` deleteExternalMediaQuery = `DELETE FROM external_medias WHERE id=:id;` getExternalMediaQuery = `SELECT * FROM external_medias WHERE type=$1 AND source=$2 AND category=$3 LIMIT 1;` @@ -16,45 +22,36 @@ const ( // Media represents an external media type Media struct { sqly.BaseModel - Type string `db:"type"` - Source string `db:"source"` - Category string `db:"category"` - IDs sqly.StringSlice `db:"ids"` + Type string `db:"type"` + Source string `db:"source"` + Category string `db:"category"` + IDs pq.StringArray `db:"ids"` } -// Add adds the Media in the database -func (m *Media) Add(q sqlx.Queryer) error { +// Upsert adds or updates or adds the Media in the database +func (m *Media) Upsert(db *sqlx.DB) error { var id string - err := q.QueryRowx(addExternalMediaQuery, m.Type, m.Source, m.Category, m.IDs).Scan(&id) + r, err := db.NamedQuery(upsertExternalMediaQuery, m) if err != nil { return err } + for r.Next() { + r.Scan(&id) + } m.ID = id return nil } -// Update ids only updates the IDs of the media -func (m *Media) Update(ex *sqlx.DB) error { - rows, err := ex.NamedQuery(updateExternalMediaQuery, m) - if err != nil { - return err - } - for rows.Next() { - rows.StructScan(m) - } - return nil -} - // Delete the media from database or raise an error -func (m *Media) Delete(ex *sqlx.DB) error { - _, err := ex.NamedExec(deleteExternalMediaQuery, m) +func (m *Media) Delete(db *sqlx.DB) error { + _, err := db.NamedExec(deleteExternalMediaQuery, m) return err } // Get gets a media -func Get(q sqlx.Queryer, mtype, msrc, mcat string) (*Media, error) { +func Get(db *sqlx.DB, mtype, msrc, mcat string) (*Media, error) { m := &Media{} - if err := q.QueryRowx(getExternalMediaQuery, mtype, msrc, mcat).StructScan(m); err != nil { + if err := db.QueryRowx(getExternalMediaQuery, mtype, msrc, mcat).StructScan(m); err != nil { return nil, err } diff --git a/src/internal/external_medias/handlers.go b/src/internal/external_medias/handlers.go new file mode 100644 index 0000000..bbe6c0a --- /dev/null +++ b/src/internal/external_medias/handlers.go @@ -0,0 +1,171 @@ +package extmedias + +import ( + "database/sql" + "fmt" + "net/http" + + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies" + + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" + + "github.com/Sirupsen/logrus" + polochon "github.com/odwrtw/polochon/lib" + traktExplorer "github.com/odwrtw/polochon/modules/trakttv" + ytsExplorer "github.com/odwrtw/polochon/modules/yts" +) + +// GetMediaIDs will get some media IDs +func GetMediaIDs(env *web.Env, mediaType string, source string, category string, force bool) ([]string, error) { + log := env.Log.WithFields(logrus.Fields{ + "mediaType": mediaType, + "source": source, + "category": category, + "function": "extmedias.GetMediaIds", + }) + + var err error + media, err := Get(env.Database, mediaType, source, category) + switch err { + case nil: + log.Debug("medias found in database") + if !force { + log.Debug("returning medias from db") + return media.IDs, nil + } + case sql.ErrNoRows: + log.Debug("medias not found in database") + default: + // Unexpected error + return nil, err + } + + explorer, err := NewExplorer(env, source) + if err != nil { + return nil, err + } + + var ids []string + movies, err := explorer.GetMovieList(polochon.ExploreByRate, log) + for _, movie := range movies { + ids = append(ids, movie.ImdbID) + } + + log.Debugf("got %d medias from %s", len(ids), source) + + media = &Media{ + Type: mediaType, + Source: source, + Category: category, + IDs: ids, + } + + log.Debugf("Inserting int to DB %+v", media) + err = media.Upsert(env.Database) + if err != nil { + return nil, err + } + + log.Debug("medias updated in database") + + return ids, nil +} + +// GetMedias get some movies +func GetMedias(env *web.Env, source string, category string, force bool) ([]*movies.Movie, error) { + movieIds, err := GetMediaIDs(env, "movie", source, category, force) + if err != nil { + return nil, err + } + + movieList := []*movies.Movie{} + for _, id := range movieIds { + movie := movies.New(id) + err := movie.GetDetails(env, force) + if err != nil { + env.Log.Error(err) + continue + } + err = movie.GetTorrents(env, force) + if err != nil { + env.Log.Error(err) + continue + } + movieList = append(movieList, movie) + } + return movieList, nil +} + +// Explore will explore some movies +func Explore(env *web.Env, w http.ResponseWriter, r *http.Request) error { + err := r.ParseForm() + if err != nil { + return err + } + + source := r.FormValue("source") + // Default source + if source == "" { + source = "yts" + } + + category := r.FormValue("category") + // Default category + if category == "" { + category = "popular" + } + + // Get the medias without trying to refresh them + movies, err := GetMedias(env, source, category, false) + if err != nil { + return err + } + + return env.RenderJSON(w, movies) +} + +// MediaSources represents the implemented media sources +var MediaSources = []string{ + "trakttv", + "yts", +} + +// Refresh will refresh the movie list +func Refresh(env *web.Env, w http.ResponseWriter, r *http.Request) error { + env.Log.Debugf("refreshing infos ...") + source := r.FormValue("source") + if source == "" { + source = "yts" + } + + category := r.FormValue("category") + if category == "" { + category = "popular" + } + + // We'll refresh the medias for each sources + for _, source := range MediaSources { + env.Log.Debugf("refreshing %s", source) + // GetMedias and refresh them + _, err := GetMedias(env, source, category, true) + if err != nil { + return err + } + } + + return env.RenderJSON(w, map[string]string{"message": "Refresh is done"}) +} + +// NewExplorer returns a polochon.Explorer +func NewExplorer(env *web.Env, source string) (polochon.Explorer, error) { + switch source { + case "trakttv": + return traktExplorer.NewExplorer(&traktExplorer.Params{ + ClientID: env.Config.TraktTVClientID, + }) + case "yts": + return ytsExplorer.NewExplorer() + default: + return nil, fmt.Errorf("unknown explorer") + } +} diff --git a/src/internal/movies/explorer.go b/src/internal/movies/explorer.go deleted file mode 100644 index 6c38820..0000000 --- a/src/internal/movies/explorer.go +++ /dev/null @@ -1,7 +0,0 @@ -package movies - -import "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" - -func updatePopular(env *web.Env, clientID string) error { - return nil -} diff --git a/src/internal/movies/handlers.go b/src/internal/movies/handlers.go index 08c14e8..be12419 100644 --- a/src/internal/movies/handlers.go +++ b/src/internal/movies/handlers.go @@ -10,7 +10,6 @@ import ( "github.com/odwrtw/papi" polochon "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/pam" - "github.com/odwrtw/trakttv" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config" @@ -63,6 +62,7 @@ func getPolochonMovies(user *users.User) ([]*Movie, error) { return movies, nil } +// FromPolochon will returns movies from Polochon func FromPolochon(env *web.Env, w http.ResponseWriter, r *http.Request) error { v := auth.GetCurrentUser(r, env.Log) user, ok := v.(*users.User) @@ -97,45 +97,6 @@ func FromPolochon(env *web.Env, w http.ResponseWriter, r *http.Request) error { return env.RenderJSON(w, movies) } -// ExplorePopular returns the popular movies -func ExplorePopular(env *web.Env, w http.ResponseWriter, r *http.Request) error { - log := env.Log.WithField("function", "movies.ExplorePopular") - - queryOption := trakttv.QueryOption{ - ExtendedInfos: []trakttv.ExtendedInfo{ - trakttv.ExtendedInfoMin, - }, - Pagination: trakttv.Pagination{ - Page: 1, - Limit: 20, - }, - } - trakt := trakttv.New(env.Config.TraktTVClientID) - trakt.Endpoint = trakttv.ProductionEndpoint - - log.Debug("getting movies from trakttv") - tmovies, err := trakt.PopularMovies(queryOption) - if err != nil { - return err - } - log.Debugf("got %d movies from trakttv", len(tmovies)) - - movies := []*Movie{} - ids := []string{} - for _, m := range tmovies { - movie := New(m.IDs.ImDB) - err := movie.GetDetails(env, false) - if err != nil { - env.Log.Error(err) - continue - } - movies = append(movies, movie) - ids = append(ids, m.IDs.ImDB) - } - - return env.RenderJSON(w, movies) -} - // GetDetailsHandler retrieves details for a movie func GetDetailsHandler(env *web.Env, w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) diff --git a/src/internal/movies/movies.go b/src/internal/movies/movies.go index 92148ef..29114d9 100644 --- a/src/internal/movies/movies.go +++ b/src/internal/movies/movies.go @@ -1,51 +1,106 @@ package movies import ( + "database/sql" "fmt" "os" "path/filepath" + "time" "github.com/Sirupsen/logrus" "github.com/jmoiron/sqlx" + "github.com/lib/pq" "github.com/odwrtw/polochon/lib" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/torrents" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" ) const ( - addMovieQuery = ` - INSERT INTO movies (imdb_id, title, rating, votes, plot, tmdb_id, year, original_title, runtime, sort_title, tagline) - VALUES (:imdbid, :title, :rating, :votes, :plot, :tmdbid, :year, :originaltitle, :runtime, :sorttitle, :tagline) + upsertMovieQuery = ` + INSERT INTO movies (imdb_id, title, rating, votes, plot, tmdb_id, year, + genres, original_title, runtime, sort_title, tagline) + VALUES (:imdb_id, :title, :rating, :votes, :plot, :tmdb_id, :year, :genres, + :original_title, :runtime, :sort_title, :tagline) + ON CONFLICT (imdb_id) + DO UPDATE + SET imdb_id=:imdb_id, title=:title, rating=:rating, votes=:votes, + plot=:plot, tmdb_id=:tmdb_id, year=:year, genres=:genres, + original_title=:original_title, runtime=:runtime, sort_title=:sort_title, + tagline=:tagline RETURNING id;` - updateMovieQuery = ` - UPDATE movies - SET imdb_id=:imdbid, title=:title, rating=:rating, votes=:votes, plot=:plot, tmdb_id=:tmdbid, year=:year, original_title=:originaltitle, runtime=:runtime, sort_title=:sorttitle, tagline=:tagline - WHERE ID = :id;` - getMovieQueryByImdbID = ` - SELECT - id, imdb_id AS imdbid, title, rating, votes, plot, - tmdb_id AS tmdbid, year, - original_title AS originaltitle, runtime, - sort_title AS sorttitle, tagline, - created_at, updated_at + SELECT * FROM movies WHERE imdb_id=$1;` + getMovieQueryByID = ` - SELECT - id, imdb_id AS imdbid, title, rating, votes, plot, - tmdb_id AS tmdbid, year, - original_title AS originaltitle, runtime, - sort_title AS sorttitle, tagline, - created_at, updated_at + SELECT * FROM movies WHERE id=$1;` + deleteMovieQuery = `DELETE FROM movies WHERE id=$1;` ) -var ( - // ErrNotFound error returned when show not found in database - ErrNotFound = fmt.Errorf("Not found") -) +// MovieDB represents the Movie in the DB +type MovieDB struct { + ID string `db:"id"` + ImdbID string `db:"imdb_id"` + TmdbID int `db:"tmdb_id"` + Title string `db:"title"` + OriginalTitle string `db:"original_title"` + SortTitle string `db:"sort_title"` + Rating float32 `db:"rating"` + Votes int `db:"votes"` + Plot string `db:"plot"` + Year int `db:"year"` + Runtime int `db:"runtime"` + Tagline string `db:"tagline"` + Genres pq.StringArray `db:"genres"` + Created time.Time `db:"created_at"` + Updated time.Time `db:"updated_at"` +} + +// NewMovieDB returns a Movie ready to be put in DB from a +// Movie +func NewMovieDB(m *Movie) MovieDB { + return MovieDB{ + ID: m.ID, + ImdbID: m.ImdbID, + Title: m.Title, + Rating: m.Rating, + Votes: m.Votes, + Plot: m.Plot, + TmdbID: m.TmdbID, + Year: m.Year, + OriginalTitle: m.OriginalTitle, + Runtime: m.Runtime, + SortTitle: m.SortTitle, + Tagline: m.Tagline, + Genres: m.Genres, + Created: m.Created, + Updated: m.Updated, + } +} + +// FillFromDB returns a Movie from a MovieDB extracted from the DB +func (m *Movie) FillFromDB(mDB *MovieDB) { + m.Created = mDB.Created + m.Updated = mDB.Updated + m.ID = mDB.ID + m.ImdbID = mDB.ImdbID + m.Title = mDB.Title + m.Rating = mDB.Rating + m.Votes = mDB.Votes + m.Plot = mDB.Plot + m.TmdbID = mDB.TmdbID + m.Year = mDB.Year + m.OriginalTitle = mDB.OriginalTitle + m.Runtime = mDB.Runtime + m.Genres = mDB.Genres + m.SortTitle = mDB.SortTitle + m.Tagline = mDB.Tagline +} // Movie represents a movie type Movie struct { @@ -55,6 +110,7 @@ type Movie struct { PosterURL string `json:"poster_url"` } +// New returns a new Movie with an ImDB id func New(imdbID string) *Movie { return &Movie{ Movie: polochon.Movie{ @@ -65,24 +121,23 @@ func New(imdbID string) *Movie { // Get returns show details in database from id or imdbid or an error func (m *Movie) Get(env *web.Env) error { + var mDB MovieDB var err error if m.ID != "" { - err = env.Database.QueryRowx(getMovieQueryByID, m.ID).StructScan(m) + err = env.Database.QueryRowx(getMovieQueryByID, m.ID).StructScan(&mDB) } else if m.ImdbID != "" { - err = env.Database.QueryRowx(getMovieQueryByImdbID, m.ImdbID).StructScan(m) + err = env.Database.QueryRowx(getMovieQueryByImdbID, m.ImdbID).StructScan(&mDB) } else { err = fmt.Errorf("Can't get movie details, you have to specify an ID or ImdbID") } if err != nil { - if err.Error() == "sql: no rows in result set" { - return ErrNotFound - } return err } // Set the poster url m.PosterURL = m.GetPosterURL(env) + m.FillFromDB(&mDB) return nil } @@ -93,37 +148,33 @@ func (m *Movie) Get(env *web.Env) error { // If force is used, the detailer will be used even if the movie is found in // database func (m *Movie) GetDetails(env *web.Env, force bool) error { - if len(m.Detailers) == 0 { - m.Detailers = env.Config.MovieDetailers - } - log := env.Log.WithFields(logrus.Fields{ "imdb_id": m.ImdbID, "function": "movies.GetDetails", }) + log.Debugf("getting details") - // If the movie is not in db, we should add it, otherwise we should update - // it - var dbFunc func(db *sqlx.DB) error + if len(m.Detailers) == 0 { + m.Detailers = env.Config.MovieDetailers + } var err error err = m.Get(env) switch err { case nil: log.Debug("movie found in database") - dbFunc = m.Update if !force { + log.Debug("returning movie from db") return nil } - case ErrNotFound: - dbFunc = m.Add + case sql.ErrNoRows: log.Debug("movie not found in database") default: // Unexpected error return err } - // so we got ErrNotFound so GetDetails from a detailer + // GetDetail err = m.Movie.GetDetails(env.Log) if err != nil { return err @@ -131,8 +182,9 @@ func (m *Movie) GetDetails(env *web.Env, force bool) error { log.Debug("got details from detailers") - err = dbFunc(env.Database) + err = m.Upsert(env.Database) if err != nil { + log.Debug("error while doing db func") return err } @@ -152,10 +204,67 @@ func (m *Movie) GetDetails(env *web.Env, force bool) error { return nil } -// Add a movie in the database -func (m *Movie) Add(db *sqlx.DB) error { +// GetTorrents retrieves torrents for the movie, first try to get info from db, +// if not exists, use polochon.Torrenter and save informations in the database +// for future use +// +// If force is used, the torrenter will be used even if the torrent is found in +// database +func (m *Movie) GetTorrents(env *web.Env, force bool) error { + log := env.Log.WithFields(logrus.Fields{ + "imdb_id": m.ImdbID, + "function": "movies.GetTorrents", + }) + log.Debugf("getting torrents") + + if len(m.Torrenters) == 0 { + m.Torrenters = env.Config.MovieTorrenters + } + + movieTorrents, err := torrents.GetMovieTorrents(env.Database, m.ImdbID) + switch err { + case nil: + log.Debug("torrents found in database") + if !force { + log.Debugf("returning %d torrents from db", len(movieTorrents)) + // Add the torrents to the movie + for _, t := range movieTorrents { + m.Torrents = append(m.Torrents, t.Torrent) + } + return nil + } + case sql.ErrNoRows: + log.Debug("torrent not found in database") + // We'll need to GetTorrents from torrenters + default: + // Unexpected error + return err + } + + err = m.Movie.GetTorrents(env.Log) + if err != nil { + return err + } + + log.Debugf("got %d torrents from torrenters", len(m.Movie.Torrents)) + + for _, t := range m.Movie.Torrents { + torrent := torrents.NewMovie(m.ImdbID, t) + err = torrent.Upsert(env.Database) + if err != nil { + log.Error("error while adding torrent", err) + continue + } + } + + return nil +} + +// Upsert a movie in the database +func (m *Movie) Upsert(db *sqlx.DB) error { + mDB := NewMovieDB(m) var id string - r, err := db.NamedQuery(addMovieQuery, m) + r, err := db.NamedQuery(upsertMovieQuery, mDB) if err != nil { return err } @@ -167,12 +276,6 @@ func (m *Movie) Add(db *sqlx.DB) error { return nil } -// Update a movie in the database -func (m *Movie) Update(db *sqlx.DB) error { - _, err := db.NamedQuery(updateMovieQuery, m) - return err -} - // Delete movie from database func (m *Movie) Delete(db *sqlx.DB) error { r, err := db.Exec(deleteMovieQuery, m.ID) diff --git a/src/internal/sqly/string_slice.go b/src/internal/sqly/string_slice.go deleted file mode 100644 index 14bada0..0000000 --- a/src/internal/sqly/string_slice.go +++ /dev/null @@ -1,58 +0,0 @@ -package sqly - -import ( - "database/sql/driver" - "encoding/csv" - "errors" - "regexp" - "strings" -) - -// This mainly comes from https://gist.github.com/adharris/4163702 - -// StringSlice represents an array of string. The custom type is needed because -// pq does not support slices yet.. -type StringSlice []string - -// for replacing escaped quotes except if it is preceded by a literal backslash -// eg "\\" should translate to a quoted element whose value is \ - -var quoteEscapeRegex = regexp.MustCompile(`([^\\]([\\]{2})*)\\"`) - -// Scan convert to a slice of strings -// http://www.postgresql.org/docs/9.1/static/arrays.html#ARRAYS-IO -func (s *StringSlice) Scan(src interface{}) error { - asBytes, ok := src.([]byte) - if !ok { - return error(errors.New("Scan source was not []bytes")) - } - str := string(asBytes) - - // change quote escapes for csv parser - str = quoteEscapeRegex.ReplaceAllString(str, `$1""`) - str = strings.Replace(str, `\\`, `\`, -1) - // remove braces - str = str[1 : len(str)-1] - csvReader := csv.NewReader(strings.NewReader(str)) - - slice, err := csvReader.Read() - - if err != nil { - return err - } - - (*s) = StringSlice(slice) - - return nil -} - -// Value implements the Valuer interface -func (s StringSlice) Value() (driver.Value, error) { - // string escapes. - // \ => \\\ - // " => \" - for i, elem := range s { - s[i] = `"` + strings.Replace(strings.Replace(elem, `\`, `\\\`, -1), `"`, `\"`, -1) + `"` - } - return "{" + strings.Join(s, ",") + "}", nil -} diff --git a/src/internal/torrents/torrents.go b/src/internal/torrents/torrents.go new file mode 100644 index 0000000..548c430 --- /dev/null +++ b/src/internal/torrents/torrents.go @@ -0,0 +1,180 @@ +package torrents + +import ( + "database/sql" + "fmt" + "time" + + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + + "github.com/jmoiron/sqlx" + polochon "github.com/odwrtw/polochon/lib" +) + +const ( + upsertMovieTorrentQuery = ` + INSERT INTO movie_torrents (imdb_id, url, source, quality, upload_user, + seeders, leechers) + VALUES (:imdb_id, :url, :source, :quality, :upload_user, :seeders, + :leechers) + ON CONFLICT (imdb_id, quality, source) + DO UPDATE SET imdb_id=:imdb_id, url=:url, source=:source, quality=:quality, + upload_user=:upload_user, seeders=:seeders, leechers=:leechers + RETURNING id;` + + getMovieTorrentQueryByImdbID = ` + SELECT * + FROM movie_torrents WHERE imdb_id=$1;` + + getMovieTorrentQueryByID = ` + SELECT * + FROM movie_torrents WHERE id=$1;` + + deleteMovieTorrentQuery = `DELETE FROM movie_torrents WHERE id=$1;` +) + +// MovieTorrent represents a movie torrent +type MovieTorrent struct { + sqly.BaseModel + polochon.Torrent + ImdbID string `json:"imdb_id"` +} + +// MovieTorrentDB represents the MovieTorrent in the DB +type MovieTorrentDB struct { + ID string `db:"id"` + ImdbID string `db:"imdb_id"` + URL string `db:"url"` + Source string `db:"source"` + Quality string `db:"quality"` + UploadUser string `db:"upload_user"` + Seeders int `db:"seeders"` + Leechers int `db:"leechers"` + Created time.Time `db:"created_at"` + Updated time.Time `db:"updated_at"` +} + +// EpisodeTorrent represents an episode torrent +type EpisodeTorrent struct { + sqly.BaseModel + polochon.Torrent + ShowImdbID string `json:"show_imdb_id"` + Season int `json:"season"` + Episode int `json:"episode"` +} + +// NewMovieFromImDB returns a new MovieTorrent with an ImDB id +func NewMovieFromImDB(imdbID string) *MovieTorrent { + return &MovieTorrent{ + ImdbID: imdbID, + } +} + +// NewMovie returns a new MovieTorrent with an ImDB id +func NewMovie(imdbID string, poTo polochon.Torrent) *MovieTorrent { + return &MovieTorrent{ + ImdbID: imdbID, + Torrent: poTo, + } +} + +// Get returns show details in database from id or imdbid or an error +func (m *MovieTorrent) Get(db *sqlx.DB) error { + var mDB MovieTorrentDB + var err error + if m.ID != "" { + err = db.QueryRowx(getMovieTorrentQueryByID, m.ID).StructScan(&mDB) + } else if m.ImdbID != "" { + err = db.QueryRowx(getMovieTorrentQueryByImdbID, m.ImdbID).StructScan(&mDB) + } else { + err = fmt.Errorf("can't get movie torrent details, you have to specify an ID or ImdbID") + } + if err != nil { + return err + } + m.FillFromDB(&mDB) + return nil +} + +// GetMovieTorrents returns show details in database from id or imdbid or an error +func GetMovieTorrents(db *sqlx.DB, imdbID string) ([]*MovieTorrent, error) { + var torrentsDB = []*MovieTorrentDB{} + err := db.Select(&torrentsDB, getMovieTorrentQueryByImdbID, imdbID) + if err != nil { + return nil, err + } + + if len(torrentsDB) == 0 { + return nil, sql.ErrNoRows + } + + var torrents []*MovieTorrent + for _, torrentDB := range torrentsDB { + movie := NewMovieFromImDB(imdbID) + movie.FillFromDB(torrentDB) + torrents = append(torrents, movie) + } + + return torrents, nil +} + +// Upsert a movie torrent in the database +func (m *MovieTorrent) Upsert(db *sqlx.DB) error { + mDB := NewMovieTorrentDB(m) + var id string + r, err := db.NamedQuery(upsertMovieTorrentQuery, mDB) + if err != nil { + return err + } + for r.Next() { + r.Scan(&id) + } + m.ID = id + + return nil +} + +// Delete movie from database +func (m *MovieTorrent) Delete(db *sqlx.DB) error { + r, err := db.Exec(deleteMovieTorrentQuery, m.ID) + if err != nil { + return err + } + count, _ := r.RowsAffected() + if count != 1 { + return fmt.Errorf("Unexpected number of row deleted: %d", count) + } + return nil +} + +// NewMovieTorrentDB returns a MovieTorrent ready to be put in DB from a +// MovieTorrent +func NewMovieTorrentDB(m *MovieTorrent) MovieTorrentDB { + return MovieTorrentDB{ + ID: m.ID, + ImdbID: m.ImdbID, + URL: m.URL, + Source: m.Source, + Quality: string(m.Quality), + UploadUser: m.UploadUser, + Seeders: m.Seeders, + Leechers: m.Leechers, + Created: m.Created, + Updated: m.Updated, + } +} + +// FillFromDB will fill a MovieTorrent from a MovieTorrentDB extracted from the DB +func (m *MovieTorrent) FillFromDB(mDB *MovieTorrentDB) { + q, _ := polochon.StringToQuality(mDB.Quality) + m.ImdbID = mDB.ImdbID + m.Created = mDB.Created + m.Updated = mDB.Updated + m.ID = mDB.ID + m.URL = mDB.URL + m.Source = mDB.Source + m.Quality = *q + m.UploadUser = mDB.UploadUser + m.Seeders = mDB.Seeders + m.Leechers = mDB.Leechers +} diff --git a/src/main.go b/src/main.go index f43ad06..544635d 100644 --- a/src/main.go +++ b/src/main.go @@ -4,9 +4,11 @@ import ( "net/http" "os" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config" - "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/external_medias" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/users" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" @@ -74,8 +76,9 @@ func main() { env.Handle("/users/edit", users.EditHandler).WithRole(users.UserRole) env.Handle("/movies/polochon", movies.FromPolochon).WithRole(users.UserRole) - env.Handle("/movies/explore/popular", movies.ExplorePopular).WithRole(users.UserRole) env.Handle("/movies/{id:tt[0-9]+}/get_details", movies.GetDetailsHandler).WithRole(users.UserRole) + env.Handle("/movies/explore", extmedias.Explore) + env.Handle("/movies/refresh", extmedias.Refresh) n := negroni.Classic() n.Use(authMiddleware) diff --git a/src/public/js/app.js b/src/public/js/app.js index be8e084..7a0a242 100644 --- a/src/public/js/app.js +++ b/src/public/js/app.js @@ -75,7 +75,7 @@ const UserIsAuthenticated = UserAuthWrapper({ // TODO find a better way const MovieListPopular = (props) => ( - + ) const MovieListPolochon = (props) => ( diff --git a/src/public/js/components/movies/list.js b/src/public/js/components/movies/list.js index 8a1fd08..4ec4dd0 100644 --- a/src/public/js/components/movies/list.js +++ b/src/public/js/components/movies/list.js @@ -130,6 +130,13 @@ class MovieButtons extends React.Component { Download } + {this.props.movie.torrents && this.props.movie.torrents.map(function(torrent, index) { + return ( + + {torrent.quality} Torrent + + )} + )} IMDB From 56617e3e324240d0f48789c1308c42d257f55c39 Mon Sep 17 00:00:00 2001 From: Lucas BEE Date: Thu, 8 Dec 2016 16:07:24 +0000 Subject: [PATCH 4/4] Make tests almost work again --- .../external_medias/external_medias_test.go | 4 +- src/internal/movies/movies_test.go | 24 +++- src/internal/shows/shows_test.go | 53 ++++++- src/internal/users/users_test.go | 129 ++++++++++++++++-- 4 files changed, 189 insertions(+), 21 deletions(-) diff --git a/src/internal/external_medias/external_medias_test.go b/src/internal/external_medias/external_medias_test.go index 2651ba3..519823a 100644 --- a/src/internal/external_medias/external_medias_test.go +++ b/src/internal/external_medias/external_medias_test.go @@ -37,13 +37,13 @@ func TestAddExternalMedias(t *testing.T) { } // Add it - if err := media.Add(db); err != nil { + if err := media.Upsert(db); err != nil { t.Fatalf("failed to add external media: %q", err) } // Update the IDs media.IDs = []string{"1", "2", "3", "4"} - if err := media.Update(db); err != nil { + if err := media.Upsert(db); err != nil { t.Fatalf("failed to update the external media: %q", err) } diff --git a/src/internal/movies/movies_test.go b/src/internal/movies/movies_test.go index d2797b1..d92fe39 100644 --- a/src/internal/movies/movies_test.go +++ b/src/internal/movies/movies_test.go @@ -1,6 +1,7 @@ package movies import ( + "database/sql" "fmt" "os" "testing" @@ -11,7 +12,9 @@ import ( _ "github.com/mattes/migrate/driver/postgres" "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/mock" - "gitlab.quimbo.fr/odwrtw/canape-sql/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" ) var db *sqlx.DB @@ -31,6 +34,15 @@ func init() { func TestIntegrate(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { + log := logrus.NewEntry(logrus.New()) + env := web.NewEnv(web.EnvParams{ + Database: db, + // Auth: authorizer, + Log: log, + Config: &config.Config{ + PublicDir: "/tmp", + }, + }) detailer, _ := mock.NewDetailer(nil) polochonConfig := polochon.MovieConfig{ Detailers: []polochon.Detailer{ @@ -41,11 +53,11 @@ func TestIntegrate(t *testing.T) { Movie: polochon.Movie{ MovieConfig: polochonConfig, ImdbID: "tt12345", + Genres: []string{}, }, } - log := logrus.NewEntry(logrus.New()) - err := movie.GetDetails(db, log) + err := movie.GetDetails(env, false) if err != nil { t.Fatal(err) } @@ -57,7 +69,7 @@ func TestIntegrate(t *testing.T) { ImdbID: "tt12345", }, } - err = movie.Get(db) + err = movie.Get(env) if err != nil { t.Fatal(err) } @@ -72,8 +84,8 @@ func TestIntegrate(t *testing.T) { } // Get it again - err = movie.Get(db) - if err != ErrNotFound { + err = movie.Get(env) + if err != sql.ErrNoRows { t.Fatalf("Unexpected error: %q", err) } }) diff --git a/src/internal/shows/shows_test.go b/src/internal/shows/shows_test.go index b7cdba9..7c411dd 100644 --- a/src/internal/shows/shows_test.go +++ b/src/internal/shows/shows_test.go @@ -5,15 +5,16 @@ import ( "os" "testing" - "gitlab.quimbo.fr/odwrtw/canape-sql/sqly" - "gitlab.quimbo.fr/odwrtw/canape-sql/users" - "github.com/Sirupsen/logrus" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" _ "github.com/mattes/migrate/driver/postgres" "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/mock" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/users" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" ) var db *sqlx.DB @@ -89,8 +90,40 @@ func TestIntegrate(t *testing.T) { t.Fatalf("Unexpected error: %q", err) } }) - } + +// UserBackend represents the data backend to get the user +type UserBackend struct { + Database *sqlx.DB +} + +// Get gets the username from the UserBackend +func (b *UserBackend) Get(username string) (auth.User, error) { + return users.Get(b.Database, username) +} + +func getEnv(db *sqlx.DB) *web.Env { + uBackend := &UserBackend{Database: db} + + authParams := auth.Params{ + Backend: uBackend, + Pepper: "pepper", + Cost: 10, + Secret: "secret", + } + authorizer := auth.New(authParams) + + log := logrus.NewEntry(logrus.New()) + env := web.NewEnv(web.EnvParams{ + Database: db, + Auth: authorizer, + Log: log, + // Config: cf, + }) + + return env +} + func TestTrackedShow(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { detailer, _ := mock.NewDetailer(nil) @@ -113,6 +146,18 @@ func TestTrackedShow(t *testing.T) { } u := &users.User{Name: "plop"} + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("pass") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + err = u.Add(db) if err != nil { t.Fatal(err) diff --git a/src/internal/users/users_test.go b/src/internal/users/users_test.go index 385c883..b1add23 100644 --- a/src/internal/users/users_test.go +++ b/src/internal/users/users_test.go @@ -5,8 +5,11 @@ import ( "os" "testing" - "gitlab.quimbo.fr/odwrtw/canape-sql/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" + "github.com/Sirupsen/logrus" "github.com/jmoiron/sqlx" "github.com/lib/pq" _ "github.com/mattes/migrate/driver/postgres" @@ -32,7 +35,19 @@ func TestUser(t *testing.T) { // Add a new user u := &User{Name: "plop", Hash: "plop"} - err := u.Add(db) + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("test") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + err = u.Add(db) if err != nil { t.Fatal(err) } @@ -99,7 +114,19 @@ func TestTokenAddDelete(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { // Add a new user u := &User{Name: "plop", Hash: "plop"} - err := u.Add(db) + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("test") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + err = u.Add(db) if err != nil { t.Fatal(err) } @@ -155,11 +182,60 @@ func TestTokenAddDelete(t *testing.T) { }) } +// UserBackend represents the data backend to get the user +type UserBackend struct { + Database *sqlx.DB +} + +// Get gets the username from the UserBackend +func (b *UserBackend) Get(username string) (auth.User, error) { + return Get(b.Database, username) +} + +func getEnv(db *sqlx.DB) *web.Env { + uBackend := &UserBackend{Database: db} + + authParams := auth.Params{ + Backend: uBackend, + Pepper: "pepper", + Cost: 10, + Secret: "secret", + } + authorizer := auth.New(authParams) + + log := logrus.NewEntry(logrus.New()) + env := web.NewEnv(web.EnvParams{ + Database: db, + Auth: authorizer, + Log: log, + // Config: cf, + }) + + return env +} + func TestTokenCheck(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { u := &User{Name: "plop", Hash: "plop"} - u.Add(db) + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("test") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + if err = u.Add(db); err != nil { + t.Fatal(err) + } token, err := u.NewToken(db) + if err != nil { + t.Fatal(err) + } ok, err := u.CheckToken(db, token.Value) if err != nil { @@ -183,10 +259,29 @@ func TestTokenCheck(t *testing.T) { func TestAutoUpdateCols(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { - u := &User{Name: "plop", Hash: "plop"} - u.Add(db) + u := &User{Name: "plop"} + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("pass") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + + err = u.Add(db) + if err != nil { + t.Fatal(err) + } u.Name = "toto" - u.Update(db) + err = u.Update(db) + if err != nil { + t.Fatal(err) + } if !u.Created.Before(u.Updated) { t.Fatalf("colum updated not auto updated on table users") @@ -197,9 +292,25 @@ func TestAutoUpdateCols(t *testing.T) { func TestConfig(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { u := &User{Name: "plop", Hash: "plop"} - u.Add(db) + env := getEnv(db) - u, err := Get(db, "plop") + var err error + u.Hash, err = env.Auth.GenHash("pass") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + + err = u.Add(db) + if err != nil { + t.Fatal(err) + } + + u, err = Get(db, "plop") if err != nil { t.Fatal(err) }