diff --git a/Makefile b/Makefile index 08c0bdd..1c1038f 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ watch-go: build-prepare CONFIG_FILE="./config.yml" fresh -c fresh.conf docker: $(DOCKER_COMPOSE) up -d - sleep 4 + sleep 8 migration-schema: docker $(MIGRATION) -path $(MIGRATION_SCHEMA) up diff --git a/sql/migration/0001_initial.up.sql b/sql/migration/0001_initial.up.sql index 20e1578..4866e47 100644 --- a/sql/migration/0001_initial.up.sql +++ b/sql/migration/0001_initial.up.sql @@ -85,6 +85,7 @@ CREATE TABLE movies ( runtime integer NOT NULL, sort_title text NOT NULL, tagline text NOT NULL, + genres text[] NOT NULL, LIKE base INCLUDING DEFAULTS ); CREATE INDEX ON movies (imdb_id); diff --git a/sql/migration/0002_external_medias.up.sql b/sql/migration/0002_external_medias.up.sql index 6168db5..86a138f 100644 --- a/sql/migration/0002_external_medias.up.sql +++ b/sql/migration/0002_external_medias.up.sql @@ -1,15 +1,16 @@ CREATE TYPE media_type AS ENUM ('movie', 'show'); CREATE TYPE media_category AS ENUM ('trending', 'popular', 'anticipated', 'box_office'); -CREATE TYPE media_source AS ENUM ('trakttv'); +CREATE TYPE media_source AS ENUM ('trakttv', 'yts'); CREATE TABLE external_medias ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), type media_type NOT NULL, source media_source NOT NULL, category media_category NOT NULL, - ids text[][] NOT NULL, + ids text[] NOT NULL, LIKE base INCLUDING DEFAULTS ); CREATE TRIGGER update_external_medias_updated_at BEFORE UPDATE ON external_medias FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +CREATE UNIQUE INDEX ON external_medias (type, source, category); CREATE INDEX ON external_medias (type, source, category); diff --git a/sql/migration/0003_torrents.down.sql b/sql/migration/0003_torrents.down.sql new file mode 100644 index 0000000..ccd21cd --- /dev/null +++ b/sql/migration/0003_torrents.down.sql @@ -0,0 +1,3 @@ +DROP TABLE movie_torrents; +DROP TABLE episode_torrents; +DROP TABLE torrents_abstract; diff --git a/sql/migration/0003_torrents.up.sql b/sql/migration/0003_torrents.up.sql new file mode 100644 index 0000000..70fef4f --- /dev/null +++ b/sql/migration/0003_torrents.up.sql @@ -0,0 +1,30 @@ +CREATE TABLE torrents_abstract ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + imdb_id text NOT NULL, + url text NOT NULL, + source text NOT NULL, + quality text NOT NULL, + upload_user text NOT NULL, + seeders integer NOT NULL, + leechers integer NOT NULL, + LIKE base INCLUDING DEFAULTS +); + +CREATE TABLE movie_torrents ( +) +INHERITS (torrents_abstract); + +CREATE TABLE episode_torrents ( + season integer NOT NULL, + episode integer NOT NULL +) +INHERITS (torrents_abstract); + +CREATE INDEX ON movie_torrents (imdb_id); +CREATE UNIQUE INDEX ON movie_torrents (imdb_id, source, quality); +CREATE INDEX ON episode_torrents (imdb_id); +CREATE INDEX ON episode_torrents (imdb_id, season, episode); +CREATE INDEX ON torrents_abstract (imdb_id); + +CREATE TRIGGER update_movie_torrents_updated_at BEFORE UPDATE ON movie_torrents FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +CREATE TRIGGER update_episode_torrents_updated_at BEFORE UPDATE ON episode_torrents FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); diff --git a/src/internal/config/canape.go b/src/internal/config/canape.go index 43dc397..8a76564 100644 --- a/src/internal/config/canape.go +++ b/src/internal/config/canape.go @@ -6,6 +6,7 @@ import ( polochon "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/tmdb" + "github.com/odwrtw/polochon/modules/yts" "gopkg.in/yaml.v2" ) @@ -17,9 +18,10 @@ type Config struct { TraktTVClientID string `yaml:"trakttv_client_id"` PublicDir string `yaml:"public_dir"` - // TODO improve the detailers configurations - TmdbAPIKey string `yaml:"tmdb_api_key"` - MovieDetailers []polochon.Detailer + // TODO improve the detailers and torrenters configurations + TmdbAPIKey string `yaml:"tmdb_api_key"` + MovieDetailers []polochon.Detailer + MovieTorrenters []polochon.Torrenter } type AuthorizerConfig struct { @@ -47,6 +49,7 @@ func Load(path string) (*Config, error) { return nil, err } + // Default detailers cf.MovieDetailers = []polochon.Detailer{} if cf.TmdbAPIKey != "" { d, err := tmdb.New(&tmdb.Params{cf.TmdbAPIKey}) @@ -56,5 +59,13 @@ func Load(path string) (*Config, error) { cf.MovieDetailers = append(cf.MovieDetailers, d) } + // Default torrenters + cf.MovieTorrenters = []polochon.Torrenter{} + d, err := yts.New() + if err != nil { + return nil, err + } + cf.MovieTorrenters = append(cf.MovieTorrenters, d) + return cf, nil } diff --git a/src/internal/external_medias/external_medias.go b/src/internal/external_medias/external_medias.go index 2cbe7ba..ba94b0a 100644 --- a/src/internal/external_medias/external_medias.go +++ b/src/internal/external_medias/external_medias.go @@ -2,12 +2,18 @@ package extmedias import ( "github.com/jmoiron/sqlx" + "github.com/lib/pq" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" ) const ( - addExternalMediaQuery = `INSERT INTO external_medias (type, source, category, ids) VALUES ($1, $2, $3, $4) RETURNING id;` - updateExternalMediaQuery = `UPDATE external_medias SET type=:type, source=:source, category=:category, ids=:ids WHERE id=:id RETURNING *;` + // addExternalMediaQuery = `INSERT INTO external_medias (type, source, category, ids) VALUES ($1, $2, $3, $4) RETURNING id;` + upsertExternalMediaQuery = ` + INSERT INTO external_medias (type, source, category, ids) + VALUES (:type, :source, :category, :ids) + ON CONFLICT (type, source, category) + DO UPDATE SET type=:type, source=:source, category=:category, ids=:ids + RETURNING id;` deleteExternalMediaQuery = `DELETE FROM external_medias WHERE id=:id;` getExternalMediaQuery = `SELECT * FROM external_medias WHERE type=$1 AND source=$2 AND category=$3 LIMIT 1;` @@ -16,45 +22,36 @@ const ( // Media represents an external media type Media struct { sqly.BaseModel - Type string `db:"type"` - Source string `db:"source"` - Category string `db:"category"` - IDs sqly.StringSlice `db:"ids"` + Type string `db:"type"` + Source string `db:"source"` + Category string `db:"category"` + IDs pq.StringArray `db:"ids"` } -// Add adds the Media in the database -func (m *Media) Add(q sqlx.Queryer) error { +// Upsert adds or updates or adds the Media in the database +func (m *Media) Upsert(db *sqlx.DB) error { var id string - err := q.QueryRowx(addExternalMediaQuery, m.Type, m.Source, m.Category, m.IDs).Scan(&id) + r, err := db.NamedQuery(upsertExternalMediaQuery, m) if err != nil { return err } + for r.Next() { + r.Scan(&id) + } m.ID = id return nil } -// Update ids only updates the IDs of the media -func (m *Media) Update(ex *sqlx.DB) error { - rows, err := ex.NamedQuery(updateExternalMediaQuery, m) - if err != nil { - return err - } - for rows.Next() { - rows.StructScan(m) - } - return nil -} - // Delete the media from database or raise an error -func (m *Media) Delete(ex *sqlx.DB) error { - _, err := ex.NamedExec(deleteExternalMediaQuery, m) +func (m *Media) Delete(db *sqlx.DB) error { + _, err := db.NamedExec(deleteExternalMediaQuery, m) return err } // Get gets a media -func Get(q sqlx.Queryer, mtype, msrc, mcat string) (*Media, error) { +func Get(db *sqlx.DB, mtype, msrc, mcat string) (*Media, error) { m := &Media{} - if err := q.QueryRowx(getExternalMediaQuery, mtype, msrc, mcat).StructScan(m); err != nil { + if err := db.QueryRowx(getExternalMediaQuery, mtype, msrc, mcat).StructScan(m); err != nil { return nil, err } diff --git a/src/internal/external_medias/external_medias_test.go b/src/internal/external_medias/external_medias_test.go index 2651ba3..519823a 100644 --- a/src/internal/external_medias/external_medias_test.go +++ b/src/internal/external_medias/external_medias_test.go @@ -37,13 +37,13 @@ func TestAddExternalMedias(t *testing.T) { } // Add it - if err := media.Add(db); err != nil { + if err := media.Upsert(db); err != nil { t.Fatalf("failed to add external media: %q", err) } // Update the IDs media.IDs = []string{"1", "2", "3", "4"} - if err := media.Update(db); err != nil { + if err := media.Upsert(db); err != nil { t.Fatalf("failed to update the external media: %q", err) } diff --git a/src/internal/external_medias/handlers.go b/src/internal/external_medias/handlers.go new file mode 100644 index 0000000..bbe6c0a --- /dev/null +++ b/src/internal/external_medias/handlers.go @@ -0,0 +1,171 @@ +package extmedias + +import ( + "database/sql" + "fmt" + "net/http" + + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies" + + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" + + "github.com/Sirupsen/logrus" + polochon "github.com/odwrtw/polochon/lib" + traktExplorer "github.com/odwrtw/polochon/modules/trakttv" + ytsExplorer "github.com/odwrtw/polochon/modules/yts" +) + +// GetMediaIDs will get some media IDs +func GetMediaIDs(env *web.Env, mediaType string, source string, category string, force bool) ([]string, error) { + log := env.Log.WithFields(logrus.Fields{ + "mediaType": mediaType, + "source": source, + "category": category, + "function": "extmedias.GetMediaIds", + }) + + var err error + media, err := Get(env.Database, mediaType, source, category) + switch err { + case nil: + log.Debug("medias found in database") + if !force { + log.Debug("returning medias from db") + return media.IDs, nil + } + case sql.ErrNoRows: + log.Debug("medias not found in database") + default: + // Unexpected error + return nil, err + } + + explorer, err := NewExplorer(env, source) + if err != nil { + return nil, err + } + + var ids []string + movies, err := explorer.GetMovieList(polochon.ExploreByRate, log) + for _, movie := range movies { + ids = append(ids, movie.ImdbID) + } + + log.Debugf("got %d medias from %s", len(ids), source) + + media = &Media{ + Type: mediaType, + Source: source, + Category: category, + IDs: ids, + } + + log.Debugf("Inserting int to DB %+v", media) + err = media.Upsert(env.Database) + if err != nil { + return nil, err + } + + log.Debug("medias updated in database") + + return ids, nil +} + +// GetMedias get some movies +func GetMedias(env *web.Env, source string, category string, force bool) ([]*movies.Movie, error) { + movieIds, err := GetMediaIDs(env, "movie", source, category, force) + if err != nil { + return nil, err + } + + movieList := []*movies.Movie{} + for _, id := range movieIds { + movie := movies.New(id) + err := movie.GetDetails(env, force) + if err != nil { + env.Log.Error(err) + continue + } + err = movie.GetTorrents(env, force) + if err != nil { + env.Log.Error(err) + continue + } + movieList = append(movieList, movie) + } + return movieList, nil +} + +// Explore will explore some movies +func Explore(env *web.Env, w http.ResponseWriter, r *http.Request) error { + err := r.ParseForm() + if err != nil { + return err + } + + source := r.FormValue("source") + // Default source + if source == "" { + source = "yts" + } + + category := r.FormValue("category") + // Default category + if category == "" { + category = "popular" + } + + // Get the medias without trying to refresh them + movies, err := GetMedias(env, source, category, false) + if err != nil { + return err + } + + return env.RenderJSON(w, movies) +} + +// MediaSources represents the implemented media sources +var MediaSources = []string{ + "trakttv", + "yts", +} + +// Refresh will refresh the movie list +func Refresh(env *web.Env, w http.ResponseWriter, r *http.Request) error { + env.Log.Debugf("refreshing infos ...") + source := r.FormValue("source") + if source == "" { + source = "yts" + } + + category := r.FormValue("category") + if category == "" { + category = "popular" + } + + // We'll refresh the medias for each sources + for _, source := range MediaSources { + env.Log.Debugf("refreshing %s", source) + // GetMedias and refresh them + _, err := GetMedias(env, source, category, true) + if err != nil { + return err + } + } + + return env.RenderJSON(w, map[string]string{"message": "Refresh is done"}) +} + +// NewExplorer returns a polochon.Explorer +func NewExplorer(env *web.Env, source string) (polochon.Explorer, error) { + switch source { + case "trakttv": + return traktExplorer.NewExplorer(&traktExplorer.Params{ + ClientID: env.Config.TraktTVClientID, + }) + case "yts": + return ytsExplorer.NewExplorer() + default: + return nil, fmt.Errorf("unknown explorer") + } +} diff --git a/src/internal/movies/explorer.go b/src/internal/movies/explorer.go deleted file mode 100644 index 6c38820..0000000 --- a/src/internal/movies/explorer.go +++ /dev/null @@ -1,7 +0,0 @@ -package movies - -import "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" - -func updatePopular(env *web.Env, clientID string) error { - return nil -} diff --git a/src/internal/movies/handlers.go b/src/internal/movies/handlers.go index 08c14e8..be12419 100644 --- a/src/internal/movies/handlers.go +++ b/src/internal/movies/handlers.go @@ -10,7 +10,6 @@ import ( "github.com/odwrtw/papi" polochon "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/pam" - "github.com/odwrtw/trakttv" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config" @@ -63,6 +62,7 @@ func getPolochonMovies(user *users.User) ([]*Movie, error) { return movies, nil } +// FromPolochon will returns movies from Polochon func FromPolochon(env *web.Env, w http.ResponseWriter, r *http.Request) error { v := auth.GetCurrentUser(r, env.Log) user, ok := v.(*users.User) @@ -97,45 +97,6 @@ func FromPolochon(env *web.Env, w http.ResponseWriter, r *http.Request) error { return env.RenderJSON(w, movies) } -// ExplorePopular returns the popular movies -func ExplorePopular(env *web.Env, w http.ResponseWriter, r *http.Request) error { - log := env.Log.WithField("function", "movies.ExplorePopular") - - queryOption := trakttv.QueryOption{ - ExtendedInfos: []trakttv.ExtendedInfo{ - trakttv.ExtendedInfoMin, - }, - Pagination: trakttv.Pagination{ - Page: 1, - Limit: 20, - }, - } - trakt := trakttv.New(env.Config.TraktTVClientID) - trakt.Endpoint = trakttv.ProductionEndpoint - - log.Debug("getting movies from trakttv") - tmovies, err := trakt.PopularMovies(queryOption) - if err != nil { - return err - } - log.Debugf("got %d movies from trakttv", len(tmovies)) - - movies := []*Movie{} - ids := []string{} - for _, m := range tmovies { - movie := New(m.IDs.ImDB) - err := movie.GetDetails(env, false) - if err != nil { - env.Log.Error(err) - continue - } - movies = append(movies, movie) - ids = append(ids, m.IDs.ImDB) - } - - return env.RenderJSON(w, movies) -} - // GetDetailsHandler retrieves details for a movie func GetDetailsHandler(env *web.Env, w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) diff --git a/src/internal/movies/movies.go b/src/internal/movies/movies.go index 92148ef..29114d9 100644 --- a/src/internal/movies/movies.go +++ b/src/internal/movies/movies.go @@ -1,51 +1,106 @@ package movies import ( + "database/sql" "fmt" "os" "path/filepath" + "time" "github.com/Sirupsen/logrus" "github.com/jmoiron/sqlx" + "github.com/lib/pq" "github.com/odwrtw/polochon/lib" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/torrents" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" ) const ( - addMovieQuery = ` - INSERT INTO movies (imdb_id, title, rating, votes, plot, tmdb_id, year, original_title, runtime, sort_title, tagline) - VALUES (:imdbid, :title, :rating, :votes, :plot, :tmdbid, :year, :originaltitle, :runtime, :sorttitle, :tagline) + upsertMovieQuery = ` + INSERT INTO movies (imdb_id, title, rating, votes, plot, tmdb_id, year, + genres, original_title, runtime, sort_title, tagline) + VALUES (:imdb_id, :title, :rating, :votes, :plot, :tmdb_id, :year, :genres, + :original_title, :runtime, :sort_title, :tagline) + ON CONFLICT (imdb_id) + DO UPDATE + SET imdb_id=:imdb_id, title=:title, rating=:rating, votes=:votes, + plot=:plot, tmdb_id=:tmdb_id, year=:year, genres=:genres, + original_title=:original_title, runtime=:runtime, sort_title=:sort_title, + tagline=:tagline RETURNING id;` - updateMovieQuery = ` - UPDATE movies - SET imdb_id=:imdbid, title=:title, rating=:rating, votes=:votes, plot=:plot, tmdb_id=:tmdbid, year=:year, original_title=:originaltitle, runtime=:runtime, sort_title=:sorttitle, tagline=:tagline - WHERE ID = :id;` - getMovieQueryByImdbID = ` - SELECT - id, imdb_id AS imdbid, title, rating, votes, plot, - tmdb_id AS tmdbid, year, - original_title AS originaltitle, runtime, - sort_title AS sorttitle, tagline, - created_at, updated_at + SELECT * FROM movies WHERE imdb_id=$1;` + getMovieQueryByID = ` - SELECT - id, imdb_id AS imdbid, title, rating, votes, plot, - tmdb_id AS tmdbid, year, - original_title AS originaltitle, runtime, - sort_title AS sorttitle, tagline, - created_at, updated_at + SELECT * FROM movies WHERE id=$1;` + deleteMovieQuery = `DELETE FROM movies WHERE id=$1;` ) -var ( - // ErrNotFound error returned when show not found in database - ErrNotFound = fmt.Errorf("Not found") -) +// MovieDB represents the Movie in the DB +type MovieDB struct { + ID string `db:"id"` + ImdbID string `db:"imdb_id"` + TmdbID int `db:"tmdb_id"` + Title string `db:"title"` + OriginalTitle string `db:"original_title"` + SortTitle string `db:"sort_title"` + Rating float32 `db:"rating"` + Votes int `db:"votes"` + Plot string `db:"plot"` + Year int `db:"year"` + Runtime int `db:"runtime"` + Tagline string `db:"tagline"` + Genres pq.StringArray `db:"genres"` + Created time.Time `db:"created_at"` + Updated time.Time `db:"updated_at"` +} + +// NewMovieDB returns a Movie ready to be put in DB from a +// Movie +func NewMovieDB(m *Movie) MovieDB { + return MovieDB{ + ID: m.ID, + ImdbID: m.ImdbID, + Title: m.Title, + Rating: m.Rating, + Votes: m.Votes, + Plot: m.Plot, + TmdbID: m.TmdbID, + Year: m.Year, + OriginalTitle: m.OriginalTitle, + Runtime: m.Runtime, + SortTitle: m.SortTitle, + Tagline: m.Tagline, + Genres: m.Genres, + Created: m.Created, + Updated: m.Updated, + } +} + +// FillFromDB returns a Movie from a MovieDB extracted from the DB +func (m *Movie) FillFromDB(mDB *MovieDB) { + m.Created = mDB.Created + m.Updated = mDB.Updated + m.ID = mDB.ID + m.ImdbID = mDB.ImdbID + m.Title = mDB.Title + m.Rating = mDB.Rating + m.Votes = mDB.Votes + m.Plot = mDB.Plot + m.TmdbID = mDB.TmdbID + m.Year = mDB.Year + m.OriginalTitle = mDB.OriginalTitle + m.Runtime = mDB.Runtime + m.Genres = mDB.Genres + m.SortTitle = mDB.SortTitle + m.Tagline = mDB.Tagline +} // Movie represents a movie type Movie struct { @@ -55,6 +110,7 @@ type Movie struct { PosterURL string `json:"poster_url"` } +// New returns a new Movie with an ImDB id func New(imdbID string) *Movie { return &Movie{ Movie: polochon.Movie{ @@ -65,24 +121,23 @@ func New(imdbID string) *Movie { // Get returns show details in database from id or imdbid or an error func (m *Movie) Get(env *web.Env) error { + var mDB MovieDB var err error if m.ID != "" { - err = env.Database.QueryRowx(getMovieQueryByID, m.ID).StructScan(m) + err = env.Database.QueryRowx(getMovieQueryByID, m.ID).StructScan(&mDB) } else if m.ImdbID != "" { - err = env.Database.QueryRowx(getMovieQueryByImdbID, m.ImdbID).StructScan(m) + err = env.Database.QueryRowx(getMovieQueryByImdbID, m.ImdbID).StructScan(&mDB) } else { err = fmt.Errorf("Can't get movie details, you have to specify an ID or ImdbID") } if err != nil { - if err.Error() == "sql: no rows in result set" { - return ErrNotFound - } return err } // Set the poster url m.PosterURL = m.GetPosterURL(env) + m.FillFromDB(&mDB) return nil } @@ -93,37 +148,33 @@ func (m *Movie) Get(env *web.Env) error { // If force is used, the detailer will be used even if the movie is found in // database func (m *Movie) GetDetails(env *web.Env, force bool) error { - if len(m.Detailers) == 0 { - m.Detailers = env.Config.MovieDetailers - } - log := env.Log.WithFields(logrus.Fields{ "imdb_id": m.ImdbID, "function": "movies.GetDetails", }) + log.Debugf("getting details") - // If the movie is not in db, we should add it, otherwise we should update - // it - var dbFunc func(db *sqlx.DB) error + if len(m.Detailers) == 0 { + m.Detailers = env.Config.MovieDetailers + } var err error err = m.Get(env) switch err { case nil: log.Debug("movie found in database") - dbFunc = m.Update if !force { + log.Debug("returning movie from db") return nil } - case ErrNotFound: - dbFunc = m.Add + case sql.ErrNoRows: log.Debug("movie not found in database") default: // Unexpected error return err } - // so we got ErrNotFound so GetDetails from a detailer + // GetDetail err = m.Movie.GetDetails(env.Log) if err != nil { return err @@ -131,8 +182,9 @@ func (m *Movie) GetDetails(env *web.Env, force bool) error { log.Debug("got details from detailers") - err = dbFunc(env.Database) + err = m.Upsert(env.Database) if err != nil { + log.Debug("error while doing db func") return err } @@ -152,10 +204,67 @@ func (m *Movie) GetDetails(env *web.Env, force bool) error { return nil } -// Add a movie in the database -func (m *Movie) Add(db *sqlx.DB) error { +// GetTorrents retrieves torrents for the movie, first try to get info from db, +// if not exists, use polochon.Torrenter and save informations in the database +// for future use +// +// If force is used, the torrenter will be used even if the torrent is found in +// database +func (m *Movie) GetTorrents(env *web.Env, force bool) error { + log := env.Log.WithFields(logrus.Fields{ + "imdb_id": m.ImdbID, + "function": "movies.GetTorrents", + }) + log.Debugf("getting torrents") + + if len(m.Torrenters) == 0 { + m.Torrenters = env.Config.MovieTorrenters + } + + movieTorrents, err := torrents.GetMovieTorrents(env.Database, m.ImdbID) + switch err { + case nil: + log.Debug("torrents found in database") + if !force { + log.Debugf("returning %d torrents from db", len(movieTorrents)) + // Add the torrents to the movie + for _, t := range movieTorrents { + m.Torrents = append(m.Torrents, t.Torrent) + } + return nil + } + case sql.ErrNoRows: + log.Debug("torrent not found in database") + // We'll need to GetTorrents from torrenters + default: + // Unexpected error + return err + } + + err = m.Movie.GetTorrents(env.Log) + if err != nil { + return err + } + + log.Debugf("got %d torrents from torrenters", len(m.Movie.Torrents)) + + for _, t := range m.Movie.Torrents { + torrent := torrents.NewMovie(m.ImdbID, t) + err = torrent.Upsert(env.Database) + if err != nil { + log.Error("error while adding torrent", err) + continue + } + } + + return nil +} + +// Upsert a movie in the database +func (m *Movie) Upsert(db *sqlx.DB) error { + mDB := NewMovieDB(m) var id string - r, err := db.NamedQuery(addMovieQuery, m) + r, err := db.NamedQuery(upsertMovieQuery, mDB) if err != nil { return err } @@ -167,12 +276,6 @@ func (m *Movie) Add(db *sqlx.DB) error { return nil } -// Update a movie in the database -func (m *Movie) Update(db *sqlx.DB) error { - _, err := db.NamedQuery(updateMovieQuery, m) - return err -} - // Delete movie from database func (m *Movie) Delete(db *sqlx.DB) error { r, err := db.Exec(deleteMovieQuery, m.ID) diff --git a/src/internal/movies/movies_test.go b/src/internal/movies/movies_test.go index d2797b1..d92fe39 100644 --- a/src/internal/movies/movies_test.go +++ b/src/internal/movies/movies_test.go @@ -1,6 +1,7 @@ package movies import ( + "database/sql" "fmt" "os" "testing" @@ -11,7 +12,9 @@ import ( _ "github.com/mattes/migrate/driver/postgres" "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/mock" - "gitlab.quimbo.fr/odwrtw/canape-sql/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" ) var db *sqlx.DB @@ -31,6 +34,15 @@ func init() { func TestIntegrate(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { + log := logrus.NewEntry(logrus.New()) + env := web.NewEnv(web.EnvParams{ + Database: db, + // Auth: authorizer, + Log: log, + Config: &config.Config{ + PublicDir: "/tmp", + }, + }) detailer, _ := mock.NewDetailer(nil) polochonConfig := polochon.MovieConfig{ Detailers: []polochon.Detailer{ @@ -41,11 +53,11 @@ func TestIntegrate(t *testing.T) { Movie: polochon.Movie{ MovieConfig: polochonConfig, ImdbID: "tt12345", + Genres: []string{}, }, } - log := logrus.NewEntry(logrus.New()) - err := movie.GetDetails(db, log) + err := movie.GetDetails(env, false) if err != nil { t.Fatal(err) } @@ -57,7 +69,7 @@ func TestIntegrate(t *testing.T) { ImdbID: "tt12345", }, } - err = movie.Get(db) + err = movie.Get(env) if err != nil { t.Fatal(err) } @@ -72,8 +84,8 @@ func TestIntegrate(t *testing.T) { } // Get it again - err = movie.Get(db) - if err != ErrNotFound { + err = movie.Get(env) + if err != sql.ErrNoRows { t.Fatalf("Unexpected error: %q", err) } }) diff --git a/src/internal/shows/shows_test.go b/src/internal/shows/shows_test.go index b7cdba9..7c411dd 100644 --- a/src/internal/shows/shows_test.go +++ b/src/internal/shows/shows_test.go @@ -5,15 +5,16 @@ import ( "os" "testing" - "gitlab.quimbo.fr/odwrtw/canape-sql/sqly" - "gitlab.quimbo.fr/odwrtw/canape-sql/users" - "github.com/Sirupsen/logrus" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" _ "github.com/mattes/migrate/driver/postgres" "github.com/odwrtw/polochon/lib" "github.com/odwrtw/polochon/modules/mock" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/users" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" ) var db *sqlx.DB @@ -89,8 +90,40 @@ func TestIntegrate(t *testing.T) { t.Fatalf("Unexpected error: %q", err) } }) - } + +// UserBackend represents the data backend to get the user +type UserBackend struct { + Database *sqlx.DB +} + +// Get gets the username from the UserBackend +func (b *UserBackend) Get(username string) (auth.User, error) { + return users.Get(b.Database, username) +} + +func getEnv(db *sqlx.DB) *web.Env { + uBackend := &UserBackend{Database: db} + + authParams := auth.Params{ + Backend: uBackend, + Pepper: "pepper", + Cost: 10, + Secret: "secret", + } + authorizer := auth.New(authParams) + + log := logrus.NewEntry(logrus.New()) + env := web.NewEnv(web.EnvParams{ + Database: db, + Auth: authorizer, + Log: log, + // Config: cf, + }) + + return env +} + func TestTrackedShow(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { detailer, _ := mock.NewDetailer(nil) @@ -113,6 +146,18 @@ func TestTrackedShow(t *testing.T) { } u := &users.User{Name: "plop"} + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("pass") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + err = u.Add(db) if err != nil { t.Fatal(err) diff --git a/src/internal/sqly/string_slice.go b/src/internal/sqly/string_slice.go deleted file mode 100644 index 14bada0..0000000 --- a/src/internal/sqly/string_slice.go +++ /dev/null @@ -1,58 +0,0 @@ -package sqly - -import ( - "database/sql/driver" - "encoding/csv" - "errors" - "regexp" - "strings" -) - -// This mainly comes from https://gist.github.com/adharris/4163702 - -// StringSlice represents an array of string. The custom type is needed because -// pq does not support slices yet.. -type StringSlice []string - -// for replacing escaped quotes except if it is preceded by a literal backslash -// eg "\\" should translate to a quoted element whose value is \ - -var quoteEscapeRegex = regexp.MustCompile(`([^\\]([\\]{2})*)\\"`) - -// Scan convert to a slice of strings -// http://www.postgresql.org/docs/9.1/static/arrays.html#ARRAYS-IO -func (s *StringSlice) Scan(src interface{}) error { - asBytes, ok := src.([]byte) - if !ok { - return error(errors.New("Scan source was not []bytes")) - } - str := string(asBytes) - - // change quote escapes for csv parser - str = quoteEscapeRegex.ReplaceAllString(str, `$1""`) - str = strings.Replace(str, `\\`, `\`, -1) - // remove braces - str = str[1 : len(str)-1] - csvReader := csv.NewReader(strings.NewReader(str)) - - slice, err := csvReader.Read() - - if err != nil { - return err - } - - (*s) = StringSlice(slice) - - return nil -} - -// Value implements the Valuer interface -func (s StringSlice) Value() (driver.Value, error) { - // string escapes. - // \ => \\\ - // " => \" - for i, elem := range s { - s[i] = `"` + strings.Replace(strings.Replace(elem, `\`, `\\\`, -1), `"`, `\"`, -1) + `"` - } - return "{" + strings.Join(s, ",") + "}", nil -} diff --git a/src/internal/torrents/torrents.go b/src/internal/torrents/torrents.go new file mode 100644 index 0000000..548c430 --- /dev/null +++ b/src/internal/torrents/torrents.go @@ -0,0 +1,180 @@ +package torrents + +import ( + "database/sql" + "fmt" + "time" + + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + + "github.com/jmoiron/sqlx" + polochon "github.com/odwrtw/polochon/lib" +) + +const ( + upsertMovieTorrentQuery = ` + INSERT INTO movie_torrents (imdb_id, url, source, quality, upload_user, + seeders, leechers) + VALUES (:imdb_id, :url, :source, :quality, :upload_user, :seeders, + :leechers) + ON CONFLICT (imdb_id, quality, source) + DO UPDATE SET imdb_id=:imdb_id, url=:url, source=:source, quality=:quality, + upload_user=:upload_user, seeders=:seeders, leechers=:leechers + RETURNING id;` + + getMovieTorrentQueryByImdbID = ` + SELECT * + FROM movie_torrents WHERE imdb_id=$1;` + + getMovieTorrentQueryByID = ` + SELECT * + FROM movie_torrents WHERE id=$1;` + + deleteMovieTorrentQuery = `DELETE FROM movie_torrents WHERE id=$1;` +) + +// MovieTorrent represents a movie torrent +type MovieTorrent struct { + sqly.BaseModel + polochon.Torrent + ImdbID string `json:"imdb_id"` +} + +// MovieTorrentDB represents the MovieTorrent in the DB +type MovieTorrentDB struct { + ID string `db:"id"` + ImdbID string `db:"imdb_id"` + URL string `db:"url"` + Source string `db:"source"` + Quality string `db:"quality"` + UploadUser string `db:"upload_user"` + Seeders int `db:"seeders"` + Leechers int `db:"leechers"` + Created time.Time `db:"created_at"` + Updated time.Time `db:"updated_at"` +} + +// EpisodeTorrent represents an episode torrent +type EpisodeTorrent struct { + sqly.BaseModel + polochon.Torrent + ShowImdbID string `json:"show_imdb_id"` + Season int `json:"season"` + Episode int `json:"episode"` +} + +// NewMovieFromImDB returns a new MovieTorrent with an ImDB id +func NewMovieFromImDB(imdbID string) *MovieTorrent { + return &MovieTorrent{ + ImdbID: imdbID, + } +} + +// NewMovie returns a new MovieTorrent with an ImDB id +func NewMovie(imdbID string, poTo polochon.Torrent) *MovieTorrent { + return &MovieTorrent{ + ImdbID: imdbID, + Torrent: poTo, + } +} + +// Get returns show details in database from id or imdbid or an error +func (m *MovieTorrent) Get(db *sqlx.DB) error { + var mDB MovieTorrentDB + var err error + if m.ID != "" { + err = db.QueryRowx(getMovieTorrentQueryByID, m.ID).StructScan(&mDB) + } else if m.ImdbID != "" { + err = db.QueryRowx(getMovieTorrentQueryByImdbID, m.ImdbID).StructScan(&mDB) + } else { + err = fmt.Errorf("can't get movie torrent details, you have to specify an ID or ImdbID") + } + if err != nil { + return err + } + m.FillFromDB(&mDB) + return nil +} + +// GetMovieTorrents returns show details in database from id or imdbid or an error +func GetMovieTorrents(db *sqlx.DB, imdbID string) ([]*MovieTorrent, error) { + var torrentsDB = []*MovieTorrentDB{} + err := db.Select(&torrentsDB, getMovieTorrentQueryByImdbID, imdbID) + if err != nil { + return nil, err + } + + if len(torrentsDB) == 0 { + return nil, sql.ErrNoRows + } + + var torrents []*MovieTorrent + for _, torrentDB := range torrentsDB { + movie := NewMovieFromImDB(imdbID) + movie.FillFromDB(torrentDB) + torrents = append(torrents, movie) + } + + return torrents, nil +} + +// Upsert a movie torrent in the database +func (m *MovieTorrent) Upsert(db *sqlx.DB) error { + mDB := NewMovieTorrentDB(m) + var id string + r, err := db.NamedQuery(upsertMovieTorrentQuery, mDB) + if err != nil { + return err + } + for r.Next() { + r.Scan(&id) + } + m.ID = id + + return nil +} + +// Delete movie from database +func (m *MovieTorrent) Delete(db *sqlx.DB) error { + r, err := db.Exec(deleteMovieTorrentQuery, m.ID) + if err != nil { + return err + } + count, _ := r.RowsAffected() + if count != 1 { + return fmt.Errorf("Unexpected number of row deleted: %d", count) + } + return nil +} + +// NewMovieTorrentDB returns a MovieTorrent ready to be put in DB from a +// MovieTorrent +func NewMovieTorrentDB(m *MovieTorrent) MovieTorrentDB { + return MovieTorrentDB{ + ID: m.ID, + ImdbID: m.ImdbID, + URL: m.URL, + Source: m.Source, + Quality: string(m.Quality), + UploadUser: m.UploadUser, + Seeders: m.Seeders, + Leechers: m.Leechers, + Created: m.Created, + Updated: m.Updated, + } +} + +// FillFromDB will fill a MovieTorrent from a MovieTorrentDB extracted from the DB +func (m *MovieTorrent) FillFromDB(mDB *MovieTorrentDB) { + q, _ := polochon.StringToQuality(mDB.Quality) + m.ImdbID = mDB.ImdbID + m.Created = mDB.Created + m.Updated = mDB.Updated + m.ID = mDB.ID + m.URL = mDB.URL + m.Source = mDB.Source + m.Quality = *q + m.UploadUser = mDB.UploadUser + m.Seeders = mDB.Seeders + m.Leechers = mDB.Leechers +} diff --git a/src/internal/users/users.go b/src/internal/users/users.go index 30b7d25..80aa56b 100644 --- a/src/internal/users/users.go +++ b/src/internal/users/users.go @@ -80,7 +80,7 @@ func (u *User) SetConfig(key string, v interface{}) error { // NewConfig creates a new empty config func (u *User) NewConfig() error { - var configMap map[string]*json.RawMessage + configMap := make(map[string]*json.RawMessage) b, err := json.Marshal(configMap) if err != nil { return err diff --git a/src/internal/users/users_test.go b/src/internal/users/users_test.go index 385c883..b1add23 100644 --- a/src/internal/users/users_test.go +++ b/src/internal/users/users_test.go @@ -5,8 +5,11 @@ import ( "os" "testing" - "gitlab.quimbo.fr/odwrtw/canape-sql/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" + "github.com/Sirupsen/logrus" "github.com/jmoiron/sqlx" "github.com/lib/pq" _ "github.com/mattes/migrate/driver/postgres" @@ -32,7 +35,19 @@ func TestUser(t *testing.T) { // Add a new user u := &User{Name: "plop", Hash: "plop"} - err := u.Add(db) + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("test") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + err = u.Add(db) if err != nil { t.Fatal(err) } @@ -99,7 +114,19 @@ func TestTokenAddDelete(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { // Add a new user u := &User{Name: "plop", Hash: "plop"} - err := u.Add(db) + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("test") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + err = u.Add(db) if err != nil { t.Fatal(err) } @@ -155,11 +182,60 @@ func TestTokenAddDelete(t *testing.T) { }) } +// UserBackend represents the data backend to get the user +type UserBackend struct { + Database *sqlx.DB +} + +// Get gets the username from the UserBackend +func (b *UserBackend) Get(username string) (auth.User, error) { + return Get(b.Database, username) +} + +func getEnv(db *sqlx.DB) *web.Env { + uBackend := &UserBackend{Database: db} + + authParams := auth.Params{ + Backend: uBackend, + Pepper: "pepper", + Cost: 10, + Secret: "secret", + } + authorizer := auth.New(authParams) + + log := logrus.NewEntry(logrus.New()) + env := web.NewEnv(web.EnvParams{ + Database: db, + Auth: authorizer, + Log: log, + // Config: cf, + }) + + return env +} + func TestTokenCheck(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { u := &User{Name: "plop", Hash: "plop"} - u.Add(db) + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("test") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + if err = u.Add(db); err != nil { + t.Fatal(err) + } token, err := u.NewToken(db) + if err != nil { + t.Fatal(err) + } ok, err := u.CheckToken(db, token.Value) if err != nil { @@ -183,10 +259,29 @@ func TestTokenCheck(t *testing.T) { func TestAutoUpdateCols(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { - u := &User{Name: "plop", Hash: "plop"} - u.Add(db) + u := &User{Name: "plop"} + var err error + env := getEnv(db) + + u.Hash, err = env.Auth.GenHash("pass") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + + err = u.Add(db) + if err != nil { + t.Fatal(err) + } u.Name = "toto" - u.Update(db) + err = u.Update(db) + if err != nil { + t.Fatal(err) + } if !u.Created.Before(u.Updated) { t.Fatalf("colum updated not auto updated on table users") @@ -197,9 +292,25 @@ func TestAutoUpdateCols(t *testing.T) { func TestConfig(t *testing.T) { sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) { u := &User{Name: "plop", Hash: "plop"} - u.Add(db) + env := getEnv(db) - u, err := Get(db, "plop") + var err error + u.Hash, err = env.Auth.GenHash("pass") + if err != nil { + t.Fatal(err) + } + + err = u.NewConfig() + if err != nil { + t.Fatal(err) + } + + err = u.Add(db) + if err != nil { + t.Fatal(err) + } + + u, err = Get(db, "plop") if err != nil { t.Fatal(err) } diff --git a/src/main.go b/src/main.go index f43ad06..544635d 100644 --- a/src/main.go +++ b/src/main.go @@ -4,9 +4,11 @@ import ( "net/http" "os" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config" - "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies" + "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/external_medias" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/users" "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web" @@ -74,8 +76,9 @@ func main() { env.Handle("/users/edit", users.EditHandler).WithRole(users.UserRole) env.Handle("/movies/polochon", movies.FromPolochon).WithRole(users.UserRole) - env.Handle("/movies/explore/popular", movies.ExplorePopular).WithRole(users.UserRole) env.Handle("/movies/{id:tt[0-9]+}/get_details", movies.GetDetailsHandler).WithRole(users.UserRole) + env.Handle("/movies/explore", extmedias.Explore) + env.Handle("/movies/refresh", extmedias.Refresh) n := negroni.Classic() n.Use(authMiddleware) diff --git a/src/public/js/app.js b/src/public/js/app.js index be8e084..7a0a242 100644 --- a/src/public/js/app.js +++ b/src/public/js/app.js @@ -75,7 +75,7 @@ const UserIsAuthenticated = UserAuthWrapper({ // TODO find a better way const MovieListPopular = (props) => ( - + ) const MovieListPolochon = (props) => ( diff --git a/src/public/js/components/movies/list.js b/src/public/js/components/movies/list.js index 8a1fd08..4ec4dd0 100644 --- a/src/public/js/components/movies/list.js +++ b/src/public/js/components/movies/list.js @@ -130,6 +130,13 @@ class MovieButtons extends React.Component { Download } + {this.props.movie.torrents && this.props.movie.torrents.map(function(torrent, index) { + return ( + + {torrent.quality} Torrent + + )} + )} IMDB