Merge branch 'update' into 'master'

Update

See merge request !25

Makefile

@@ -22,7 +22,7 @@ watch-go: build-prepare
 	CONFIG_FILE="./config.yml" fresh -c fresh.conf
 docker:
 	$(DOCKER_COMPOSE) up -d
-	sleep 4
+	sleep 8
 
 migration-schema: docker
 	$(MIGRATION) -path $(MIGRATION_SCHEMA) up

sql/migration/0001_initial.up.sql

@@ -85,6 +85,7 @@ CREATE TABLE movies (
     runtime integer NOT NULL,
     sort_title text NOT NULL,
     tagline text NOT NULL,
+    genres text[] NOT NULL,
     LIKE base INCLUDING DEFAULTS
 );
 CREATE INDEX ON movies (imdb_id);

sql/migration/0002_external_medias.up.sql

@@ -1,15 +1,16 @@
 CREATE TYPE media_type AS ENUM ('movie', 'show');
 CREATE TYPE media_category AS ENUM ('trending', 'popular', 'anticipated', 'box_office');
-CREATE TYPE media_source AS ENUM ('trakttv');
+CREATE TYPE media_source AS ENUM ('trakttv', 'yts');
 
 CREATE TABLE external_medias (
     id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
     type media_type NOT NULL,
     source media_source NOT NULL,
     category media_category NOT NULL,
-    ids text[][] NOT NULL,
+    ids text[] NOT NULL,
     LIKE base INCLUDING DEFAULTS
 );
 CREATE TRIGGER update_external_medias_updated_at BEFORE UPDATE ON external_medias FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+CREATE UNIQUE INDEX ON external_medias (type, source, category);
 CREATE INDEX ON external_medias (type, source, category);
 

sql/migration/0003_torrents.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE movie_torrents;
+DROP TABLE episode_torrents;
+DROP TABLE torrents_abstract;

sql/migration/0003_torrents.up.sql

@@ -0,0 +1,30 @@
+CREATE TABLE torrents_abstract (
+    id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    imdb_id text NOT NULL,
+    url text NOT NULL,
+    source text NOT NULL,
+    quality text NOT NULL,
+    upload_user text NOT NULL,
+    seeders integer NOT NULL,
+    leechers integer NOT NULL,
+    LIKE base INCLUDING DEFAULTS
+);
+
+CREATE TABLE movie_torrents (
+)
+INHERITS (torrents_abstract);
+
+CREATE TABLE episode_torrents (
+    season integer NOT NULL,
+    episode integer NOT NULL
+)
+INHERITS (torrents_abstract);
+
+CREATE INDEX ON movie_torrents (imdb_id);
+CREATE UNIQUE INDEX ON movie_torrents (imdb_id, source, quality);
+CREATE INDEX ON episode_torrents (imdb_id);
+CREATE INDEX ON episode_torrents (imdb_id, season, episode);
+CREATE INDEX ON torrents_abstract (imdb_id);
+
+CREATE TRIGGER update_movie_torrents_updated_at BEFORE UPDATE ON movie_torrents FOR EACH ROW EXECUTE PROCEDURE  update_updated_at_column();
+CREATE TRIGGER update_episode_torrents_updated_at BEFORE UPDATE ON episode_torrents FOR EACH ROW EXECUTE PROCEDURE  update_updated_at_column();

src/internal/config/canape.go

@@ -6,6 +6,7 @@ import (
 
 	polochon "github.com/odwrtw/polochon/lib"
 	"github.com/odwrtw/polochon/modules/tmdb"
+	"github.com/odwrtw/polochon/modules/yts"
 
 	"gopkg.in/yaml.v2"
 )
@@ -17,9 +18,10 @@ type Config struct {
 	TraktTVClientID string           `yaml:"trakttv_client_id"`
 	PublicDir       string           `yaml:"public_dir"`
 
-	// TODO improve the detailers configurations
+	// TODO improve the detailers and torrenters configurations
 	TmdbAPIKey      string `yaml:"tmdb_api_key"`
 	MovieDetailers  []polochon.Detailer
+	MovieTorrenters []polochon.Torrenter
 }
 
 type AuthorizerConfig struct {
@@ -47,6 +49,7 @@ func Load(path string) (*Config, error) {
 		return nil, err
 	}
 
+	// Default detailers
 	cf.MovieDetailers = []polochon.Detailer{}
 	if cf.TmdbAPIKey != "" {
 		d, err := tmdb.New(&tmdb.Params{cf.TmdbAPIKey})
@@ -56,5 +59,13 @@ func Load(path string) (*Config, error) {
 		cf.MovieDetailers = append(cf.MovieDetailers, d)
 	}
 
+	// Default torrenters
+	cf.MovieTorrenters = []polochon.Torrenter{}
+	d, err := yts.New()
+	if err != nil {
+		return nil, err
+	}
+	cf.MovieTorrenters = append(cf.MovieTorrenters, d)
+
 	return cf, nil
 }

src/internal/external_medias/external_medias.go

@@ -2,12 +2,18 @@ package extmedias
 
 import (
 	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly"
 )
 
 const (
-	addExternalMediaQuery    = `INSERT INTO external_medias (type, source, category, ids) VALUES ($1, $2, $3, $4) RETURNING id;`
-	updateExternalMediaQuery = `UPDATE external_medias SET type=:type, source=:source, category=:category, ids=:ids WHERE id=:id RETURNING *;`
+	// addExternalMediaQuery    = `INSERT INTO external_medias (type, source, category, ids) VALUES ($1, $2, $3, $4) RETURNING id;`
+	upsertExternalMediaQuery = `
+	INSERT INTO external_medias (type, source, category, ids)
+	VALUES (:type, :source, :category, :ids)
+	ON CONFLICT (type, source, category)
+	DO UPDATE SET type=:type, source=:source, category=:category, ids=:ids
+	RETURNING id;`
 
 	deleteExternalMediaQuery = `DELETE FROM external_medias WHERE id=:id;`
 	getExternalMediaQuery    = `SELECT * FROM external_medias WHERE type=$1 AND source=$2 AND category=$3 LIMIT 1;`
@@ -19,42 +25,33 @@ type Media struct {
 	Type     string         `db:"type"`
 	Source   string         `db:"source"`
 	Category string         `db:"category"`
-	IDs      sqly.StringSlice `db:"ids"`
+	IDs      pq.StringArray `db:"ids"`
 }
 
-// Add adds the Media in the database
-func (m *Media) Add(q sqlx.Queryer) error {
+// Upsert adds or updates or adds the Media in the database
+func (m *Media) Upsert(db *sqlx.DB) error {
 	var id string
-	err := q.QueryRowx(addExternalMediaQuery, m.Type, m.Source, m.Category, m.IDs).Scan(&id)
+	r, err := db.NamedQuery(upsertExternalMediaQuery, m)
 	if err != nil {
 		return err
 	}
+	for r.Next() {
+		r.Scan(&id)
+	}
 	m.ID = id
 	return nil
 }
 
-// Update ids only updates the IDs of the media
-func (m *Media) Update(ex *sqlx.DB) error {
-	rows, err := ex.NamedQuery(updateExternalMediaQuery, m)
-	if err != nil {
-		return err
-	}
-	for rows.Next() {
-		rows.StructScan(m)
-	}
-	return nil
-}
-
 // Delete the media from database or raise an error
-func (m *Media) Delete(ex *sqlx.DB) error {
-	_, err := ex.NamedExec(deleteExternalMediaQuery, m)
+func (m *Media) Delete(db *sqlx.DB) error {
+	_, err := db.NamedExec(deleteExternalMediaQuery, m)
 	return err
 }
 
 // Get gets a media
-func Get(q sqlx.Queryer, mtype, msrc, mcat string) (*Media, error) {
+func Get(db *sqlx.DB, mtype, msrc, mcat string) (*Media, error) {
 	m := &Media{}
-	if err := q.QueryRowx(getExternalMediaQuery, mtype, msrc, mcat).StructScan(m); err != nil {
+	if err := db.QueryRowx(getExternalMediaQuery, mtype, msrc, mcat).StructScan(m); err != nil {
 		return nil, err
 	}
 

src/internal/external_medias/external_medias_test.go

@@ -37,13 +37,13 @@ func TestAddExternalMedias(t *testing.T) {
 		}
 
 		// Add it
-		if err := media.Add(db); err != nil {
+		if err := media.Upsert(db); err != nil {
 			t.Fatalf("failed to add external media: %q", err)
 		}
 
 		// Update the IDs
 		media.IDs = []string{"1", "2", "3", "4"}
-		if err := media.Update(db); err != nil {
+		if err := media.Upsert(db); err != nil {
 			t.Fatalf("failed to update the external media: %q", err)
 		}
 

src/internal/external_medias/handlers.go

@@ -0,0 +1,171 @@
+package extmedias
+
+import (
+	"database/sql"
+	"fmt"
+	"net/http"
+
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies"
+
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web"
+
+	"github.com/Sirupsen/logrus"
+	polochon "github.com/odwrtw/polochon/lib"
+	traktExplorer "github.com/odwrtw/polochon/modules/trakttv"
+	ytsExplorer "github.com/odwrtw/polochon/modules/yts"
+)
+
+// GetMediaIDs will get some media IDs
+func GetMediaIDs(env *web.Env, mediaType string, source string, category string, force bool) ([]string, error) {
+	log := env.Log.WithFields(logrus.Fields{
+		"mediaType": mediaType,
+		"source":    source,
+		"category":  category,
+		"function":  "extmedias.GetMediaIds",
+	})
+
+	var err error
+	media, err := Get(env.Database, mediaType, source, category)
+	switch err {
+	case nil:
+		log.Debug("medias found in database")
+		if !force {
+			log.Debug("returning medias from db")
+			return media.IDs, nil
+		}
+	case sql.ErrNoRows:
+		log.Debug("medias not found in database")
+	default:
+		// Unexpected error
+		return nil, err
+	}
+
+	explorer, err := NewExplorer(env, source)
+	if err != nil {
+		return nil, err
+	}
+
+	var ids []string
+	movies, err := explorer.GetMovieList(polochon.ExploreByRate, log)
+	for _, movie := range movies {
+		ids = append(ids, movie.ImdbID)
+	}
+
+	log.Debugf("got %d medias from %s", len(ids), source)
+
+	media = &Media{
+		Type:     mediaType,
+		Source:   source,
+		Category: category,
+		IDs:      ids,
+	}
+
+	log.Debugf("Inserting int to DB %+v", media)
+	err = media.Upsert(env.Database)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debug("medias updated in database")
+
+	return ids, nil
+}
+
+// GetMedias get some movies
+func GetMedias(env *web.Env, source string, category string, force bool) ([]*movies.Movie, error) {
+	movieIds, err := GetMediaIDs(env, "movie", source, category, force)
+	if err != nil {
+		return nil, err
+	}
+
+	movieList := []*movies.Movie{}
+	for _, id := range movieIds {
+		movie := movies.New(id)
+		err := movie.GetDetails(env, force)
+		if err != nil {
+			env.Log.Error(err)
+			continue
+		}
+		err = movie.GetTorrents(env, force)
+		if err != nil {
+			env.Log.Error(err)
+			continue
+		}
+		movieList = append(movieList, movie)
+	}
+	return movieList, nil
+}
+
+// Explore will explore some movies
+func Explore(env *web.Env, w http.ResponseWriter, r *http.Request) error {
+	err := r.ParseForm()
+	if err != nil {
+		return err
+	}
+
+	source := r.FormValue("source")
+	// Default source
+	if source == "" {
+		source = "yts"
+	}
+
+	category := r.FormValue("category")
+	// Default category
+	if category == "" {
+		category = "popular"
+	}
+
+	// Get the medias without trying to refresh them
+	movies, err := GetMedias(env, source, category, false)
+	if err != nil {
+		return err
+	}
+
+	return env.RenderJSON(w, movies)
+}
+
+// MediaSources represents the implemented media sources
+var MediaSources = []string{
+	"trakttv",
+	"yts",
+}
+
+// Refresh will refresh the movie list
+func Refresh(env *web.Env, w http.ResponseWriter, r *http.Request) error {
+	env.Log.Debugf("refreshing infos ...")
+	source := r.FormValue("source")
+	if source == "" {
+		source = "yts"
+	}
+
+	category := r.FormValue("category")
+	if category == "" {
+		category = "popular"
+	}
+
+	// We'll refresh the medias for each sources
+	for _, source := range MediaSources {
+		env.Log.Debugf("refreshing %s", source)
+		// GetMedias and refresh them
+		_, err := GetMedias(env, source, category, true)
+		if err != nil {
+			return err
+		}
+	}
+
+	return env.RenderJSON(w, map[string]string{"message": "Refresh is done"})
+}
+
+// NewExplorer returns a polochon.Explorer
+func NewExplorer(env *web.Env, source string) (polochon.Explorer, error) {
+	switch source {
+	case "trakttv":
+		return traktExplorer.NewExplorer(&traktExplorer.Params{
+			ClientID: env.Config.TraktTVClientID,
+		})
+	case "yts":
+		return ytsExplorer.NewExplorer()
+	default:
+		return nil, fmt.Errorf("unknown explorer")
+	}
+}

src/internal/movies/explorer.go

@@ -1,7 +0,0 @@
-package movies
-
-import "gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web"
-
-func updatePopular(env *web.Env, clientID string) error {
-	return nil
-}

src/internal/movies/handlers.go

@@ -10,7 +10,6 @@ import (
 	"github.com/odwrtw/papi"
 	polochon "github.com/odwrtw/polochon/lib"
 	"github.com/odwrtw/polochon/modules/pam"
-	"github.com/odwrtw/trakttv"
 
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth"
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config"
@@ -63,6 +62,7 @@ func getPolochonMovies(user *users.User) ([]*Movie, error) {
 	return movies, nil
 }
 
+// FromPolochon will returns movies from Polochon
 func FromPolochon(env *web.Env, w http.ResponseWriter, r *http.Request) error {
 	v := auth.GetCurrentUser(r, env.Log)
 	user, ok := v.(*users.User)
@@ -97,45 +97,6 @@ func FromPolochon(env *web.Env, w http.ResponseWriter, r *http.Request) error {
 	return env.RenderJSON(w, movies)
 }
 
-// ExplorePopular returns the popular movies
-func ExplorePopular(env *web.Env, w http.ResponseWriter, r *http.Request) error {
-	log := env.Log.WithField("function", "movies.ExplorePopular")
-
-	queryOption := trakttv.QueryOption{
-		ExtendedInfos: []trakttv.ExtendedInfo{
-			trakttv.ExtendedInfoMin,
-		},
-		Pagination: trakttv.Pagination{
-			Page:  1,
-			Limit: 20,
-		},
-	}
-	trakt := trakttv.New(env.Config.TraktTVClientID)
-	trakt.Endpoint = trakttv.ProductionEndpoint
-
-	log.Debug("getting movies from trakttv")
-	tmovies, err := trakt.PopularMovies(queryOption)
-	if err != nil {
-		return err
-	}
-	log.Debugf("got %d movies from trakttv", len(tmovies))
-
-	movies := []*Movie{}
-	ids := []string{}
-	for _, m := range tmovies {
-		movie := New(m.IDs.ImDB)
-		err := movie.GetDetails(env, false)
-		if err != nil {
-			env.Log.Error(err)
-			continue
-		}
-		movies = append(movies, movie)
-		ids = append(ids, m.IDs.ImDB)
-	}
-
-	return env.RenderJSON(w, movies)
-}
-
 // GetDetailsHandler retrieves details for a movie
 func GetDetailsHandler(env *web.Env, w http.ResponseWriter, r *http.Request) error {
 	vars := mux.Vars(r)

src/internal/movies/movies.go

@@ -1,51 +1,106 @@
 package movies
 
 import (
+	"database/sql"
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
 	"github.com/odwrtw/polochon/lib"
+
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/torrents"
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web"
 )
 
 const (
-	addMovieQuery = `
-	INSERT INTO movies (imdb_id, title, rating, votes, plot, tmdb_id, year, original_title, runtime, sort_title, tagline)
-	VALUES (:imdbid, :title, :rating, :votes, :plot, :tmdbid, :year, :originaltitle, :runtime, :sorttitle, :tagline)
+	upsertMovieQuery = `
+	INSERT INTO movies (imdb_id, title, rating, votes, plot, tmdb_id, year,
+	genres, original_title, runtime, sort_title, tagline)
+	VALUES (:imdb_id, :title, :rating, :votes, :plot, :tmdb_id, :year, :genres,
+	:original_title, :runtime, :sort_title, :tagline)
+	ON CONFLICT (imdb_id)
+	DO UPDATE
+	SET imdb_id=:imdb_id, title=:title, rating=:rating, votes=:votes,
+	plot=:plot, tmdb_id=:tmdb_id, year=:year, genres=:genres,
+	original_title=:original_title, runtime=:runtime, sort_title=:sort_title,
+	tagline=:tagline
 	RETURNING id;`
 
-	updateMovieQuery = `
-	UPDATE movies
-	SET imdb_id=:imdbid, title=:title, rating=:rating, votes=:votes, plot=:plot, tmdb_id=:tmdbid, year=:year, original_title=:originaltitle, runtime=:runtime, sort_title=:sorttitle, tagline=:tagline
-	WHERE ID = :id;`
-
 	getMovieQueryByImdbID = `
-	SELECT
-	id, imdb_id AS imdbid, title, rating, votes, plot,
-	tmdb_id AS tmdbid, year,
-	original_title AS originaltitle, runtime,
-	sort_title AS sorttitle, tagline,
-	created_at, updated_at
+	SELECT *
 	FROM movies WHERE imdb_id=$1;`
+
 	getMovieQueryByID = `
-	SELECT
-	id, imdb_id AS imdbid, title, rating, votes, plot,
-	tmdb_id AS tmdbid, year,
-	original_title AS originaltitle, runtime,
-	sort_title AS sorttitle, tagline,
-	created_at, updated_at
+	SELECT *
 	FROM movies WHERE id=$1;`
+
 	deleteMovieQuery = `DELETE FROM movies WHERE id=$1;`
 )
 
-var (
-	// ErrNotFound error returned when show not found in database
-	ErrNotFound = fmt.Errorf("Not found")
-)
+// MovieDB represents the Movie in the DB
+type MovieDB struct {
+	ID            string         `db:"id"`
+	ImdbID        string         `db:"imdb_id"`
+	TmdbID        int            `db:"tmdb_id"`
+	Title         string         `db:"title"`
+	OriginalTitle string         `db:"original_title"`
+	SortTitle     string         `db:"sort_title"`
+	Rating        float32        `db:"rating"`
+	Votes         int            `db:"votes"`
+	Plot          string         `db:"plot"`
+	Year          int            `db:"year"`
+	Runtime       int            `db:"runtime"`
+	Tagline       string         `db:"tagline"`
+	Genres        pq.StringArray `db:"genres"`
+	Created       time.Time      `db:"created_at"`
+	Updated       time.Time      `db:"updated_at"`
+}
+
+// NewMovieDB returns a Movie ready to be put in DB from a
+// Movie
+func NewMovieDB(m *Movie) MovieDB {
+	return MovieDB{
+		ID:            m.ID,
+		ImdbID:        m.ImdbID,
+		Title:         m.Title,
+		Rating:        m.Rating,
+		Votes:         m.Votes,
+		Plot:          m.Plot,
+		TmdbID:        m.TmdbID,
+		Year:          m.Year,
+		OriginalTitle: m.OriginalTitle,
+		Runtime:       m.Runtime,
+		SortTitle:     m.SortTitle,
+		Tagline:       m.Tagline,
+		Genres:        m.Genres,
+		Created:       m.Created,
+		Updated:       m.Updated,
+	}
+}
+
+// FillFromDB returns a Movie from a MovieDB extracted from the DB
+func (m *Movie) FillFromDB(mDB *MovieDB) {
+	m.Created = mDB.Created
+	m.Updated = mDB.Updated
+	m.ID = mDB.ID
+	m.ImdbID = mDB.ImdbID
+	m.Title = mDB.Title
+	m.Rating = mDB.Rating
+	m.Votes = mDB.Votes
+	m.Plot = mDB.Plot
+	m.TmdbID = mDB.TmdbID
+	m.Year = mDB.Year
+	m.OriginalTitle = mDB.OriginalTitle
+	m.Runtime = mDB.Runtime
+	m.Genres = mDB.Genres
+	m.SortTitle = mDB.SortTitle
+	m.Tagline = mDB.Tagline
+}
 
 // Movie represents a movie
 type Movie struct {
@@ -55,6 +110,7 @@ type Movie struct {
 	PosterURL   string `json:"poster_url"`
 }
 
+// New returns a new Movie with an ImDB id
 func New(imdbID string) *Movie {
 	return &Movie{
 		Movie: polochon.Movie{
@@ -65,24 +121,23 @@ func New(imdbID string) *Movie {
 
 // Get returns show details in database from id or imdbid or an error
 func (m *Movie) Get(env *web.Env) error {
+	var mDB MovieDB
 	var err error
 	if m.ID != "" {
-		err = env.Database.QueryRowx(getMovieQueryByID, m.ID).StructScan(m)
+		err = env.Database.QueryRowx(getMovieQueryByID, m.ID).StructScan(&mDB)
 	} else if m.ImdbID != "" {
-		err = env.Database.QueryRowx(getMovieQueryByImdbID, m.ImdbID).StructScan(m)
+		err = env.Database.QueryRowx(getMovieQueryByImdbID, m.ImdbID).StructScan(&mDB)
 	} else {
 		err = fmt.Errorf("Can't get movie details, you have to specify an ID or ImdbID")
 	}
 	if err != nil {
-		if err.Error() == "sql: no rows in result set" {
-			return ErrNotFound
-		}
 		return err
 	}
 
 	// Set the poster url
 	m.PosterURL = m.GetPosterURL(env)
 
+	m.FillFromDB(&mDB)
 	return nil
 }
 
@@ -93,37 +148,33 @@ func (m *Movie) Get(env *web.Env) error {
 // If force is used, the detailer will be used even if the movie is found in
 // database
 func (m *Movie) GetDetails(env *web.Env, force bool) error {
-	if len(m.Detailers) == 0 {
-		m.Detailers = env.Config.MovieDetailers
-	}
-
 	log := env.Log.WithFields(logrus.Fields{
 		"imdb_id":  m.ImdbID,
 		"function": "movies.GetDetails",
 	})
+	log.Debugf("getting details")
 
-	// If the movie is not in db, we should add it, otherwise we should update
-	// it
-	var dbFunc func(db *sqlx.DB) error
+	if len(m.Detailers) == 0 {
+		m.Detailers = env.Config.MovieDetailers
+	}
 
 	var err error
 	err = m.Get(env)
 	switch err {
 	case nil:
 		log.Debug("movie found in database")
-		dbFunc = m.Update
 		if !force {
+			log.Debug("returning movie from db")
 			return nil
 		}
-	case ErrNotFound:
-		dbFunc = m.Add
+	case sql.ErrNoRows:
 		log.Debug("movie not found in database")
 	default:
 		// Unexpected error
 		return err
 	}
 
-	// so we got ErrNotFound so GetDetails from a detailer
+	// GetDetail
 	err = m.Movie.GetDetails(env.Log)
 	if err != nil {
 		return err
@@ -131,8 +182,9 @@ func (m *Movie) GetDetails(env *web.Env, force bool) error {
 
 	log.Debug("got details from detailers")
 
-	err = dbFunc(env.Database)
+	err = m.Upsert(env.Database)
 	if err != nil {
+		log.Debug("error while doing db func")
 		return err
 	}
 
@@ -152,10 +204,67 @@ func (m *Movie) GetDetails(env *web.Env, force bool) error {
 	return nil
 }
 
-// Add a movie in the database
-func (m *Movie) Add(db *sqlx.DB) error {
+// GetTorrents retrieves torrents for the movie, first try to get info from db,
+// if not exists, use polochon.Torrenter and save informations in the database
+// for future use
+//
+// If force is used, the torrenter will be used even if the torrent is found in
+// database
+func (m *Movie) GetTorrents(env *web.Env, force bool) error {
+	log := env.Log.WithFields(logrus.Fields{
+		"imdb_id":  m.ImdbID,
+		"function": "movies.GetTorrents",
+	})
+	log.Debugf("getting torrents")
+
+	if len(m.Torrenters) == 0 {
+		m.Torrenters = env.Config.MovieTorrenters
+	}
+
+	movieTorrents, err := torrents.GetMovieTorrents(env.Database, m.ImdbID)
+	switch err {
+	case nil:
+		log.Debug("torrents found in database")
+		if !force {
+			log.Debugf("returning %d torrents from db", len(movieTorrents))
+			// Add the torrents to the movie
+			for _, t := range movieTorrents {
+				m.Torrents = append(m.Torrents, t.Torrent)
+			}
+			return nil
+		}
+	case sql.ErrNoRows:
+		log.Debug("torrent not found in database")
+		// We'll need to GetTorrents from torrenters
+	default:
+		// Unexpected error
+		return err
+	}
+
+	err = m.Movie.GetTorrents(env.Log)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("got %d torrents from torrenters", len(m.Movie.Torrents))
+
+	for _, t := range m.Movie.Torrents {
+		torrent := torrents.NewMovie(m.ImdbID, t)
+		err = torrent.Upsert(env.Database)
+		if err != nil {
+			log.Error("error while adding torrent", err)
+			continue
+		}
+	}
+
+	return nil
+}
+
+// Upsert a movie in the database
+func (m *Movie) Upsert(db *sqlx.DB) error {
+	mDB := NewMovieDB(m)
 	var id string
-	r, err := db.NamedQuery(addMovieQuery, m)
+	r, err := db.NamedQuery(upsertMovieQuery, mDB)
 	if err != nil {
 		return err
 	}
@@ -167,12 +276,6 @@ func (m *Movie) Add(db *sqlx.DB) error {
 	return nil
 }
 
-// Update a movie in the database
-func (m *Movie) Update(db *sqlx.DB) error {
-	_, err := db.NamedQuery(updateMovieQuery, m)
-	return err
-}
-
 // Delete movie from database
 func (m *Movie) Delete(db *sqlx.DB) error {
 	r, err := db.Exec(deleteMovieQuery, m.ID)

src/internal/movies/movies_test.go

@@ -1,6 +1,7 @@
 package movies
 
 import (
+	"database/sql"
 	"fmt"
 	"os"
 	"testing"
@@ -11,7 +12,9 @@ import (
 	_ "github.com/mattes/migrate/driver/postgres"
 	"github.com/odwrtw/polochon/lib"
 	"github.com/odwrtw/polochon/modules/mock"
-	"gitlab.quimbo.fr/odwrtw/canape-sql/sqly"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web"
 )
 
 var db *sqlx.DB
@@ -31,6 +34,15 @@ func init() {
 
 func TestIntegrate(t *testing.T) {
 	sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) {
+		log := logrus.NewEntry(logrus.New())
+		env := web.NewEnv(web.EnvParams{
+			Database: db,
+			// Auth:     authorizer,
+			Log: log,
+			Config: &config.Config{
+				PublicDir: "/tmp",
+			},
+		})
 		detailer, _ := mock.NewDetailer(nil)
 		polochonConfig := polochon.MovieConfig{
 			Detailers: []polochon.Detailer{
@@ -41,11 +53,11 @@ func TestIntegrate(t *testing.T) {
 			Movie: polochon.Movie{
 				MovieConfig: polochonConfig,
 				ImdbID:      "tt12345",
+				Genres:      []string{},
 			},
 		}
 
-		log := logrus.NewEntry(logrus.New())
-		err := movie.GetDetails(db, log)
+		err := movie.GetDetails(env, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -57,7 +69,7 @@ func TestIntegrate(t *testing.T) {
 				ImdbID:      "tt12345",
 			},
 		}
-		err = movie.Get(db)
+		err = movie.Get(env)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -72,8 +84,8 @@ func TestIntegrate(t *testing.T) {
 		}
 
 		// Get it again
-		err = movie.Get(db)
-		if err != ErrNotFound {
+		err = movie.Get(env)
+		if err != sql.ErrNoRows {
 			t.Fatalf("Unexpected error: %q", err)
 		}
 	})

src/internal/shows/shows_test.go

@@ -5,15 +5,16 @@ import (
 	"os"
 	"testing"
 
-	"gitlab.quimbo.fr/odwrtw/canape-sql/sqly"
-	"gitlab.quimbo.fr/odwrtw/canape-sql/users"
-
 	"github.com/Sirupsen/logrus"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq"
 	_ "github.com/mattes/migrate/driver/postgres"
 	"github.com/odwrtw/polochon/lib"
 	"github.com/odwrtw/polochon/modules/mock"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/users"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web"
 )
 
 var db *sqlx.DB
@@ -89,8 +90,40 @@ func TestIntegrate(t *testing.T) {
 			t.Fatalf("Unexpected error: %q", err)
 		}
 	})
-
 }
+
+// UserBackend represents the data backend to get the user
+type UserBackend struct {
+	Database *sqlx.DB
+}
+
+// Get gets the username from the UserBackend
+func (b *UserBackend) Get(username string) (auth.User, error) {
+	return users.Get(b.Database, username)
+}
+
+func getEnv(db *sqlx.DB) *web.Env {
+	uBackend := &UserBackend{Database: db}
+
+	authParams := auth.Params{
+		Backend: uBackend,
+		Pepper:  "pepper",
+		Cost:    10,
+		Secret:  "secret",
+	}
+	authorizer := auth.New(authParams)
+
+	log := logrus.NewEntry(logrus.New())
+	env := web.NewEnv(web.EnvParams{
+		Database: db,
+		Auth:     authorizer,
+		Log:      log,
+		// Config:   cf,
+	})
+
+	return env
+}
+
 func TestTrackedShow(t *testing.T) {
 	sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) {
 		detailer, _ := mock.NewDetailer(nil)
@@ -113,6 +146,18 @@ func TestTrackedShow(t *testing.T) {
 		}
 
 		u := &users.User{Name: "plop"}
+		env := getEnv(db)
+
+		u.Hash, err = env.Auth.GenHash("pass")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.NewConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		err = u.Add(db)
 		if err != nil {
 			t.Fatal(err)

src/internal/sqly/string_slice.go

@@ -1,58 +0,0 @@
-package sqly
-
-import (
-	"database/sql/driver"
-	"encoding/csv"
-	"errors"
-	"regexp"
-	"strings"
-)
-
-// This mainly comes from https://gist.github.com/adharris/4163702
-
-// StringSlice represents an array of string. The custom type is needed because
-// pq does not support slices yet..
-type StringSlice []string
-
-// for replacing escaped quotes except if it is preceded by a literal backslash
-//  eg "\\" should translate to a quoted element whose value is \
-
-var quoteEscapeRegex = regexp.MustCompile(`([^\\]([\\]{2})*)\\"`)
-
-// Scan convert to a slice of strings
-// http://www.postgresql.org/docs/9.1/static/arrays.html#ARRAYS-IO
-func (s *StringSlice) Scan(src interface{}) error {
-	asBytes, ok := src.([]byte)
-	if !ok {
-		return error(errors.New("Scan source was not []bytes"))
-	}
-	str := string(asBytes)
-
-	// change quote escapes for csv parser
-	str = quoteEscapeRegex.ReplaceAllString(str, `$1""`)
-	str = strings.Replace(str, `\\`, `\`, -1)
-	// remove braces
-	str = str[1 : len(str)-1]
-	csvReader := csv.NewReader(strings.NewReader(str))
-
-	slice, err := csvReader.Read()
-
-	if err != nil {
-		return err
-	}
-
-	(*s) = StringSlice(slice)
-
-	return nil
-}
-
-// Value implements the Valuer interface
-func (s StringSlice) Value() (driver.Value, error) {
-	// string escapes.
-	// \ => \\\
-	// " => \"
-	for i, elem := range s {
-		s[i] = `"` + strings.Replace(strings.Replace(elem, `\`, `\\\`, -1), `"`, `\"`, -1) + `"`
-	}
-	return "{" + strings.Join(s, ",") + "}", nil
-}

src/internal/torrents/torrents.go

@@ -0,0 +1,180 @@
+package torrents
+
+import (
+	"database/sql"
+	"fmt"
+	"time"
+
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly"
+
+	"github.com/jmoiron/sqlx"
+	polochon "github.com/odwrtw/polochon/lib"
+)
+
+const (
+	upsertMovieTorrentQuery = `
+	INSERT INTO movie_torrents (imdb_id, url, source, quality, upload_user,
+	seeders, leechers)
+	VALUES (:imdb_id, :url, :source, :quality, :upload_user, :seeders,
+	:leechers)
+	ON CONFLICT (imdb_id, quality, source)
+	DO UPDATE SET imdb_id=:imdb_id, url=:url, source=:source, quality=:quality,
+	upload_user=:upload_user, seeders=:seeders, leechers=:leechers
+	RETURNING id;`
+
+	getMovieTorrentQueryByImdbID = `
+	SELECT *
+	FROM movie_torrents WHERE imdb_id=$1;`
+
+	getMovieTorrentQueryByID = `
+	SELECT *
+	FROM movie_torrents WHERE id=$1;`
+
+	deleteMovieTorrentQuery = `DELETE FROM movie_torrents WHERE id=$1;`
+)
+
+// MovieTorrent represents a movie torrent
+type MovieTorrent struct {
+	sqly.BaseModel
+	polochon.Torrent
+	ImdbID string `json:"imdb_id"`
+}
+
+// MovieTorrentDB represents the MovieTorrent in the DB
+type MovieTorrentDB struct {
+	ID         string    `db:"id"`
+	ImdbID     string    `db:"imdb_id"`
+	URL        string    `db:"url"`
+	Source     string    `db:"source"`
+	Quality    string    `db:"quality"`
+	UploadUser string    `db:"upload_user"`
+	Seeders    int       `db:"seeders"`
+	Leechers   int       `db:"leechers"`
+	Created    time.Time `db:"created_at"`
+	Updated    time.Time `db:"updated_at"`
+}
+
+// EpisodeTorrent represents an episode torrent
+type EpisodeTorrent struct {
+	sqly.BaseModel
+	polochon.Torrent
+	ShowImdbID string `json:"show_imdb_id"`
+	Season     int    `json:"season"`
+	Episode    int    `json:"episode"`
+}
+
+// NewMovieFromImDB returns a new MovieTorrent with an ImDB id
+func NewMovieFromImDB(imdbID string) *MovieTorrent {
+	return &MovieTorrent{
+		ImdbID: imdbID,
+	}
+}
+
+// NewMovie returns a new MovieTorrent with an ImDB id
+func NewMovie(imdbID string, poTo polochon.Torrent) *MovieTorrent {
+	return &MovieTorrent{
+		ImdbID:  imdbID,
+		Torrent: poTo,
+	}
+}
+
+// Get returns show details in database from id or imdbid or an error
+func (m *MovieTorrent) Get(db *sqlx.DB) error {
+	var mDB MovieTorrentDB
+	var err error
+	if m.ID != "" {
+		err = db.QueryRowx(getMovieTorrentQueryByID, m.ID).StructScan(&mDB)
+	} else if m.ImdbID != "" {
+		err = db.QueryRowx(getMovieTorrentQueryByImdbID, m.ImdbID).StructScan(&mDB)
+	} else {
+		err = fmt.Errorf("can't get movie torrent details, you have to specify an ID or ImdbID")
+	}
+	if err != nil {
+		return err
+	}
+	m.FillFromDB(&mDB)
+	return nil
+}
+
+// GetMovieTorrents returns show details in database from id or imdbid or an error
+func GetMovieTorrents(db *sqlx.DB, imdbID string) ([]*MovieTorrent, error) {
+	var torrentsDB = []*MovieTorrentDB{}
+	err := db.Select(&torrentsDB, getMovieTorrentQueryByImdbID, imdbID)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(torrentsDB) == 0 {
+		return nil, sql.ErrNoRows
+	}
+
+	var torrents []*MovieTorrent
+	for _, torrentDB := range torrentsDB {
+		movie := NewMovieFromImDB(imdbID)
+		movie.FillFromDB(torrentDB)
+		torrents = append(torrents, movie)
+	}
+
+	return torrents, nil
+}
+
+// Upsert a movie torrent in the database
+func (m *MovieTorrent) Upsert(db *sqlx.DB) error {
+	mDB := NewMovieTorrentDB(m)
+	var id string
+	r, err := db.NamedQuery(upsertMovieTorrentQuery, mDB)
+	if err != nil {
+		return err
+	}
+	for r.Next() {
+		r.Scan(&id)
+	}
+	m.ID = id
+
+	return nil
+}
+
+// Delete movie from database
+func (m *MovieTorrent) Delete(db *sqlx.DB) error {
+	r, err := db.Exec(deleteMovieTorrentQuery, m.ID)
+	if err != nil {
+		return err
+	}
+	count, _ := r.RowsAffected()
+	if count != 1 {
+		return fmt.Errorf("Unexpected number of row deleted: %d", count)
+	}
+	return nil
+}
+
+// NewMovieTorrentDB returns a MovieTorrent ready to be put in DB from a
+// MovieTorrent
+func NewMovieTorrentDB(m *MovieTorrent) MovieTorrentDB {
+	return MovieTorrentDB{
+		ID:         m.ID,
+		ImdbID:     m.ImdbID,
+		URL:        m.URL,
+		Source:     m.Source,
+		Quality:    string(m.Quality),
+		UploadUser: m.UploadUser,
+		Seeders:    m.Seeders,
+		Leechers:   m.Leechers,
+		Created:    m.Created,
+		Updated:    m.Updated,
+	}
+}
+
+// FillFromDB will fill a MovieTorrent from a MovieTorrentDB extracted from the DB
+func (m *MovieTorrent) FillFromDB(mDB *MovieTorrentDB) {
+	q, _ := polochon.StringToQuality(mDB.Quality)
+	m.ImdbID = mDB.ImdbID
+	m.Created = mDB.Created
+	m.Updated = mDB.Updated
+	m.ID = mDB.ID
+	m.URL = mDB.URL
+	m.Source = mDB.Source
+	m.Quality = *q
+	m.UploadUser = mDB.UploadUser
+	m.Seeders = mDB.Seeders
+	m.Leechers = mDB.Leechers
+}

src/internal/users/users.go

@@ -80,7 +80,7 @@ func (u *User) SetConfig(key string, v interface{}) error {
 
 // NewConfig creates a new empty config
 func (u *User) NewConfig() error {
-	var configMap map[string]*json.RawMessage
+	configMap := make(map[string]*json.RawMessage)
 	b, err := json.Marshal(configMap)
 	if err != nil {
 		return err

src/internal/users/users_test.go

@@ -5,8 +5,11 @@ import (
 	"os"
 	"testing"
 
-	"gitlab.quimbo.fr/odwrtw/canape-sql/sqly"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/sqly"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 	_ "github.com/mattes/migrate/driver/postgres"
@@ -32,7 +35,19 @@ func TestUser(t *testing.T) {
 
 		// Add a new user
 		u := &User{Name: "plop", Hash: "plop"}
-		err := u.Add(db)
+		var err error
+		env := getEnv(db)
+
+		u.Hash, err = env.Auth.GenHash("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.NewConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = u.Add(db)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -99,7 +114,19 @@ func TestTokenAddDelete(t *testing.T) {
 	sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) {
 		// Add a new user
 		u := &User{Name: "plop", Hash: "plop"}
-		err := u.Add(db)
+		var err error
+		env := getEnv(db)
+
+		u.Hash, err = env.Auth.GenHash("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.NewConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = u.Add(db)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -155,11 +182,60 @@ func TestTokenAddDelete(t *testing.T) {
 	})
 }
 
+// UserBackend represents the data backend to get the user
+type UserBackend struct {
+	Database *sqlx.DB
+}
+
+// Get gets the username from the UserBackend
+func (b *UserBackend) Get(username string) (auth.User, error) {
+	return Get(b.Database, username)
+}
+
+func getEnv(db *sqlx.DB) *web.Env {
+	uBackend := &UserBackend{Database: db}
+
+	authParams := auth.Params{
+		Backend: uBackend,
+		Pepper:  "pepper",
+		Cost:    10,
+		Secret:  "secret",
+	}
+	authorizer := auth.New(authParams)
+
+	log := logrus.NewEntry(logrus.New())
+	env := web.NewEnv(web.EnvParams{
+		Database: db,
+		Auth:     authorizer,
+		Log:      log,
+		// Config:   cf,
+	})
+
+	return env
+}
+
 func TestTokenCheck(t *testing.T) {
 	sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) {
 		u := &User{Name: "plop", Hash: "plop"}
-		u.Add(db)
+		var err error
+		env := getEnv(db)
+
+		u.Hash, err = env.Auth.GenHash("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.NewConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err = u.Add(db); err != nil {
+			t.Fatal(err)
+		}
 		token, err := u.NewToken(db)
+		if err != nil {
+			t.Fatal(err)
+		}
 
 		ok, err := u.CheckToken(db, token.Value)
 		if err != nil {
@@ -183,10 +259,29 @@ func TestTokenCheck(t *testing.T) {
 
 func TestAutoUpdateCols(t *testing.T) {
 	sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) {
-		u := &User{Name: "plop", Hash: "plop"}
-		u.Add(db)
+		u := &User{Name: "plop"}
+		var err error
+		env := getEnv(db)
+
+		u.Hash, err = env.Auth.GenHash("pass")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.NewConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.Add(db)
+		if err != nil {
+			t.Fatal(err)
+		}
 		u.Name = "toto"
-		u.Update(db)
+		err = u.Update(db)
+		if err != nil {
+			t.Fatal(err)
+		}
 
 		if !u.Created.Before(u.Updated) {
 			t.Fatalf("colum updated not auto updated on table users")
@@ -197,9 +292,25 @@ func TestAutoUpdateCols(t *testing.T) {
 func TestConfig(t *testing.T) {
 	sqly.RunWithLastestMigration(db, pgdsn, t, func(db *sqlx.DB, t *testing.T) {
 		u := &User{Name: "plop", Hash: "plop"}
-		u.Add(db)
+		env := getEnv(db)
 
-		u, err := Get(db, "plop")
+		var err error
+		u.Hash, err = env.Auth.GenHash("pass")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.NewConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = u.Add(db)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		u, err = Get(db, "plop")
 		if err != nil {
 			t.Fatal(err)
 		}

src/main.go

@@ -4,9 +4,11 @@ import (
 	"net/http"
 	"os"
 
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies"
+
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/auth"
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/config"
-	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/movies"
+	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/external_medias"
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/users"
 	"gitlab.quimbo.fr/odwrtw/canape-sql/src/internal/web"
 
@@ -74,8 +76,9 @@ func main() {
 	env.Handle("/users/edit", users.EditHandler).WithRole(users.UserRole)
 
 	env.Handle("/movies/polochon", movies.FromPolochon).WithRole(users.UserRole)
-	env.Handle("/movies/explore/popular", movies.ExplorePopular).WithRole(users.UserRole)
 	env.Handle("/movies/{id:tt[0-9]+}/get_details", movies.GetDetailsHandler).WithRole(users.UserRole)
+	env.Handle("/movies/explore", extmedias.Explore)
+	env.Handle("/movies/refresh", extmedias.Refresh)
 
 	n := negroni.Classic()
 	n.Use(authMiddleware)

src/public/js/app.js

@@ -75,7 +75,7 @@ const UserIsAuthenticated = UserAuthWrapper({
 
 // TODO find a better way
 const MovieListPopular = (props) => (
-  <MovieList {...props} moviesUrl='/movies/explore/popular'/>
+  <MovieList {...props} moviesUrl='/movies/explore'/>
 )
 const MovieListPolochon = (props) => (
   <MovieList {...props} moviesUrl='/movies/polochon'/>

src/public/js/components/movies/list.js

@@ -130,6 +130,13 @@ class MovieButtons extends React.Component {
             <i className="fa fa-download"></i> Download
           </a>
         }
+        {this.props.movie.torrents && this.props.movie.torrents.map(function(torrent, index) {
+          return (
+            <a key={torrent.url} type="button" className="btn btn-primary btn-sm" href={torrent.url}>
+              <i className="fa fa-download"></i> {torrent.quality} Torrent
+            </a>
+          )}
+        )}
         <a type="button" className="btn btn-warning btn-sm" href={imdb_link}>
           <i className="fa fa-external-link"></i> IMDB
         </a>