<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcp

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"strings"
	"sync"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/yaml"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/util/jsonpath"
	"k8s.io/klog/v2"
)

func init() {
	if err := restclient.RegisterAuthProviderPlugin("gcp", newGCPAuthProvider); err != nil {
		klog.Fatalf("Failed to register gcp auth plugin: %v", err)
	}
}

var (
	// Stubbable for testing
	execCommand = exec.Command

	// defaultScopes:
	// - cloud-platform is the base scope to authenticate to GCP.
	// - userinfo.email is used to authenticate to GKE APIs with gserviceaccount
	//   email instead of numeric uniqueID.
	defaultScopes = []string{
		"https://www.googleapis.com/auth/cloud-platform",
		"https://www.googleapis.com/auth/userinfo.email"}
)

// gcpAuthProvider is an auth provider plugin that uses GCP credentials to provide
// tokens for kubectl to authenticate itself to the apiserver. A sample json config
// is provided below with all recognized options described.
//
// {
//   'auth-provider': {
//     # Required
//     "name": "gcp",
//
//     'config': {
//       # Authentication options
//       # These options are used while getting a token.
//
//       # comma-separated list of GCP API scopes. default value of this field
//       # is "https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/userinfo.email".
// 		 # to override the API scopes, specify this field explicitly.
//       "scopes": "https://www.googleapis.com/auth/cloud-platform"
//
//       # Caching options
//
//       # Raw string data representing cached access token.
//       "access-token": "ya29.CjWdA4GiBPTt",
//       # RFC3339Nano expiration timestamp for cached access token.
//       "expiry": "2016-10-31 22:31:9.123",
//
//       # Command execution options
//       # These options direct the plugin to execute a specified command and parse
//       # token and expiry time from the output of the command.
//
//       # Command to execute for access token. Command output will be parsed as JSON.
//       # If "cmd-args" is not present, this value will be split on whitespace, with
//       # the first element interpreted as the command, remaining elements as args.
//       "cmd-path": "/usr/bin/gcloud",
//
//       # Arguments to pass to command to execute for access token.
//       "cmd-args": "config config-helper --output=json"
//
//       # JSONPath to the string field that represents the access token in
//       # command output. If omitted, defaults to "{.access_token}".
//       "token-key": "{.credential.access_token}",
//
//       # JSONPath to the string field that represents expiration timestamp
//       # of the access token in the command output. If omitted, defaults to
//       # "{.token_expiry}"
//       "expiry-key": ""{.credential.token_expiry}",
//
//       # golang reference time in the format that the expiration timestamp uses.
//       # If omitted, defaults to time.RFC3339Nano
//       "time-fmt": "2006-01-02 15:04:05.999999999"
//     }
//   }
// }
//
type gcpAuthProvider struct {
	tokenSource oauth2.TokenSource
	persister   restclient.AuthProviderConfigPersister
}

var warnOnce sync.Once

func newGCPAuthProvider(_ string, gcpConfig map[string]string, persister restclient.AuthProviderConfigPersister) (restclient.AuthProvider, error) {
	// deprecated in v1.22, remove in v1.25
	// this should be updated to use klog.Warningf in v1.24 to more actively warn consumers
	warnOnce.Do(func() {
		klog.V(1).Infof(`WARNING: the gcp auth plugin is deprecated in v1.22+, unavailable in v1.25+; use gcloud instead.
To learn more, consult https://kubernetes.io/docs/reference/access-authn-authz/authentication/#client-go-credential-plugins`)
	})

	ts, err := tokenSource(isCmdTokenSource(gcpConfig), gcpConfig)
	if err != nil {
		return nil, err
	}
	cts, err := newCachedTokenSource(gcpConfig["access-token"], gcpConfig["expiry"], persister, ts, gcpConfig)
	if err != nil {
		return nil, err
	}
	return &gcpAuthProvider{cts, persister}, nil
}

func isCmdTokenSource(gcpConfig map[string]string) bool {
	_, ok := gcpConfig["cmd-path"]
	return ok
}

func tokenSource(isCmd bool, gcpConfig map[string]string) (oauth2.TokenSource, error) {
	// Command-based token source
	if isCmd {
		cmd := gcpConfig["cmd-path"]
		if len(cmd) == 0 {
			return nil, fmt.Errorf("missing access token cmd")
		}
		if gcpConfig["scopes"] != "" {
			return nil, fmt.Errorf("scopes can only be used when kubectl is using a gcp service account key")
		}
		var args []string
		if cmdArgs, ok := gcpConfig["cmd-args"]; ok {
			args = strings.Fields(cmdArgs)
		} else {
			fields := strings.Fields(cmd)
			cmd = fields[0]
			args = fields[1:]
		}
		return newCmdTokenSource(cmd, args, gcpConfig["token-key"], gcpConfig["expiry-key"], gcpConfig["time-fmt"]), nil
	}

	// Google Application Credentials-based token source
	scopes := parseScopes(gcpConfig)
	ts, err := google.DefaultTokenSource(context.Background(), scopes...)
	if err != nil {
		return nil, fmt.Errorf("cannot construct google default token source: %v", err)
	}
	return ts, nil
}

// parseScopes constructs a list of scopes that should be included in token source
// from the config map.
func parseScopes(gcpConfig map[string]string) []string {
	scopes, ok := gcpConfig["scopes"]
	if !ok {
		return defaultScopes
	}
	if scopes == "" {
		return []string{}
	}
	return strings.Split(gcpConfig["scopes"], ",")
}

func (g *gcpAuthProvider) WrapTransport(rt http.RoundTripper) http.RoundTripper {
	var resetCache map[string]string
	if cts, ok := g.tokenSource.(*cachedTokenSource); ok {
		resetCache = cts.baseCache()
	} else {
		resetCache = make(map[string]string)
	}
	return &conditionalTransport{&oauth2.Transport{Source: g.tokenSource, Base: rt}, g.persister, resetCache}
}

func (g *gcpAuthProvider) Login() error { return nil }

type cachedTokenSource struct {
	lk          sync.Mutex
	source      oauth2.TokenSource
	accessToken string `datapolicy:"token"`
	expiry      time.Time
	persister   restclient.AuthProviderConfigPersister
	cache       map[string]string
}

func newCachedTokenSource(accessToken, expiry string, persister restclient.AuthProviderConfigPersister, ts oauth2.TokenSource, cache map[string]string) (*cachedTokenSource, error) {
	var expiryTime time.Time
	if parsedTime, err := time.Parse(time.RFC3339Nano, expiry); err == nil {
		expiryTime = parsedTime
	}
	if cache == nil {
		cache = make(map[string]string)
	}
	return &cachedTokenSource{
		source:      ts,
		accessToken: accessToken,
		expiry:      expiryTime,
		persister:   persister,
		cache:       cache,
	}, nil
}

func (t *cachedTokenSource) Token() (*oauth2.Token, error) {
	tok := t.cachedToken()
	if tok.Valid() && !tok.Expiry.IsZero() {
		return tok, nil
	}
	tok, err := t.source.Token()
	if err != nil {
		return nil, err
	}
	cache := t.update(tok)
	if t.persister != nil {
		if err := t.persister.Persist(cache); err != nil {
			klog.V(4).Infof("Failed to persist token: %v", err)
		}
	}
	return tok, nil
}

func (t *cachedTokenSource) cachedToken() *oauth2.Token {
	t.lk.Lock()
	defer t.lk.Unlock()
	return &oauth2.Token{
		AccessToken: t.accessToken,
		TokenType:   "Bearer",
		Expiry:      t.expiry,
	}
}

func (t *cachedTokenSource) update(tok *oauth2.Token) map[string]string {
	t.lk.Lock()
	defer t.lk.Unlock()
	t.accessToken = tok.AccessToken
	t.expiry = tok.Expiry
	ret := map[string]string{}
	for k, v := range t.cache {
		ret[k] = v
	}
	ret["access-token"] = t.accessToken
	ret["expiry"] = t.expiry.Format(time.RFC3339Nano)
	return ret
}

// baseCache is the base configuration value for this TokenSource, without any cached ephemeral tokens.
func (t *cachedTokenSource) baseCache() map[string]string {
	t.lk.Lock()
	defer t.lk.Unlock()
	ret := map[string]string{}
	for k, v := range t.cache {
		ret[k] = v
	}
	delete(ret, "access-token")
	delete(ret, "expiry")
	return ret
}

type commandTokenSource struct {
	cmd       string
	args      []string
	tokenKey  string `datapolicy:"token"`
	expiryKey string `datapolicy:"secret-key"`
	timeFmt   string
}

func newCmdTokenSource(cmd string, args []string, tokenKey, expiryKey, timeFmt string) *commandTokenSource {
	if len(timeFmt) == 0 {
		timeFmt = time.RFC3339Nano
	}
	if len(tokenKey) == 0 {
		tokenKey = "{.access_token}"
	}
	if len(expiryKey) == 0 {
		expiryKey = "{.token_expiry}"
	}
	return &commandTokenSource{
		cmd:       cmd,
		args:      args,
		tokenKey:  tokenKey,
		expiryKey: expiryKey,
		timeFmt:   timeFmt,
	}
}

func (c *commandTokenSource) Token() (*oauth2.Token, error) {
	fullCmd := strings.Join(append([]string{c.cmd}, c.args...), " ")
	cmd := execCommand(c.cmd, c.args...)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	output, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("error executing access token command %q: err=%v output=%s stderr=%s", fullCmd, err, output, string(stderr.Bytes()))
	}
	token, err := c.parseTokenCmdOutput(output)
	if err != nil {
		return nil, fmt.Errorf("error parsing output for access token command %q: %v", fullCmd, err)
	}
	return token, nil
}

func (c *commandTokenSource) parseTokenCmdOutput(output []byte) (*oauth2.Token, error) {
	output, err := yaml.ToJSON(output)
	if err != nil {
		return nil, err
	}
	var data interface{}
	if err := json.Unmarshal(output, &data); err != nil {
		return nil, err
	}

	accessToken, err := parseJSONPath(data, "token-key", c.tokenKey)
	if err != nil {
		return nil, fmt.Errorf("error parsing token-key %q from %q: %v", c.tokenKey, string(output), err)
	}
	expiryStr, err := parseJSONPath(data, "expiry-key", c.expiryKey)
	if err != nil {
		return nil, fmt.Errorf("error parsing expiry-key %q from %q: %v", c.expiryKey, string(output), err)
	}
	var expiry time.Time
	if t, err := time.Parse(c.timeFmt, expiryStr); err != nil {
		klog.V(4).Infof("Failed to parse token expiry from %s (fmt=%s): %v", expiryStr, c.timeFmt, err)
	} else {
		expiry = t
	}

	return &oauth2.Token{
		AccessToken: accessToken,
		TokenType:   "Bearer",
		Expiry:      expiry,
	}, nil
}

func parseJSONPath(input interface{}, name, template string) (string, error) {
	j := jsonpath.New(name)
	buf := new(bytes.Buffer)
	if err := j.Parse(template); err != nil {
		return "", err
	}
	if err := j.Execute(buf, input); err != nil {
		return "", err
	}
	return buf.String(), nil
}

type conditionalTransport struct {
	oauthTransport *oauth2.Transport
	persister      restclient.AuthProviderConfigPersister
	resetCache     map[string]string
}

var _ net.RoundTripperWrapper = &conditionalTransport{}

func (t *conditionalTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if len(req.Header.Get("Authorization")) != 0 {
		return t.oauthTransport.Base.RoundTrip(req)
	}

	res, err := t.oauthTransport.RoundTrip(req)

	if err != nil {
		return nil, err
	}

	if res.StatusCode == 401 {
		klog.V(4).Infof("The credentials that were supplied are invalid for the target cluster")
		t.persister.Persist(t.resetCache)
	}

	return res, nil
}

func (t *conditionalTransport) WrappedRoundTripper() http.RoundTripper { return t.oauthTransport.Base }
<commit_msg>add better link for gcp auth plugin doc<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcp

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"strings"
	"sync"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/yaml"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/util/jsonpath"
	"k8s.io/klog/v2"
)

func init() {
	if err := restclient.RegisterAuthProviderPlugin("gcp", newGCPAuthProvider); err != nil {
		klog.Fatalf("Failed to register gcp auth plugin: %v", err)
	}
}

var (
	// Stubbable for testing
	execCommand = exec.Command

	// defaultScopes:
	// - cloud-platform is the base scope to authenticate to GCP.
	// - userinfo.email is used to authenticate to GKE APIs with gserviceaccount
	//   email instead of numeric uniqueID.
	defaultScopes = []string{
		"https://www.googleapis.com/auth/cloud-platform",
		"https://www.googleapis.com/auth/userinfo.email"}
)

// gcpAuthProvider is an auth provider plugin that uses GCP credentials to provide
// tokens for kubectl to authenticate itself to the apiserver. A sample json config
// is provided below with all recognized options described.
//
// {
//   'auth-provider': {
//     # Required
//     "name": "gcp",
//
//     'config': {
//       # Authentication options
//       # These options are used while getting a token.
//
//       # comma-separated list of GCP API scopes. default value of this field
//       # is "https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/userinfo.email".
// 		 # to override the API scopes, specify this field explicitly.
//       "scopes": "https://www.googleapis.com/auth/cloud-platform"
//
//       # Caching options
//
//       # Raw string data representing cached access token.
//       "access-token": "ya29.CjWdA4GiBPTt",
//       # RFC3339Nano expiration timestamp for cached access token.
//       "expiry": "2016-10-31 22:31:9.123",
//
//       # Command execution options
//       # These options direct the plugin to execute a specified command and parse
//       # token and expiry time from the output of the command.
//
//       # Command to execute for access token. Command output will be parsed as JSON.
//       # If "cmd-args" is not present, this value will be split on whitespace, with
//       # the first element interpreted as the command, remaining elements as args.
//       "cmd-path": "/usr/bin/gcloud",
//
//       # Arguments to pass to command to execute for access token.
//       "cmd-args": "config config-helper --output=json"
//
//       # JSONPath to the string field that represents the access token in
//       # command output. If omitted, defaults to "{.access_token}".
//       "token-key": "{.credential.access_token}",
//
//       # JSONPath to the string field that represents expiration timestamp
//       # of the access token in the command output. If omitted, defaults to
//       # "{.token_expiry}"
//       "expiry-key": ""{.credential.token_expiry}",
//
//       # golang reference time in the format that the expiration timestamp uses.
//       # If omitted, defaults to time.RFC3339Nano
//       "time-fmt": "2006-01-02 15:04:05.999999999"
//     }
//   }
// }
//
type gcpAuthProvider struct {
	tokenSource oauth2.TokenSource
	persister   restclient.AuthProviderConfigPersister
}

var warnOnce sync.Once

func newGCPAuthProvider(_ string, gcpConfig map[string]string, persister restclient.AuthProviderConfigPersister) (restclient.AuthProvider, error) {
	warnOnce.Do(func() {
		klog.Warningf(`WARNING: the gcp auth plugin is deprecated in v1.22+, unavailable in v1.25+; use gcloud instead.
To learn more, consult https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke`)
	})

	ts, err := tokenSource(isCmdTokenSource(gcpConfig), gcpConfig)
	if err != nil {
		return nil, err
	}
	cts, err := newCachedTokenSource(gcpConfig["access-token"], gcpConfig["expiry"], persister, ts, gcpConfig)
	if err != nil {
		return nil, err
	}
	return &gcpAuthProvider{cts, persister}, nil
}

func isCmdTokenSource(gcpConfig map[string]string) bool {
	_, ok := gcpConfig["cmd-path"]
	return ok
}

func tokenSource(isCmd bool, gcpConfig map[string]string) (oauth2.TokenSource, error) {
	// Command-based token source
	if isCmd {
		cmd := gcpConfig["cmd-path"]
		if len(cmd) == 0 {
			return nil, fmt.Errorf("missing access token cmd")
		}
		if gcpConfig["scopes"] != "" {
			return nil, fmt.Errorf("scopes can only be used when kubectl is using a gcp service account key")
		}
		var args []string
		if cmdArgs, ok := gcpConfig["cmd-args"]; ok {
			args = strings.Fields(cmdArgs)
		} else {
			fields := strings.Fields(cmd)
			cmd = fields[0]
			args = fields[1:]
		}
		return newCmdTokenSource(cmd, args, gcpConfig["token-key"], gcpConfig["expiry-key"], gcpConfig["time-fmt"]), nil
	}

	// Google Application Credentials-based token source
	scopes := parseScopes(gcpConfig)
	ts, err := google.DefaultTokenSource(context.Background(), scopes...)
	if err != nil {
		return nil, fmt.Errorf("cannot construct google default token source: %v", err)
	}
	return ts, nil
}

// parseScopes constructs a list of scopes that should be included in token source
// from the config map.
func parseScopes(gcpConfig map[string]string) []string {
	scopes, ok := gcpConfig["scopes"]
	if !ok {
		return defaultScopes
	}
	if scopes == "" {
		return []string{}
	}
	return strings.Split(gcpConfig["scopes"], ",")
}

func (g *gcpAuthProvider) WrapTransport(rt http.RoundTripper) http.RoundTripper {
	var resetCache map[string]string
	if cts, ok := g.tokenSource.(*cachedTokenSource); ok {
		resetCache = cts.baseCache()
	} else {
		resetCache = make(map[string]string)
	}
	return &conditionalTransport{&oauth2.Transport{Source: g.tokenSource, Base: rt}, g.persister, resetCache}
}

func (g *gcpAuthProvider) Login() error { return nil }

type cachedTokenSource struct {
	lk          sync.Mutex
	source      oauth2.TokenSource
	accessToken string `datapolicy:"token"`
	expiry      time.Time
	persister   restclient.AuthProviderConfigPersister
	cache       map[string]string
}

func newCachedTokenSource(accessToken, expiry string, persister restclient.AuthProviderConfigPersister, ts oauth2.TokenSource, cache map[string]string) (*cachedTokenSource, error) {
	var expiryTime time.Time
	if parsedTime, err := time.Parse(time.RFC3339Nano, expiry); err == nil {
		expiryTime = parsedTime
	}
	if cache == nil {
		cache = make(map[string]string)
	}
	return &cachedTokenSource{
		source:      ts,
		accessToken: accessToken,
		expiry:      expiryTime,
		persister:   persister,
		cache:       cache,
	}, nil
}

func (t *cachedTokenSource) Token() (*oauth2.Token, error) {
	tok := t.cachedToken()
	if tok.Valid() && !tok.Expiry.IsZero() {
		return tok, nil
	}
	tok, err := t.source.Token()
	if err != nil {
		return nil, err
	}
	cache := t.update(tok)
	if t.persister != nil {
		if err := t.persister.Persist(cache); err != nil {
			klog.V(4).Infof("Failed to persist token: %v", err)
		}
	}
	return tok, nil
}

func (t *cachedTokenSource) cachedToken() *oauth2.Token {
	t.lk.Lock()
	defer t.lk.Unlock()
	return &oauth2.Token{
		AccessToken: t.accessToken,
		TokenType:   "Bearer",
		Expiry:      t.expiry,
	}
}

func (t *cachedTokenSource) update(tok *oauth2.Token) map[string]string {
	t.lk.Lock()
	defer t.lk.Unlock()
	t.accessToken = tok.AccessToken
	t.expiry = tok.Expiry
	ret := map[string]string{}
	for k, v := range t.cache {
		ret[k] = v
	}
	ret["access-token"] = t.accessToken
	ret["expiry"] = t.expiry.Format(time.RFC3339Nano)
	return ret
}

// baseCache is the base configuration value for this TokenSource, without any cached ephemeral tokens.
func (t *cachedTokenSource) baseCache() map[string]string {
	t.lk.Lock()
	defer t.lk.Unlock()
	ret := map[string]string{}
	for k, v := range t.cache {
		ret[k] = v
	}
	delete(ret, "access-token")
	delete(ret, "expiry")
	return ret
}

type commandTokenSource struct {
	cmd       string
	args      []string
	tokenKey  string `datapolicy:"token"`
	expiryKey string `datapolicy:"secret-key"`
	timeFmt   string
}

func newCmdTokenSource(cmd string, args []string, tokenKey, expiryKey, timeFmt string) *commandTokenSource {
	if len(timeFmt) == 0 {
		timeFmt = time.RFC3339Nano
	}
	if len(tokenKey) == 0 {
		tokenKey = "{.access_token}"
	}
	if len(expiryKey) == 0 {
		expiryKey = "{.token_expiry}"
	}
	return &commandTokenSource{
		cmd:       cmd,
		args:      args,
		tokenKey:  tokenKey,
		expiryKey: expiryKey,
		timeFmt:   timeFmt,
	}
}

func (c *commandTokenSource) Token() (*oauth2.Token, error) {
	fullCmd := strings.Join(append([]string{c.cmd}, c.args...), " ")
	cmd := execCommand(c.cmd, c.args...)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	output, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("error executing access token command %q: err=%v output=%s stderr=%s", fullCmd, err, output, string(stderr.Bytes()))
	}
	token, err := c.parseTokenCmdOutput(output)
	if err != nil {
		return nil, fmt.Errorf("error parsing output for access token command %q: %v", fullCmd, err)
	}
	return token, nil
}

func (c *commandTokenSource) parseTokenCmdOutput(output []byte) (*oauth2.Token, error) {
	output, err := yaml.ToJSON(output)
	if err != nil {
		return nil, err
	}
	var data interface{}
	if err := json.Unmarshal(output, &data); err != nil {
		return nil, err
	}

	accessToken, err := parseJSONPath(data, "token-key", c.tokenKey)
	if err != nil {
		return nil, fmt.Errorf("error parsing token-key %q from %q: %v", c.tokenKey, string(output), err)
	}
	expiryStr, err := parseJSONPath(data, "expiry-key", c.expiryKey)
	if err != nil {
		return nil, fmt.Errorf("error parsing expiry-key %q from %q: %v", c.expiryKey, string(output), err)
	}
	var expiry time.Time
	if t, err := time.Parse(c.timeFmt, expiryStr); err != nil {
		klog.V(4).Infof("Failed to parse token expiry from %s (fmt=%s): %v", expiryStr, c.timeFmt, err)
	} else {
		expiry = t
	}

	return &oauth2.Token{
		AccessToken: accessToken,
		TokenType:   "Bearer",
		Expiry:      expiry,
	}, nil
}

func parseJSONPath(input interface{}, name, template string) (string, error) {
	j := jsonpath.New(name)
	buf := new(bytes.Buffer)
	if err := j.Parse(template); err != nil {
		return "", err
	}
	if err := j.Execute(buf, input); err != nil {
		return "", err
	}
	return buf.String(), nil
}

type conditionalTransport struct {
	oauthTransport *oauth2.Transport
	persister      restclient.AuthProviderConfigPersister
	resetCache     map[string]string
}

var _ net.RoundTripperWrapper = &conditionalTransport{}

func (t *conditionalTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if len(req.Header.Get("Authorization")) != 0 {
		return t.oauthTransport.Base.RoundTrip(req)
	}

	res, err := t.oauthTransport.RoundTrip(req)

	if err != nil {
		return nil, err
	}

	if res.StatusCode == 401 {
		klog.V(4).Infof("The credentials that were supplied are invalid for the target cluster")
		t.persister.Persist(t.resetCache)
	}

	return res, nil
}

func (t *conditionalTransport) WrappedRoundTripper() http.RoundTripper { return t.oauthTransport.Base }
<|endoftext|>


<commit_before>package core

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/mdlayher/wavepipe/data"

	"github.com/wtolson/go-taglib"
)

// fsFileSource represents a file source which indexes files in the local filesystem
type fsFileSource struct{}

// folderArtPair contains a folder ID and associated art ID
type folderArtPair struct {
	folderID int
	artID    int
}

// MediaScan scans for media files in the local filesystem
func (fsFileSource) MediaScan(mediaFolder string, verbose bool, walkCancelChan chan struct{}) error {
	// Halt walk if needed
	var mutex sync.RWMutex
	haltWalk := false
	go func() {
		// Wait for signal
		<-walkCancelChan

		// Halt!
		mutex.Lock()
		haltWalk = true
		mutex.Unlock()
	}()

	// Track metrics about the walk
	artCount := 0
	artistCount := 0
	albumCount := 0
	songCount := 0
	songUpdateCount := 0
	folderCount := 0
	startTime := time.Now()

	// Track all folder IDs containing new art, and hold their art IDs
	artFiles := make([]folderArtPair, 0)

	// Cache entries which have been seen previously, to reduce database load
	folderCache := map[string]*data.Folder{}
	artistCache := map[string]*data.Artist{}
	albumCache := map[string]*data.Album{}

	if verbose {
		log.Println("fs: beginning media scan:", mediaFolder)
	} else {
		log.Println("fs: scanning:", mediaFolder)
	}

	// Invoke a recursive file walk on the given media folder, passing closure variables into
	// walkFunc to enable additional functionality
	err := filepath.Walk(mediaFolder, func(currPath string, info os.FileInfo, err error) error {
		// Stop walking immediately if needed
		mutex.RLock()
		if haltWalk {
			return errors.New("media scan: halted by channel")
		}
		mutex.RUnlock()

		// Make sure path is actually valid
		if info == nil {
			return errors.New("media scan: invalid path: " + currPath)
		}

		// Check for an existing folder for this item
		folder := new(data.Folder)
		if info.IsDir() {
			// If directory, use this path
			folder.Path = currPath
		} else {
			// If file, use the directory path
			folder.Path = path.Dir(currPath)
		}

		// Check for a cached folder, or attempt to load it
		if tempFolder, ok := folderCache[folder.Path]; ok {
			folder = tempFolder
		} else if err := folder.Load(); err != nil && err == sql.ErrNoRows {
			// Make sure items actually exist at this path
			files, err := ioutil.ReadDir(folder.Path)
			if err != nil {
				log.Println(err)
				return err
			}

			// No items, skip it
			if len(files) == 0 {
				return nil
			}

			// Set short title
			folder.Title = path.Base(folder.Path)

			// Check for a parent folder
			pFolder := new(data.Folder)

			// If scan is triggered by a file, we have to check the dir twice to get parent
			if info.IsDir() {
				pFolder.Path = path.Dir(currPath)
			} else {
				pFolder.Path = path.Dir(path.Dir(currPath))
			}

			// Load parent
			if err := pFolder.Load(); err != nil && err != sql.ErrNoRows {
				log.Println(err)
				return err
			}

			// Copy parent folder's ID
			folder.ParentID = pFolder.ID

			// Save new folder
			if err := folder.Save(); err != nil {
				log.Println(err)
				return err
			}

			// Continue traversal
			folderCount++
		}

		// Cache this folder
		folderCache[folder.Path] = folder

		// Check for a valid media or art extension
		ext := path.Ext(currPath)
		if !mediaSet.Has(ext) && !artSet.Has(ext) {
			return nil
		}

		// If item is art, check for existing art
		if artSet.Has(ext) {
			// Attempt to load existing art
			art := new(data.Art)
			art.FileName = currPath
			if err := art.Load(); err == sql.ErrNoRows {
				// On new art, capture art information from filesystem
				art.FileSize = info.Size()
				art.LastModified = info.ModTime().Unix()

				// Save new art
				if err := art.Save(); err != nil {
					log.Println(err)
				} else if err == nil {
					log.Printf("Art: [#%05d] %s", art.ID, art.FileName)
					artCount++

					// Add folder ID and to new art ID to slice
					artFiles = append(artFiles, folderArtPair{
						folderID: folder.ID,
						artID:    art.ID,
					})
				}
			}

			// Continue to next file
			return nil
		}

		// Attempt to scan media file with taglib
		file, err := taglib.Read(currPath)
		if err != nil {
			log.Println(err)
			return fmt.Errorf("%s: %s", currPath, err.Error())
		}
		defer file.Close()

		// Generate a song model from the TagLib file
		song, err := data.SongFromFile(file)
		if err != nil {
			log.Println(err)
			return err
		}

		// Populate filesystem-related struct fields using OS info
		song.FileName = currPath
		song.FileSize = info.Size()
		song.LastModified = info.ModTime().Unix()

		// Use this folder's ID
		song.FolderID = folder.ID

		// Check for a valid wavepipe file type integer
		ext = path.Ext(info.Name())
		fileType, ok := data.FileTypeMap[ext]
		if !ok {
			return fmt.Errorf("fs: invalid file type: %s", ext)
		}
		song.FileTypeID = fileType

		// Generate an artist model from this song's metadata
		artist := data.ArtistFromSong(song)

		// Check for existing artist
		// Note: if the artist exists, this operation also loads necessary scanning information
		// such as their artist ID, for use in album and song generation
		if tempArtist, ok := artistCache[artist.Title]; ok {
			artist = tempArtist
		} else if err := artist.Load(); err == sql.ErrNoRows {
			// Save new artist
			if err := artist.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("Artist: [#%05d] %s", artist.ID, artist.Title)
				artistCount++
			}
		}

		// Cache this artist
		artistCache[artist.Title] = artist

		// Generate the album model from this song's metadata
		album := data.AlbumFromSong(song)
		album.ArtistID = artist.ID

		// Generate cache key
		albumCacheKey := strconv.Itoa(album.ArtistID) + "_" + album.Title

		// Check for existing album
		// Note: if the album exists, this operation also loads necessary scanning information
		// such as the album ID, for use in song generation
		if tempAlbum, ok := albumCache[albumCacheKey]; ok {
			album = tempAlbum
		} else if err := album.Load(); err == sql.ErrNoRows {
			// Save album
			if err := album.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("  - Album: [#%05d] %s - %d - %s", album.ID, album.Artist, album.Year, album.Title)
				albumCount++
			}
		}

		// Cache this album
		albumCache[albumCacheKey] = album

		// Add ID fields to song
		song.ArtistID = artist.ID
		song.AlbumID = album.ID

		// Make a duplicate song to check if song has been modified since last scan
		song2 := new(data.Song)
		song2.FileName = song.FileName

		// Check for existing song
		if err := song2.Load(); err == sql.ErrNoRows {
			// Save song (don't log these because they really slow things down)
			if err2 := song.Save(); err2 != nil && err2 != sql.ErrNoRows {
				log.Println(err2)
			} else if err2 == nil {
				songCount++
			}
		} else {
			// Song already existed.  Check if it's been updated
			if song.LastModified > song2.LastModified {
				// Update existing song
				song.ID = song2.ID
				if err2 := song.Update(); err2 != nil {
					log.Println(err2)
				}

				songUpdateCount++
			}
		}

		// Successful media scan
		return nil
	})

	// Check for filesystem walk errors
	if err != nil {
		return err
	}

	// Iterate all new folder/art ID pairs
	for _, a := range artFiles {
		// Fetch all songs for the folder from the pair
		songs, err := data.DB.SongsForFolder(a.folderID)
		if err != nil {
			return err
		}

		// Iterate and update songs with their new art ID
		for _, s := range songs {
			s.ArtID = a.artID
			if err := s.Update(); err != nil {
				return err
			}
		}
	}

	// Print metrics
	if verbose {
		log.Printf("fs: media scan complete [time: %s]", time.Since(startTime).String())
		log.Printf("fs: added: [art: %d] [artists: %d] [albums: %d] [songs: %d] [folders: %d]",
			artCount, artistCount, albumCount, songCount, folderCount)
		log.Printf("fs: updated: [songs: %d]", songUpdateCount)
	}

	// No errors
	return nil
}

// OrphanScan scans for missing "orphaned" media files in the local filesystem
func (fsFileSource) OrphanScan(baseFolder string, subFolder string, verbose bool, orphanCancelChan chan struct{}) error {
	// Halt scan if needed
	var mutex sync.RWMutex
	haltOrphanScan := false
	go func() {
		// Wait for signal
		<-orphanCancelChan

		// Halt!
		mutex.Lock()
		haltOrphanScan = true
		mutex.Unlock()
	}()

	// Track metrics about the scan
	artCount := 0
	folderCount := 0
	songCount := 0
	startTime := time.Now()

	// Check if a baseFolder is set, meaning remove ANYTHING not under this base
	if baseFolder != "" {
		if verbose {
			log.Println("fs: orphan scanning base folder:", baseFolder)
		}

		// Scan for all art NOT under the base folder
		art, err := data.DB.ArtNotInPath(baseFolder)
		if err != nil {
			log.Println(err)
			return err
		}

		// Remove all art which is not in this path
		for _, a := range art {
			// Remove art from database
			if err := a.Delete(); err != nil {
				log.Println(err)
				return err
			}

			artCount++
		}

		// Scan for all songs NOT under the base folder
		songs, err := data.DB.SongsNotInPath(baseFolder)
		if err != nil {
			log.Println(err)
			return err
		}

		// Remove all songs which are not in this path
		for _, s := range songs {
			// Remove song from database
			if err := s.Delete(); err != nil {
				log.Println(err)
				return err
			}

			songCount++
		}

		// Scan for all folders NOT under the base folder
		folders, err := data.DB.FoldersNotInPath(baseFolder)
		if err != nil {
			log.Println(err)
			return err
		}

		// Remove all folders which are not in this path
		for _, f := range folders {
			// Remove folder from database
			if err := f.Delete(); err != nil {
				log.Println(err)
				return err
			}

			folderCount++
		}
	}

	// If no subfolder set, use the base folder to check file existence
	if subFolder == "" {
		subFolder = baseFolder
	}

	if verbose {
		log.Println("fs: orphan scanning subfolder:", subFolder)
	} else {
		log.Println("fs: removing:", subFolder)
	}

	// Scan for all art in subfolder
	art, err := data.DB.ArtInPath(subFolder)
	if err != nil {
		log.Println(err)
		return err
	}

	// Iterate all art in this path
	for _, a := range art {
		// Check that the art still exists in this place
		if _, err := os.Stat(a.FileName); os.IsNotExist(err) {
			// Remove art from database
			if err := a.Delete(); err != nil {
				log.Println(err)
				return err
			}

			artCount++
		}
	}

	// Scan for all songs in subfolder
	songs, err := data.DB.SongsInPath(subFolder)
	if err != nil {
		log.Println(err)
		return err
	}

	// Iterate all songs in this path
	for _, s := range songs {
		// Check that the song still exists in this place
		if _, err := os.Stat(s.FileName); os.IsNotExist(err) {
			// Remove song from database
			if err := s.Delete(); err != nil {
				log.Println(err)
				return err
			}

			songCount++
		}
	}

	// Scan for all folders in subfolder
	folders, err := data.DB.FoldersInPath(subFolder)
	if err != nil {
		return err
	}

	// Iterate all folders in this path
	for _, f := range folders {
		// Check that the folder still has items within it
		files, err := ioutil.ReadDir(f.Path)
		if err != nil && !os.IsNotExist(err) {
			log.Println(err)
			return err
		}

		// Delete any folders with 0 items
		if len(files) == 0 {
			if err := f.Delete(); err != nil {
				log.Println(err)
				return err
			}

			folderCount++
		}
	}

	// Now that songs have been purged, check for albums
	albumCount, err := data.DB.PurgeOrphanAlbums()
	if err != nil {
		log.Println(err)
		return err
	}

	// Check for artists
	artistCount, err := data.DB.PurgeOrphanArtists()
	if err != nil {
		log.Println(err)
		return err
	}

	// Print metrics
	if verbose {
		log.Printf("fs: orphan scan complete [time: %s]", time.Since(startTime).String())
		log.Printf("fs: removed: [art: %d] [artists: %d] [albums: %d] [songs: %d] [folders: %d]",
			artCount, artistCount, albumCount, songCount, folderCount)
	}
	return nil
}
<commit_msg>Don't print art file discovery during scans<commit_after>package core

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/mdlayher/wavepipe/data"

	"github.com/wtolson/go-taglib"
)

// fsFileSource represents a file source which indexes files in the local filesystem
type fsFileSource struct{}

// folderArtPair contains a folder ID and associated art ID
type folderArtPair struct {
	folderID int
	artID    int
}

// MediaScan scans for media files in the local filesystem
func (fsFileSource) MediaScan(mediaFolder string, verbose bool, walkCancelChan chan struct{}) error {
	// Halt walk if needed
	var mutex sync.RWMutex
	haltWalk := false
	go func() {
		// Wait for signal
		<-walkCancelChan

		// Halt!
		mutex.Lock()
		haltWalk = true
		mutex.Unlock()
	}()

	// Track metrics about the walk
	artCount := 0
	artistCount := 0
	albumCount := 0
	songCount := 0
	songUpdateCount := 0
	folderCount := 0
	startTime := time.Now()

	// Track all folder IDs containing new art, and hold their art IDs
	artFiles := make([]folderArtPair, 0)

	// Cache entries which have been seen previously, to reduce database load
	folderCache := map[string]*data.Folder{}
	artistCache := map[string]*data.Artist{}
	albumCache := map[string]*data.Album{}

	if verbose {
		log.Println("fs: beginning media scan:", mediaFolder)
	} else {
		log.Println("fs: scanning:", mediaFolder)
	}

	// Invoke a recursive file walk on the given media folder, passing closure variables into
	// walkFunc to enable additional functionality
	err := filepath.Walk(mediaFolder, func(currPath string, info os.FileInfo, err error) error {
		// Stop walking immediately if needed
		mutex.RLock()
		if haltWalk {
			return errors.New("media scan: halted by channel")
		}
		mutex.RUnlock()

		// Make sure path is actually valid
		if info == nil {
			return errors.New("media scan: invalid path: " + currPath)
		}

		// Check for an existing folder for this item
		folder := new(data.Folder)
		if info.IsDir() {
			// If directory, use this path
			folder.Path = currPath
		} else {
			// If file, use the directory path
			folder.Path = path.Dir(currPath)
		}

		// Check for a cached folder, or attempt to load it
		if tempFolder, ok := folderCache[folder.Path]; ok {
			folder = tempFolder
		} else if err := folder.Load(); err != nil && err == sql.ErrNoRows {
			// Make sure items actually exist at this path
			files, err := ioutil.ReadDir(folder.Path)
			if err != nil {
				log.Println(err)
				return err
			}

			// No items, skip it
			if len(files) == 0 {
				return nil
			}

			// Set short title
			folder.Title = path.Base(folder.Path)

			// Check for a parent folder
			pFolder := new(data.Folder)

			// If scan is triggered by a file, we have to check the dir twice to get parent
			if info.IsDir() {
				pFolder.Path = path.Dir(currPath)
			} else {
				pFolder.Path = path.Dir(path.Dir(currPath))
			}

			// Load parent
			if err := pFolder.Load(); err != nil && err != sql.ErrNoRows {
				log.Println(err)
				return err
			}

			// Copy parent folder's ID
			folder.ParentID = pFolder.ID

			// Save new folder
			if err := folder.Save(); err != nil {
				log.Println(err)
				return err
			}

			// Continue traversal
			folderCount++
		}

		// Cache this folder
		folderCache[folder.Path] = folder

		// Check for a valid media or art extension
		ext := path.Ext(currPath)
		if !mediaSet.Has(ext) && !artSet.Has(ext) {
			return nil
		}

		// If item is art, check for existing art
		if artSet.Has(ext) {
			// Attempt to load existing art
			art := new(data.Art)
			art.FileName = currPath
			if err := art.Load(); err == sql.ErrNoRows {
				// On new art, capture art information from filesystem
				art.FileSize = info.Size()
				art.LastModified = info.ModTime().Unix()

				// Save new art
				if err := art.Save(); err != nil {
					log.Println(err)
				} else if err == nil {
					artCount++

					// Add folder ID and to new art ID to slice
					artFiles = append(artFiles, folderArtPair{
						folderID: folder.ID,
						artID:    art.ID,
					})
				}
			}

			// Continue to next file
			return nil
		}

		// Attempt to scan media file with taglib
		file, err := taglib.Read(currPath)
		if err != nil {
			log.Println(err)
			return fmt.Errorf("%s: %s", currPath, err.Error())
		}
		defer file.Close()

		// Generate a song model from the TagLib file
		song, err := data.SongFromFile(file)
		if err != nil {
			log.Println(err)
			return err
		}

		// Populate filesystem-related struct fields using OS info
		song.FileName = currPath
		song.FileSize = info.Size()
		song.LastModified = info.ModTime().Unix()

		// Use this folder's ID
		song.FolderID = folder.ID

		// Check for a valid wavepipe file type integer
		ext = path.Ext(info.Name())
		fileType, ok := data.FileTypeMap[ext]
		if !ok {
			return fmt.Errorf("fs: invalid file type: %s", ext)
		}
		song.FileTypeID = fileType

		// Generate an artist model from this song's metadata
		artist := data.ArtistFromSong(song)

		// Check for existing artist
		// Note: if the artist exists, this operation also loads necessary scanning information
		// such as their artist ID, for use in album and song generation
		if tempArtist, ok := artistCache[artist.Title]; ok {
			artist = tempArtist
		} else if err := artist.Load(); err == sql.ErrNoRows {
			// Save new artist
			if err := artist.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("Artist: [#%05d] %s", artist.ID, artist.Title)
				artistCount++
			}
		}

		// Cache this artist
		artistCache[artist.Title] = artist

		// Generate the album model from this song's metadata
		album := data.AlbumFromSong(song)
		album.ArtistID = artist.ID

		// Generate cache key
		albumCacheKey := strconv.Itoa(album.ArtistID) + "_" + album.Title

		// Check for existing album
		// Note: if the album exists, this operation also loads necessary scanning information
		// such as the album ID, for use in song generation
		if tempAlbum, ok := albumCache[albumCacheKey]; ok {
			album = tempAlbum
		} else if err := album.Load(); err == sql.ErrNoRows {
			// Save album
			if err := album.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("  - Album: [#%05d] %s - %d - %s", album.ID, album.Artist, album.Year, album.Title)
				albumCount++
			}
		}

		// Cache this album
		albumCache[albumCacheKey] = album

		// Add ID fields to song
		song.ArtistID = artist.ID
		song.AlbumID = album.ID

		// Make a duplicate song to check if song has been modified since last scan
		song2 := new(data.Song)
		song2.FileName = song.FileName

		// Check for existing song
		if err := song2.Load(); err == sql.ErrNoRows {
			// Save song (don't log these because they really slow things down)
			if err2 := song.Save(); err2 != nil && err2 != sql.ErrNoRows {
				log.Println(err2)
			} else if err2 == nil {
				songCount++
			}
		} else {
			// Song already existed.  Check if it's been updated
			if song.LastModified > song2.LastModified {
				// Update existing song
				song.ID = song2.ID
				if err2 := song.Update(); err2 != nil {
					log.Println(err2)
				}

				songUpdateCount++
			}
		}

		// Successful media scan
		return nil
	})

	// Check for filesystem walk errors
	if err != nil {
		return err
	}

	// Iterate all new folder/art ID pairs
	for _, a := range artFiles {
		// Fetch all songs for the folder from the pair
		songs, err := data.DB.SongsForFolder(a.folderID)
		if err != nil {
			return err
		}

		// Iterate and update songs with their new art ID
		for _, s := range songs {
			s.ArtID = a.artID
			if err := s.Update(); err != nil {
				return err
			}
		}
	}

	// Print metrics
	if verbose {
		log.Printf("fs: media scan complete [time: %s]", time.Since(startTime).String())
		log.Printf("fs: added: [art: %d] [artists: %d] [albums: %d] [songs: %d] [folders: %d]",
			artCount, artistCount, albumCount, songCount, folderCount)
		log.Printf("fs: updated: [songs: %d]", songUpdateCount)
	}

	// No errors
	return nil
}

// OrphanScan scans for missing "orphaned" media files in the local filesystem
func (fsFileSource) OrphanScan(baseFolder string, subFolder string, verbose bool, orphanCancelChan chan struct{}) error {
	// Halt scan if needed
	var mutex sync.RWMutex
	haltOrphanScan := false
	go func() {
		// Wait for signal
		<-orphanCancelChan

		// Halt!
		mutex.Lock()
		haltOrphanScan = true
		mutex.Unlock()
	}()

	// Track metrics about the scan
	artCount := 0
	folderCount := 0
	songCount := 0
	startTime := time.Now()

	// Check if a baseFolder is set, meaning remove ANYTHING not under this base
	if baseFolder != "" {
		if verbose {
			log.Println("fs: orphan scanning base folder:", baseFolder)
		}

		// Scan for all art NOT under the base folder
		art, err := data.DB.ArtNotInPath(baseFolder)
		if err != nil {
			log.Println(err)
			return err
		}

		// Remove all art which is not in this path
		for _, a := range art {
			// Remove art from database
			if err := a.Delete(); err != nil {
				log.Println(err)
				return err
			}

			artCount++
		}

		// Scan for all songs NOT under the base folder
		songs, err := data.DB.SongsNotInPath(baseFolder)
		if err != nil {
			log.Println(err)
			return err
		}

		// Remove all songs which are not in this path
		for _, s := range songs {
			// Remove song from database
			if err := s.Delete(); err != nil {
				log.Println(err)
				return err
			}

			songCount++
		}

		// Scan for all folders NOT under the base folder
		folders, err := data.DB.FoldersNotInPath(baseFolder)
		if err != nil {
			log.Println(err)
			return err
		}

		// Remove all folders which are not in this path
		for _, f := range folders {
			// Remove folder from database
			if err := f.Delete(); err != nil {
				log.Println(err)
				return err
			}

			folderCount++
		}
	}

	// If no subfolder set, use the base folder to check file existence
	if subFolder == "" {
		subFolder = baseFolder
	}

	if verbose {
		log.Println("fs: orphan scanning subfolder:", subFolder)
	} else {
		log.Println("fs: removing:", subFolder)
	}

	// Scan for all art in subfolder
	art, err := data.DB.ArtInPath(subFolder)
	if err != nil {
		log.Println(err)
		return err
	}

	// Iterate all art in this path
	for _, a := range art {
		// Check that the art still exists in this place
		if _, err := os.Stat(a.FileName); os.IsNotExist(err) {
			// Remove art from database
			if err := a.Delete(); err != nil {
				log.Println(err)
				return err
			}

			artCount++
		}
	}

	// Scan for all songs in subfolder
	songs, err := data.DB.SongsInPath(subFolder)
	if err != nil {
		log.Println(err)
		return err
	}

	// Iterate all songs in this path
	for _, s := range songs {
		// Check that the song still exists in this place
		if _, err := os.Stat(s.FileName); os.IsNotExist(err) {
			// Remove song from database
			if err := s.Delete(); err != nil {
				log.Println(err)
				return err
			}

			songCount++
		}
	}

	// Scan for all folders in subfolder
	folders, err := data.DB.FoldersInPath(subFolder)
	if err != nil {
		return err
	}

	// Iterate all folders in this path
	for _, f := range folders {
		// Check that the folder still has items within it
		files, err := ioutil.ReadDir(f.Path)
		if err != nil && !os.IsNotExist(err) {
			log.Println(err)
			return err
		}

		// Delete any folders with 0 items
		if len(files) == 0 {
			if err := f.Delete(); err != nil {
				log.Println(err)
				return err
			}

			folderCount++
		}
	}

	// Now that songs have been purged, check for albums
	albumCount, err := data.DB.PurgeOrphanAlbums()
	if err != nil {
		log.Println(err)
		return err
	}

	// Check for artists
	artistCount, err := data.DB.PurgeOrphanArtists()
	if err != nil {
		log.Println(err)
		return err
	}

	// Print metrics
	if verbose {
		log.Printf("fs: orphan scan complete [time: %s]", time.Since(startTime).String())
		log.Printf("fs: removed: [art: %d] [artists: %d] [albums: %d] [songs: %d] [folders: %d]",
			artCount, artistCount, albumCount, songCount, folderCount)
	}
	return nil
}
<|endoftext|>


<commit_before>package soundtrack

import (
	"bytes"
	"fmt"
	"net/http"
	"reflect"
	"strings"

	"github.com/animenotifier/notify.moe/components"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
)

// Edit track.
func Edit(ctx *aero.Context) string {
	id := ctx.Get("id")
	track, err := arn.GetSoundTrack(id)

	if err != nil {
		return ctx.Error(http.StatusNotFound, "Track not found", err)
	}

	ctx.Data = &arn.OpenGraph{
		Tags: map[string]string{
			"og:title":     track.Title,
			"og:image":     track.MainAnime().Image.Large,
			"og:url":       "https://" + ctx.App.Config.Domain + track.Link(),
			"og:site_name": "notify.moe",
			"og:type":      "music.song",
		},
	}

	return ctx.HTML(EditForm(track, "Edit soundtrack"))
}

// EditForm ...
func EditForm(obj interface{}, title string) string {
	t := reflect.TypeOf(obj).Elem()
	v := reflect.ValueOf(obj).Elem()
	id := reflect.Indirect(v.FieldByName("ID"))
	lowerCaseTypeName := strings.ToLower(t.Name())

	var b bytes.Buffer
	b.WriteString(`<div class="widget-form">`)
	b.WriteString(`<div class="widget" data-api="/api/` + lowerCaseTypeName + `/` + id.String() + `">`)
	b.WriteString(`<h1>`)
	b.WriteString(title)
	b.WriteString(`</h1>`)

	RenderObject(&b, obj, "")

	b.WriteString("
")
	b.WriteString("
")

	return b.String()
}

// RenderObject ...
func RenderObject(b *bytes.Buffer, obj interface{}, idPrefix string) {
	t := reflect.TypeOf(obj).Elem()
	v := reflect.ValueOf(obj).Elem()

	// Fields
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		RenderField(b, &v, field, idPrefix)
	}
}

// RenderField ...
func RenderField(b *bytes.Buffer, v *reflect.Value, field reflect.StructField, idPrefix string) {
	if field.Anonymous || field.Tag.Get("editable") != "true" {
		return
	}

	fieldValue := reflect.Indirect(v.FieldByName(field.Name))

	switch field.Type.String() {
	case "string":
		b.WriteString(components.InputText(idPrefix+field.Name, fieldValue.String(), field.Name, ""))
	case "[]string":
		b.WriteString(components.InputTags(idPrefix+field.Name, fieldValue.Interface().([]string), field.Name))
	case "[]*arn.ExternalMedia":
		for sliceIndex := 0; sliceIndex < fieldValue.Len(); sliceIndex++ {
			arrayObj := fieldValue.Index(sliceIndex).Interface()
			arrayIDPrefix := fmt.Sprintf("%s[%d].", field.Name, sliceIndex)
			RenderObject(b, arrayObj, arrayIDPrefix)
		}
	default:
		panic("No edit form implementation for " + idPrefix + field.Name + " with type " + field.Type.String())
	}
}
<commit_msg>Add media button<commit_after>package soundtrack

import (
	"bytes"
	"fmt"
	"net/http"
	"reflect"
	"strings"

	"github.com/animenotifier/notify.moe/components"
	"github.com/animenotifier/notify.moe/utils"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
)

// Edit track.
func Edit(ctx *aero.Context) string {
	id := ctx.Get("id")
	track, err := arn.GetSoundTrack(id)

	if err != nil {
		return ctx.Error(http.StatusNotFound, "Track not found", err)
	}

	ctx.Data = &arn.OpenGraph{
		Tags: map[string]string{
			"og:title":     track.Title,
			"og:image":     track.MainAnime().Image.Large,
			"og:url":       "https://" + ctx.App.Config.Domain + track.Link(),
			"og:site_name": "notify.moe",
			"og:type":      "music.song",
		},
	}

	return ctx.HTML(EditForm(track, "Edit soundtrack"))
}

// EditForm ...
func EditForm(obj interface{}, title string) string {
	t := reflect.TypeOf(obj).Elem()
	v := reflect.ValueOf(obj).Elem()
	id := reflect.Indirect(v.FieldByName("ID"))
	lowerCaseTypeName := strings.ToLower(t.Name())

	var b bytes.Buffer
	b.WriteString(`<div class="widget-form">`)
	b.WriteString(`<div class="widget" data-api="/api/` + lowerCaseTypeName + `/` + id.String() + `">`)
	b.WriteString(`<h1>`)
	b.WriteString(title)
	b.WriteString(`</h1>`)

	RenderObject(&b, obj, "")

	b.WriteString("
")
	b.WriteString("
")

	return b.String()
}

// RenderObject ...
func RenderObject(b *bytes.Buffer, obj interface{}, idPrefix string) {
	t := reflect.TypeOf(obj).Elem()
	v := reflect.ValueOf(obj).Elem()

	// Fields
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		RenderField(b, &v, field, idPrefix)
	}
}

// RenderField ...
func RenderField(b *bytes.Buffer, v *reflect.Value, field reflect.StructField, idPrefix string) {
	if field.Anonymous || field.Tag.Get("editable") != "true" {
		return
	}

	fieldValue := reflect.Indirect(v.FieldByName(field.Name))

	switch field.Type.String() {
	case "string":
		b.WriteString(components.InputText(idPrefix+field.Name, fieldValue.String(), field.Name, ""))
	case "[]string":
		b.WriteString(components.InputTags(idPrefix+field.Name, fieldValue.Interface().([]string), field.Name))
	case "[]*arn.ExternalMedia":
		for sliceIndex := 0; sliceIndex < fieldValue.Len(); sliceIndex++ {
			arrayObj := fieldValue.Index(sliceIndex).Interface()
			arrayIDPrefix := fmt.Sprintf("%s[%d].", field.Name, sliceIndex)
			RenderObject(b, arrayObj, arrayIDPrefix)
		}

		b.WriteString(`<div class="buttons"><button>` + utils.Icon("plus") + `Add ` + field.Name + `</button></div>`)
	default:
		panic("No edit form implementation for " + idPrefix + field.Name + " with type " + field.Type.String())
	}
}
<|endoftext|>


<commit_before>package main

import "database/sql"

type SQLDatabase struct {
	sql.DB
	// The string written to `streams.server` of streams owned by this server.
	localhost string
	// Security tokens of active streams owned by this server.
	// Some broadcasting software (*cough* gstreamer *cough*) wraps each frame
	// in a separate request, which may or may not overload the database...
	streamTokenCache map[string]string
}

var SQLDatabaseSchema = `
create table if not exists users (
    id                integer  not null,
    activated         integer  not null,
    activation_token  text     not null,
    name              text     not null,
    email             text     not null,
    display_name      text     not null,
    about             text     not null,
    password          blob     not null,

    primary key (id), unique (name), unique (email)
);

create table if not exists streams (
    id      integer  not null,
    name    text     not null,
    about   text     not null,
    token   text     not null,
    server  text,

    primary key (id)
);`

func NewSQLDatabase(localhost string, driver string, server string) (Database, error) {
	db, err := sql.Open(driver, server)
	if err != nil {
		return nil, err
	}
	wrapped := &SQLDatabase{*db, localhost, make(map[string]string)}
	if err = wrapped.Ping(); err != nil {
		wrapped.Close()
		return nil, err
	}
	if _, err = wrapped.Exec(SQLDatabaseSchema); err != nil {
		wrapped.Close()
		return nil, err
	}
	return wrapped, nil
}

func (d *SQLDatabase) NewUser(name string, email string, password []byte) (*UserMetadata, error) {
	if err := validateUsername(name); err != nil {
		return nil, err
	}
	if err := validateEmail(email); err != nil {
		return nil, err
	}
	hash, err := generatePwHash(password)
	if err != nil {
		return nil, err
	}
	activationToken := makeToken(defaultTokenLength)
	streamToken := makeToken(defaultTokenLength)
	r, err := d.Exec(
		`begin;
		 insert into users   values (NULL, 0, ?, ?, ?, ?, "", ?);
		 insert into streams values (NULL, "", "", ?, NULL);
		 commit;`,
		activationToken, name, email, name, hash, streamToken,
	)
	if err != nil {
		var i int
		q := d.QueryRow(`select 1 from users where name = ? or email = ?`, name, email)
		if q.Scan(&i) != sql.ErrNoRows {
			return nil, ErrUserNotUnique
		}
		return nil, err
	}

	uid, err := r.LastInsertId()
	if err != nil {
		// FIXME uh...
		return nil, err
	}

	return &UserMetadata{
		UserShortData{uid, name, email, name, hash}, "", false, activationToken, streamToken,
	}, nil
}

func (d *SQLDatabase) ActivateUser(id int64, token string) error {
	r, err := d.Exec(
		`update users set activated = 1 where id = ? and activation_token = ?`,
		id, token,
	)
	if err != nil {
		return err
	}

	changed, err := r.RowsAffected()
	if err != nil {
		return err
	}

	if changed != 1 {
		return ErrInvalidToken
	}
	return nil
}

func (d *SQLDatabase) GetUserID(name string, password []byte) (int64, error) {
	var meta UserShortData
	err := d.QueryRow(
		`select id, password from users where name = ?`, name,
	).Scan(&meta.ID, &meta.PwHash)
	if err != nil {
		if err == sql.ErrNoRows {
			return 0, ErrUserNotExist
		}
		return 0, err
	}
	return meta.ID, meta.CheckPassword(password)
}

func (d *SQLDatabase) GetUserShort(id int64) (*UserShortData, error) {
	meta := UserShortData{ID: id}
	err := d.QueryRow(
		`select name, password, display_name, email from users where users.id = ?`, id,
	).Scan(&meta.Login, &meta.PwHash, &meta.Name, &meta.Email)
	if err == sql.ErrNoRows {
		return nil, ErrUserNotExist
	}
	return &meta, err
}

func (d *SQLDatabase) GetUserFull(id int64) (*UserMetadata, error) {
	meta := UserMetadata{UserShortData: UserShortData{ID: id}}
	err := d.QueryRow(
		`select users.name, password, email, display_name, users.about,
		        activated, activation_token, streams.token from users, streams
		 where users.id = ? and streams.id = users.id`,
		id,
	).Scan(
		&meta.Login, &meta.PwHash, &meta.Email, &meta.Name, &meta.About,
		&meta.Activated, &meta.ActivationToken, &meta.StreamToken,
	)
	if err == sql.ErrNoRows {
		return nil, ErrUserNotExist
	}
	return &meta, err
}

func (d *SQLDatabase) SetUserMetadata(id int64, name string, displayName string, email string, about string, password []byte) (string, error) {
	token := ""
	query := "update users set "
	params := make([]interface{}, 0, 6)

	if name != "" {
		if err := validateUsername(name); err != nil {
			return "", err
		}
		query += "name = ?, "
		params = append(params, name)
	}

	if displayName != "" {
		query += "display_name = ?, "
		params = append(params, displayName)
	}

	if email != "" {
		if err := validateEmail(email); err != nil {
			return "", err
		}
		token = makeToken(defaultTokenLength)
		query += "activated = 0, activation_token = ?, email = ?, "
		params = append(params, token, email)
	}

	if len(password) != 0 {
		hash, err := generatePwHash(password)
		if err != nil {
			return "", err
		}
		query += "password = ?, "
		params = append(params, hash)
	}

	query += "about = ? where id in (select id from streams where id = ? and server is null)"
	params = append(params, about, id)

	r, err := d.Exec(query, params...)
	if err != nil {
		if name != "" || email != "" {
			var i int
			q := d.QueryRow(`select 1 from users where name = ? or email = ?`, name, email)
			if q.Scan(&i) != sql.ErrNoRows {
				return "", ErrUserNotUnique
			}
		}
		return "", err
	}
	rows, err := r.RowsAffected()
	if err != nil {
		return "", err
	}
	if rows != 1 {
		return "", ErrStreamActive
	}
	return token, err
}

func (d *SQLDatabase) SetStreamName(id int64, name string) error {
	_, err := d.Exec(`update streams set name = ? where id = ?`, name, id)
	return err
}

func (d *SQLDatabase) SetStreamAbout(id int64, about string) error {
	_, err := d.Exec(`update streams set about = ? where id = ?`, about, id)
	return err
}

func (d *SQLDatabase) NewStreamToken(id int64) error {
	// TODO invalidate token cache on all nodes
	//      damn, it appears I ran into the most difficult problem...
	token := makeToken(defaultTokenLength)
	_, err := d.Exec(`update streams set token = ? where id = ?`, token, id)
	return err
}

func (d *SQLDatabase) StartStream(user string, token string) error {
	if expect, ok := d.streamTokenCache[user]; ok {
		if expect != token {
			return ErrInvalidToken
		}
		return nil
	}

	var id int64
	var expect string
	var server sql.NullString

	err := d.QueryRow(`select id from users where name = ? and activated = 1`, user).Scan(&id)
	if err != nil {
		if err == sql.ErrNoRows {
			return ErrStreamNotExist
		}
		return err
	}

	_, err = d.Exec(
		`update streams set server = ? where id = ? and server is null and token = ?`,
		d.localhost, id, token,
	)
	if err != nil {
		return err
	}

	err = d.QueryRow(`select token, server from streams where id = ?`, id).Scan(&expect, &server)
	if err != nil {
		return err
	}

	if expect != token {
		return ErrInvalidToken
	}
	if !server.Valid || server.String != d.localhost {
		return ErrStreamNotHere
	}
	d.streamTokenCache[user] = expect
	return nil
}

func (d *SQLDatabase) StopStream(user string) error {
	_, err := d.Exec(
		`update streams set server = NULL where id in (select id from users where name = ?)`,
		user,
	)
	if err != nil {
		return err
	}
	delete(d.streamTokenCache, user)
	return nil
}

func (d *SQLDatabase) GetStreamServer(user string) (string, error) {
	if _, ok := d.streamTokenCache[user]; ok {
		return d.localhost, nil
	}

	var server sql.NullString
	err := d.QueryRow(
		`select server from streams where id in (select id from users where name = ?)`,
		user,
	).Scan(&server)
	if err != nil {
		if err == sql.ErrNoRows {
			return "", ErrStreamNotExist
		}
		return "", err
	}
	if server.String == d.localhost {
		// it should have been offline??
		return "", ErrStreamActive
	}
	if !server.Valid {
		return "", ErrStreamOffline
	}
	return server.String, ErrStreamNotHere
}

func (d *SQLDatabase) GetStreamMetadata(user string) (*StreamMetadata, error) {
	var server sql.NullString
	meta := StreamMetadata{}
	err := d.QueryRow(
		`select display_name, users.about, email, streams.name, streams.about, streams.server
		 from   users, streams
		 where  users.name = ? and streams.id = users.id`,
		user,
	).Scan(&meta.UserName, &meta.UserAbout, &meta.Email, &meta.Name, &meta.About, &server)
	if err == sql.ErrNoRows {
		return nil, ErrStreamNotExist
	}
	if err != nil {
		return nil, err
	}
	meta.Server = server.String
	if !server.Valid {
		return &meta, ErrStreamOffline
	}
	return &meta, nil
}
<commit_msg>Streamline the SQL schema.<commit_after>package main

import "database/sql"

type SQLDatabase struct {
	sql.DB
	// The string written to `streams.server` of streams owned by this server.
	localhost string
	// Security tokens of active streams owned by this server.
	// Some broadcasting software (*cough* gstreamer *cough*) wraps each frame
	// in a separate request, which may or may not overload the database...
	streamTokenCache map[string]string
}

var SQLDatabaseSchema = `
create table if not exists users (
    id                integer      not null,
    activated         integer      not null,
    activation_token  varchar(64)  not null,
    name              varchar(256) not null,
    email             varchar(256) not null,
    display_name      varchar(256) not null,
    about             text         not null,
    password          varchar(256) not null,
    stream_name       varchar(256) not null,
    stream_about      text         not null,
    stream_token      varchar(64)  not null,
    stream_server     varchar(128),

    primary key (id), unique (name), unique (email)
);`

func NewSQLDatabase(localhost string, driver string, server string) (Database, error) {
	db, err := sql.Open(driver, server)
	if err == nil {
		wrapped := &SQLDatabase{*db, localhost, make(map[string]string)}
		if _, err = wrapped.Exec(SQLDatabaseSchema); err == nil {
			return wrapped, nil
		}
		wrapped.Close()
	}
	return nil, err
}

func (d *SQLDatabase) NewUser(name string, email string, password []byte) (*UserMetadata, error) {
	if err := validateUsername(name); err != nil {
		return nil, err
	}
	if err := validateEmail(email); err != nil {
		return nil, err
	}
	hash, err := generatePwHash(password)
	if err != nil {
		return nil, err
	}
	activationToken := makeToken(defaultTokenLength)
	streamToken := makeToken(defaultTokenLength)
	r, err := d.Exec(
		`insert into users   values (NULL, 0, ?, ?, ?, ?, "", ?, "", "", ?, NULL);`,
		activationToken, name, email, name, hash, streamToken,
	)
	if err != nil {
		var i int
		q := d.QueryRow(`select 1 from users where name = ? or email = ?`, name, email)
		if q.Scan(&i) != sql.ErrNoRows {
			return nil, ErrUserNotUnique
		}
		return nil, err
	}

	uid, err := r.LastInsertId()
	if err != nil {
		// FIXME uh...
		return nil, err
	}

	return &UserMetadata{
		UserShortData{uid, name, email, name, hash}, "", false, activationToken, streamToken,
	}, nil
}

func (d *SQLDatabase) ActivateUser(id int64, token string) error {
	r, err := d.Exec(
		`update users set activated = 1 where id = ? and activation_token = ?`,
		id, token,
	)
	if err != nil {
		return err
	}

	changed, err := r.RowsAffected()
	if err != nil {
		return err
	}

	if changed != 1 {
		return ErrInvalidToken
	}
	return nil
}

func (d *SQLDatabase) GetUserID(name string, password []byte) (int64, error) {
	var meta UserShortData
	err := d.QueryRow(
		`select id, password from users where name = ?`, name,
	).Scan(&meta.ID, &meta.PwHash)
	if err != nil {
		if err == sql.ErrNoRows {
			return 0, ErrUserNotExist
		}
		return 0, err
	}
	return meta.ID, meta.CheckPassword(password)
}

func (d *SQLDatabase) GetUserShort(id int64) (*UserShortData, error) {
	meta := UserShortData{ID: id}
	err := d.QueryRow(
		`select name, password, display_name, email from users where users.id = ?`, id,
	).Scan(&meta.Login, &meta.PwHash, &meta.Name, &meta.Email)
	if err == sql.ErrNoRows {
		return nil, ErrUserNotExist
	}
	return &meta, err
}

func (d *SQLDatabase) GetUserFull(id int64) (*UserMetadata, error) {
	meta := UserMetadata{UserShortData: UserShortData{ID: id}}
	err := d.QueryRow(
		`select name, password, email, display_name, about, activated,
		        activation_token, stream_token from users where users.id = ?`,
		id,
	).Scan(
		&meta.Login, &meta.PwHash, &meta.Email, &meta.Name, &meta.About,
		&meta.Activated, &meta.ActivationToken, &meta.StreamToken,
	)
	if err == sql.ErrNoRows {
		return nil, ErrUserNotExist
	}
	return &meta, err
}

func (d *SQLDatabase) SetUserMetadata(id int64, name string, displayName string, email string, about string, password []byte) (string, error) {
	token := ""
	query := "update users set "
	params := make([]interface{}, 0, 6)

	if name != "" {
		if err := validateUsername(name); err != nil {
			return "", err
		}
		query += "name = ?, "
		params = append(params, name)
	}

	if displayName != "" {
		query += "display_name = ?, "
		params = append(params, displayName)
	}

	if email != "" {
		if err := validateEmail(email); err != nil {
			return "", err
		}
		token = makeToken(defaultTokenLength)
		query += "activated = 0, activation_token = ?, email = ?, "
		params = append(params, token, email)
	}

	if len(password) != 0 {
		hash, err := generatePwHash(password)
		if err != nil {
			return "", err
		}
		query += "password = ?, "
		params = append(params, hash)
	}

	query += "about = ? where id = ? and stream_server is null"
	params = append(params, about, id)

	r, err := d.Exec(query, params...)
	if err != nil {
		if name != "" || email != "" {
			var i int
			q := d.QueryRow(`select 1 from users where name = ? or email = ?`, name, email)
			if q.Scan(&i) != sql.ErrNoRows {
				return "", ErrUserNotUnique
			}
		}
		return "", err
	}
	rows, err := r.RowsAffected()
	if err != nil {
		return "", err
	}
	if rows != 1 {
		return "", ErrStreamActive
	}
	return token, err
}

func (d *SQLDatabase) SetStreamName(id int64, name string) error {
	_, err := d.Exec(`update users set stream_name = ? where id = ?`, name, id)
	return err
}

func (d *SQLDatabase) SetStreamAbout(id int64, about string) error {
	_, err := d.Exec(`update users set stream_about = ? where id = ?`, about, id)
	return err
}

func (d *SQLDatabase) NewStreamToken(id int64) error {
	// TODO invalidate token cache on all nodes
	//      damn, it appears I ran into the most difficult problem...
	token := makeToken(defaultTokenLength)
	_, err := d.Exec(`update users set stream_token = ? where id = ?`, token, id)
	return err
}

func (d *SQLDatabase) StartStream(user string, token string) error {
	if expect, ok := d.streamTokenCache[user]; ok {
		if expect != token {
			return ErrInvalidToken
		}
		return nil
	}

	var id int64
	var expect string
	var server sql.NullString

	err := d.QueryRow(`select id from users where name = ? and activated = 1`, user).Scan(&id)
	if err != nil {
		if err == sql.ErrNoRows {
			return ErrStreamNotExist
		}
		return err
	}

	_, err = d.Exec(
		`update users set stream_server = ? where id = ? and stream_server is null and stream_token = ?`,
		d.localhost, id, token,
	)
	if err != nil {
		return err
	}

	err = d.QueryRow(
		`select stream_token, stream_server from users where id = ?`, id,
	).Scan(&expect, &server)
	if err != nil {
		return err
	}

	if expect != token {
		return ErrInvalidToken
	}
	if !server.Valid || server.String != d.localhost {
		return ErrStreamNotHere
	}
	d.streamTokenCache[user] = expect
	return nil
}

func (d *SQLDatabase) StopStream(user string) error {
	_, err := d.Exec(`update users set stream_server = NULL where name = ?`, user)
	if err != nil {
		return err
	}
	delete(d.streamTokenCache, user)
	return nil
}

func (d *SQLDatabase) GetStreamServer(user string) (string, error) {
	if _, ok := d.streamTokenCache[user]; ok {
		return d.localhost, nil
	}

	var server sql.NullString
	err := d.QueryRow(`select stream_server from users where name = ?`, user).Scan(&server)
	if err != nil {
		if err == sql.ErrNoRows {
			return "", ErrStreamNotExist
		}
		return "", err
	}
	if server.String == d.localhost {
		// it should have been offline??
		return "", ErrStreamActive
	}
	if !server.Valid {
		return "", ErrStreamOffline
	}
	return server.String, ErrStreamNotHere
}

func (d *SQLDatabase) GetStreamMetadata(user string) (*StreamMetadata, error) {
	var server sql.NullString
	meta := StreamMetadata{}
	err := d.QueryRow(
		`select display_name, about, email, stream_name, stream_about, stream_server
		 from users where users.name = ?`,
		user,
	).Scan(&meta.UserName, &meta.UserAbout, &meta.Email, &meta.Name, &meta.About, &server)
	if err == sql.ErrNoRows {
		return nil, ErrStreamNotExist
	}
	if err != nil {
		return nil, err
	}
	meta.Server = server.String
	if !server.Valid {
		return &meta, ErrStreamOffline
	}
	return &meta, nil
}
<|endoftext|>


<commit_before>package datastore

import (
	"encoding/json"
	"fmt"
	"github.com/grafov/m3u8"
	"log"
	"strconv"
	"time"
)

// struct Channel describes a broadcast channel
type Channel struct {
	Identifier      string // Identifier is the unique part of the redis key
	PlaybackCounter int64
	mediaPlaylist   *m3u8.MediaPlaylist
}

// function GetChannel retrieves Channel information from the datastore
// given a channel identifier. It returns the channel and any error
func GetChannel(channelId string) (rv *Channel, err error) {
	rv = &Channel{}
	rv.Identifier = channelId
	rv.PlaybackCounter, err = client.IncrBy(rv.PlaybackCounterKey(), 0).Result()
	rv.mediaPlaylist, _ = m3u8.NewMediaPlaylist(50, 50)
	return
}

func (c *Channel) PlaylistData() string {
	return c.mediaPlaylist.Encode().String()
}

func (c Channel) CurrentItem() (PlaylistItem, error) {
	jsonString, err := client.LIndex(c.PlaybackQueueKey(), 0).Result()
	if err != nil {
		return PlaylistItem{}, err
	}

	item := PlaylistItem{}
	if err := json.Unmarshal([]byte(jsonString), &item); err != nil {
		return PlaylistItem{}, err
	}

	return item, nil
}

func (c Channel) PushItem(i *PlaylistItem) error {
	jsonBytes, err := json.Marshal(i)
	if err != nil {
		return err
	}

	err = client.RPush(c.PlaybackQueueKey(), string(jsonBytes)).Err()
	return err
}

// function AdvanceCounter advances the playback counter, and updates the datastore
func (c *Channel) AdvanceCounter() error {
	currentItem, err := c.CurrentItem()
	if err != nil {
		return err
	}

	if currentItem.StartAt > c.PlaybackCounter {
		c.PlaybackCounter = currentItem.StartAt
	}
	videoFile := fmt.Sprintf(currentItem.URLFormat, c.PlaybackCounter)
	if c.mediaPlaylist.Count() > 10 {
		err = c.mediaPlaylist.Append(videoFile, 5.0, "")
	} else {
		err = c.mediaPlaylist.Slide(videoFile, 5.0, "")
	}
	if err != nil {
		log.Println("Error appending item to playlist:", err)
	}
	c.PlaybackCounter = c.PlaybackCounter + 1

	if c.PlaybackCounter > currentItem.EndAt {
		c.ResetCounter()
		if !currentItem.Loop {
			client.LPop(c.PlaybackQueueKey())
		}
	}

	return c.SaveCounter()
}

// function ResetCounter resets the playback counter to 0, and updates the datastore
func (c *Channel) ResetCounter() error {
	c.PlaybackCounter = 0
	return c.SaveCounter()
}

// function AdvanceEvery increments the counter every duration d
// the second argument should be a channel that will cancel the operation on receive
func (c *Channel) AdvanceEvery(d time.Duration, cancel chan int) {
	ticker := time.Tick(d)
	c.AdvanceCounter()

	for {
		select {
		case <-cancel:
			return
		case <-ticker:
			c.AdvanceCounter()
		}
	}

	return
}

// function SaveCounter saves the current PlaybackCounter in the data store
func (c Channel) SaveCounter() error {
	return client.Set(c.PlaybackCounterKey(), strconv.FormatInt(c.PlaybackCounter, 10), 0).Err()
}

// function PlaybackQueueKey returns the data store key for the playback queue
func (c Channel) PlaybackQueueKey() string {
	return "broadcast-channel-" + c.Identifier + "-queue"
}

// function PlaybackCounterKey returns the data store key for the playback counter
func (c Channel) PlaybackCounterKey() string {
	return "broadcast-channel-" + c.Identifier + "-counter"
}
<commit_msg>use Slide instead of Append<commit_after>package datastore

import (
	"encoding/json"
	"fmt"
	"github.com/grafov/m3u8"
	"strconv"
	"time"
)

// struct Channel describes a broadcast channel
type Channel struct {
	Identifier      string // Identifier is the unique part of the redis key
	PlaybackCounter int64
	mediaPlaylist   *m3u8.MediaPlaylist
}

// function GetChannel retrieves Channel information from the datastore
// given a channel identifier. It returns the channel and any error
func GetChannel(channelId string) (rv *Channel, err error) {
	rv = &Channel{}
	rv.Identifier = channelId
	rv.PlaybackCounter, err = client.IncrBy(rv.PlaybackCounterKey(), 0).Result()
	rv.mediaPlaylist, _ = m3u8.NewMediaPlaylist(60, 60) // include five minutes of history
	return
}

func (c *Channel) PlaylistData() string {
	return c.mediaPlaylist.Encode().String()
}

func (c Channel) CurrentItem() (PlaylistItem, error) {
	jsonString, err := client.LIndex(c.PlaybackQueueKey(), 0).Result()
	if err != nil {
		return PlaylistItem{}, err
	}

	item := PlaylistItem{}
	if err := json.Unmarshal([]byte(jsonString), &item); err != nil {
		return PlaylistItem{}, err
	}

	return item, nil
}

func (c Channel) PushItem(i *PlaylistItem) error {
	jsonBytes, err := json.Marshal(i)
	if err != nil {
		return err
	}

	err = client.RPush(c.PlaybackQueueKey(), string(jsonBytes)).Err()
	return err
}

// function AdvanceCounter advances the playback counter, and updates the datastore
func (c *Channel) AdvanceCounter() error {
	currentItem, err := c.CurrentItem()
	if err != nil {
		return err
	}

	if currentItem.StartAt > c.PlaybackCounter {
		c.PlaybackCounter = currentItem.StartAt
	}
	c.mediaPlaylist.Slide(fmt.Sprintf(currentItem.URLFormat, c.PlaybackCounter), 5.0, "")
	c.PlaybackCounter = c.PlaybackCounter + 1

	if c.PlaybackCounter > currentItem.EndAt {
		c.ResetCounter()
		if !currentItem.Loop {
			client.LPop(c.PlaybackQueueKey())
		}
	}

	return c.SaveCounter()
}

// function ResetCounter resets the playback counter to 0, and updates the datastore
func (c *Channel) ResetCounter() error {
	c.PlaybackCounter = 0
	return c.SaveCounter()
}

// function AdvanceEvery increments the counter every duration d
// the second argument should be a channel that will cancel the operation on receive
func (c *Channel) AdvanceEvery(d time.Duration, cancel chan int) {
	ticker := time.Tick(d)
	c.AdvanceCounter()

	for {
		select {
		case <-cancel:
			return
		case <-ticker:
			c.AdvanceCounter()
		}
	}

	return
}

// function SaveCounter saves the current PlaybackCounter in the data store
func (c Channel) SaveCounter() error {
	return client.Set(c.PlaybackCounterKey(), strconv.FormatInt(c.PlaybackCounter, 10), 0).Err()
}

// function PlaybackQueueKey returns the data store key for the playback queue
func (c Channel) PlaybackQueueKey() string {
	return "broadcast-channel-" + c.Identifier + "-queue"
}

// function PlaybackCounterKey returns the data store key for the playback counter
func (c Channel) PlaybackCounterKey() string {
	return "broadcast-channel-" + c.Identifier + "-counter"
}
<|endoftext|>


<commit_before>package datastore

import (
	"context"
	"database/sql"
	"fmt"
	"time"

	"github.com/hellodudu/Ultimate/global"
	"github.com/hellodudu/Ultimate/iface"
	"github.com/hellodudu/Ultimate/logger"
	"github.com/jinzhu/gorm"
)

// Global mysql table global
type Global struct {
	gorm.Model
	Id                 int `gorm:"type:int(10);primary_key;column:id"`
	TimeStamp          int `gorm:"type:int(10);column:time_stamp"`
	ArenaSeason        int `gorm:"type:int(10);column:arena_season"`
	ArenaWeekEndTime   int `gorm:"type:int(10);column:arena_week_end_time"`
	ArenaSeasonEndTime int `gorm:"type:int(10);column:arena_season_end_time"`
}

type Datastore struct {
	db     *gorm.DB
	ctx    context.Context
	cancel context.CancelFunc
	chStop chan struct{}
	chExec chan string

	// table
	global *Global
}

func NewDatastore() (iface.IDatastore, error) {
	datastore := &Datastore{
		chStop: make(chan struct{}, 1),
		chExec: make(chan string, 1000),
	}

	datastore.ctx, datastore.cancel = context.WithCancel(context.Background())

	mysqlDSN := fmt.Sprintf("%s:%s@(%s:%s)/%s", global.MysqlUser, global.MysqlPwd, global.MysqlAddr, global.MysqlPort, global.MysqlDB)
	var err error
	datastore.db, err = gorm.Open("mysql", mysqlSDN)
	if err != nil {
		logger.Fatal(err)
		return nil, err
	}

	datastore.initDatastore()
	return datastore, nil
}

func (m *Datastore) Run() {
	for {
		select {
		case <-m.ctx.Done():
			logger.Print("db mgr context done!")
			m.chStop <- struct{}{}
			return
		case query := <-m.chExec:
			if _, err := m.db.ExecContext(m.ctx, query); err != nil {
				logger.Error(fmt.Sprintf("db exec<%s> failed:", query), err)
			}
		}
	}

}

func (m *Datastore) Stop() chan struct{} {
	m.db.Close()
	m.cancel()
	return m.chStop
}

func (m *Datastore) initDatastore() {
	m.loadGlobal()
}

func (m *Datastore) loadGlobal() {

	m.db.AutoMigrate(&m.global)

	query := "select * from global"
	stmt, err := m.db.PrepareContext(m.ctx, query)
	if err != nil {
		logger.Warning("api initdb failed:", err)
		return
	}

	rows, err := stmt.QueryContext(m.ctx)
	if err != nil {
		logger.Warning("api initdb failed:", err)
		return
	}

	if !rows.Next() {
		query = fmt.Sprintf("replace into global set id=%d, time_stamp=%d, arena_season=%d, arena_week_end_time=%d, arena_season_end_time=%d", global.UltimateID, int32(time.Now().Unix()), 0, 0, 0)
		if stmp, err := m.db.PrepareContext(m.ctx, query); err == nil {
			if _, err := stmp.ExecContext(m.ctx); err == nil {
				logger.Info("sql global init query exec success:", query)
			}
		}
	}
}

func (m *Datastore) Exec(q string) {
	m.chExec <- q
}

func (m *Datastore) Query(q string) (*sql.Rows, error) {
	return func() (*sql.Rows, error) {
		stmt, err := m.db.PrepareContext(m.ctx, q)
		if err != nil {
			logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
			return nil, err
		}

		rows, err := stmt.QueryContext(m.ctx)
		if err != nil {
			logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
			return nil, err
		}

		return rows, nil
	}()
}
<commit_msg>table global firstorcreate<commit_after>package datastore

import (
	"context"
	"database/sql"
	"fmt"
	"time"

	"github.com/hellodudu/Ultimate/global"
	"github.com/hellodudu/Ultimate/iface"
	"github.com/hellodudu/Ultimate/logger"
	"github.com/jinzhu/gorm"
)

// Global mysql table global
type Global struct {
	Id                 int `gorm:"type:int(10);primary_key;column:id;default:0;not null"`
	TimeStamp          int `gorm:"type:int(10);column:time_stamp;default:0;not null"`
	ArenaSeason        int `gorm:"type:int(10);column:arena_season;default:0;not null"`
	ArenaWeekEndTime   int `gorm:"type:int(10);column:arena_week_end_timede;default:0;not null"`
	ArenaSeasonEndTime int `gorm:"type:int(10);column:arena_season_end_time;default:0;not null"`
}

// TableName set global table name to be `global`
func (Global) TableName() string {
	return "global"
}

type Datastore struct {
	db     *gorm.DB
	ctx    context.Context
	cancel context.CancelFunc
	chStop chan struct{}
	chExec chan string

	// table
	global *Global
}

func NewDatastore() (iface.IDatastore, error) {
	datastore := &Datastore{
		chStop: make(chan struct{}, 1),
		chExec: make(chan string, 1000),
	}

	datastore.ctx, datastore.cancel = context.WithCancel(context.Background())

	mysqlDSN := fmt.Sprintf("%s:%s@(%s:%s)/%s", global.MysqlUser, global.MysqlPwd, global.MysqlAddr, global.MysqlPort, global.MysqlDB)
	var err error
	datastore.db, err = gorm.Open("mysql", mysqlDSN)
	if err != nil {
		logger.Fatal(err)
		return nil, err
	}

	datastore.initDatastore()
	return datastore, nil
}

func (m *Datastore) Run() {
	for {
		select {
		case <-m.ctx.Done():
			logger.Print("db mgr context done!")
			m.chStop <- struct{}{}
			return
		case _ = <-m.chExec:
			// if _, err := m.db.ExecContext(m.ctx, query); err != nil {
			// 	logger.Error(fmt.Sprintf("db exec<%s> failed:", query), err)
			// }
		}
	}

}

func (m *Datastore) Stop() chan struct{} {
	m.db.Close()
	m.cancel()
	return m.chStop
}

func (m *Datastore) initDatastore() {
	m.loadGlobal()
}

func (m *Datastore) loadGlobal() {
	m.global = &Global{
		Id:                 global.UltimateID,
		TimeStamp:          int(int32(time.Now().Unix())),
		ArenaSeason:        0,
		ArenaWeekEndTime:   0,
		ArenaSeasonEndTime: 0,
	}

	m.db.AutoMigrate(m.global)
	if m.db.FirstOrCreate(m.global, global.UltimateID).RecordNotFound() {
		m.db.Create(m.global)
	}

	logger.Info("datastore loadGlobal success:", m.global)
}

func (m *Datastore) Exec(q string) {
	m.chExec <- q
}

func (m *Datastore) Query(q string) (*sql.Rows, error) {
	return nil, fmt.Errorf("test error")
	// return func() (*sql.Rows, error) {
	// 	stmt, err := m.db.PrepareContext(m.ctx, q)
	// 	if err != nil {
	// 		logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
	// 		return nil, err
	// 	}

	// 	rows, err := stmt.QueryContext(m.ctx)
	// 	if err != nil {
	// 		logger.Warning(fmt.Sprintf("db query<%s> failed:", q), err)
	// 		return nil, err
	// 	}

	// 	return rows, nil
	// }()
}
<|endoftext|>


<commit_before>// Copyright 2014 The mqrouter Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dbrouter

import (
	"context"
	"fmt"
	"github.com/opentracing/opentracing-go"
	"github.com/shawnfeng/sutil/slog"
	stat "github.com/shawnfeng/sutil/stat"
	"github.com/shawnfeng/sutil/stime"
	"gopkg.in/mgo.v2"
)

type Router struct {
	configer  Configer
	instances *InstanceManager
	report    *stat.StatReport
}

func NewRouter(data []byte) (*Router, error) {
	configer := NewSimpleConfiger(data)
	factory := func(configer Configer) func(ctx context.Context, key string) (in Instancer, err error) {
		return func(ctx context.Context, key string) (in Instancer, err error) {
			return Factory(ctx, key, configer)
		}
	}(configer)
	return &Router{
		configer:  configer,
		instances: NewInstanceManager(factory),
		report:    stat.NewStat(),
	}, nil
}

func (m *Router) StatInfo() []*stat.QueryStat {
	return m.report.StatInfo()
}

func (m *Router) SqlExec(ctx context.Context, cluster string, query func(*DB, []interface{}) error, tables ...string) error {
	fun := "Router.SqlExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.SqlExec")
	if span != nil {
		defer span.Finish()
	}

	st := stime.NewTimeStat()

	if len(tables) <= 0 {
		return fmt.Errorf("tables is empty")
	}

	table := tables[0]
	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		return fmt.Errorf("db instance not find: instance:%s", instance)
	}

	dbsql, ok := in.(*Sql)
	if !ok {
		return fmt.Errorf("db instance type error: instance:%s, dbtype:%s", instance, in.GetType())
	}

	db := dbsql.getDB()

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Infof("%s type:%s instance:%s query:%d", fun, in.GetType(), instance, dur)
	}()

	var tmptables []interface{}
	for _, item := range tables {
		tmptables = append(tmptables, item)
	}

	return query(db, tmptables)
}

func (m *Router) MongoExecEventual(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, eventual, cluster, table, query)
}

func (m *Router) MongoExecMonotonic(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, monotonic, cluster, table, query)
}

func (m *Router) MongoExecStrong(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, strong, cluster, table, query)
}

func (m *Router) mongoExec(ctx context.Context, consistency mode, cluster, table string, query func(*mgo.Collection) error) error {
	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.mongoExec")
	if span != nil {
		defer span.Finish()
	}

	st := stime.NewTimeStat()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		return fmt.Errorf("db instance not find: cluster:%s table:%s", cluster, table)
	}

	db, ok := in.(*dbMongo)
	if !ok {
		return fmt.Errorf("db instance type error: cluster:%s table:%s type:%s", cluster, table, in.GetType())
	}

	sess, err := db.getSession(consistency)
	if err != nil {
		return err
	}

	if sess == nil {
		return fmt.Errorf("db instance session empty: cluster:%s table:%s type:%s", cluster, table, in.GetType())
	}

	sessionCopy := sess.Copy()
	defer sessionCopy.Close()
	c := sessionCopy.DB("").C(table)

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef("[MONGO] const:%d cls:%s table:%s dur:%d", consistency, cluster, table, dur)
	}()

	return query(c)
}
<commit_msg>MOD:log<commit_after>// Copyright 2014 The mqrouter Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dbrouter

import (
	"context"
	"fmt"
	"github.com/opentracing/opentracing-go"
	"github.com/shawnfeng/sutil/slog"
	stat "github.com/shawnfeng/sutil/stat"
	"github.com/shawnfeng/sutil/stime"
	"gopkg.in/mgo.v2"
)

type Router struct {
	configer  Configer
	instances *InstanceManager
	report    *stat.StatReport
}

func NewRouter(data []byte) (*Router, error) {
	configer := NewSimpleConfiger(data)
	factory := func(configer Configer) func(ctx context.Context, key string) (in Instancer, err error) {
		return func(ctx context.Context, key string) (in Instancer, err error) {
			return Factory(ctx, key, configer)
		}
	}(configer)
	return &Router{
		configer:  configer,
		instances: NewInstanceManager(factory),
		report:    stat.NewStat(),
	}, nil
}

func (m *Router) StatInfo() []*stat.QueryStat {
	return m.report.StatInfo()
}

func (m *Router) SqlExec(ctx context.Context, cluster string, query func(*DB, []interface{}) error, tables ...string) error {
	fun := "Router.SqlExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.SqlExec")
	if span != nil {
		defer span.Finish()
	}

	st := stime.NewTimeStat()

	if len(tables) <= 0 {
		return fmt.Errorf("tables is empty")
	}

	table := tables[0]
	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		return fmt.Errorf("db instance not find: instance:%s", instance)
	}

	dbsql, ok := in.(*Sql)
	if !ok {
		return fmt.Errorf("db instance type error: instance:%s, dbtype:%s", instance, in.GetType())
	}

	db := dbsql.getDB()

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef("%s cls:%s table:%s dur:%d", fun, cluster, table, dur)
	}()

	var tmptables []interface{}
	for _, item := range tables {
		tmptables = append(tmptables, item)
	}

	return query(db, tmptables)
}

func (m *Router) MongoExecEventual(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, eventual, cluster, table, query)
}

func (m *Router) MongoExecMonotonic(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, monotonic, cluster, table, query)
}

func (m *Router) MongoExecStrong(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, strong, cluster, table, query)
}

func (m *Router) mongoExec(ctx context.Context, consistency mode, cluster, table string, query func(*mgo.Collection) error) error {
	fun := "Router.mongoExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.mongoExec")
	if span != nil {
		defer span.Finish()
	}

	st := stime.NewTimeStat()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		return fmt.Errorf("db instance not find: cluster:%s table:%s", cluster, table)
	}

	db, ok := in.(*dbMongo)
	if !ok {
		return fmt.Errorf("db instance type error: cluster:%s table:%s type:%s", cluster, table, in.GetType())
	}

	sess, err := db.getSession(consistency)
	if err != nil {
		return err
	}

	if sess == nil {
		return fmt.Errorf("db instance session empty: cluster:%s table:%s type:%s", cluster, table, in.GetType())
	}

	sessionCopy := sess.Copy()
	defer sessionCopy.Close()
	c := sessionCopy.DB("").C(table)

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef("%s const:%d cls:%s table:%s dur:%d", fun, consistency, cluster, table, dur)
	}()

	return query(c)
}
<|endoftext|>