<commit_before>/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	jwtgo "github.com/dgrijalva/jwt-go"
	jwtreq "github.com/dgrijalva/jwt-go/request"
	"github.com/minio/minio/pkg/auth"
)

const (
	jwtAlgorithm = "Bearer"

	// Default JWT token for web handlers is one day.
	defaultJWTExpiry = 24 * time.Hour

	// Inter-node JWT token expiry is 100 years approx.
	defaultInterNodeJWTExpiry = 100 * 365 * 24 * time.Hour

	// URL JWT token expiry is one minute (might be exposed).
	defaultURLJWTExpiry = time.Minute
)

var (
	errInvalidAccessKeyID   = errors.New("The access key ID you provided does not exist in our records")
	errChangeCredNotAllowed = errors.New("Changing access key and secret key not allowed")
	errAuthentication       = errors.New("Authentication failed, check your access credentials")
	errNoAuthToken          = errors.New("JWT token missing")
)

func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string, error) {
	passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
	if err != nil {
		return "", err
	}

	serverCred := globalServerConfig.GetCredential()

	if serverCred.AccessKey != passedCredential.AccessKey {
		return "", errInvalidAccessKeyID
	}

	if !serverCred.Equal(passedCredential) {
		return "", errAuthentication
	}

	utcNow := UTCNow()
	token := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
		ExpiresAt: utcNow.Add(expiry).Unix(),
		IssuedAt:  utcNow.Unix(),
		Subject:   accessKey,
	})

	return token.SignedString([]byte(serverCred.SecretKey))
}

func authenticateNode(accessKey, secretKey string) (string, error) {
	return authenticateJWT(accessKey, secretKey, defaultInterNodeJWTExpiry)
}

func authenticateWeb(accessKey, secretKey string) (string, error) {
	return authenticateJWT(accessKey, secretKey, defaultJWTExpiry)
}

func authenticateURL(accessKey, secretKey string) (string, error) {
	return authenticateJWT(accessKey, secretKey, defaultURLJWTExpiry)
}

func keyFuncCallback(jwtToken *jwtgo.Token) (interface{}, error) {
	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
	}

	return []byte(globalServerConfig.GetCredential().SecretKey), nil
}

func isAuthTokenValid(tokenString string) bool {
	if tokenString == "" {
		return false
	}
	var claims jwtgo.StandardClaims
	jwtToken, err := jwtgo.ParseWithClaims(tokenString, &claims, keyFuncCallback)
	if err != nil {
		errorIf(err, "Unable to parse JWT token string")
		return false
	}
	if err = claims.Valid(); err != nil {
		errorIf(err, "Invalid claims in JWT token string")
		return false
	}
	return jwtToken.Valid && claims.Subject == globalServerConfig.GetCredential().AccessKey
}

func isHTTPRequestValid(req *http.Request) bool {
	return webRequestAuthenticate(req) == nil
}

// Check if the request is authenticated.
// Returns nil if the request is authenticated. errNoAuthToken if token missing.
// Returns errAuthentication for all other errors.
func webRequestAuthenticate(req *http.Request) error {
	var claims jwtgo.StandardClaims
	jwtToken, err := jwtreq.ParseFromRequestWithClaims(req, jwtreq.AuthorizationHeaderExtractor, &claims, keyFuncCallback)
	if err != nil {
		if err == jwtreq.ErrNoTokenInRequest {
			return errNoAuthToken
		}
		return errAuthentication
	}
	if err = claims.Valid(); err != nil {
		return err
	}
	if claims.Subject != globalServerConfig.GetCredential().AccessKey {
		return errInvalidAccessKeyID
	}
	if !jwtToken.Valid {
		return errAuthentication
	}
	return nil
}
<commit_msg>Remove requirement for issued at JWT claims (#5364)<commit_after>/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	jwtgo "github.com/dgrijalva/jwt-go"
	jwtreq "github.com/dgrijalva/jwt-go/request"
	"github.com/minio/minio/pkg/auth"
)

const (
	jwtAlgorithm = "Bearer"

	// Default JWT token for web handlers is one day.
	defaultJWTExpiry = 24 * time.Hour

	// Inter-node JWT token expiry is 100 years approx.
	defaultInterNodeJWTExpiry = 100 * 365 * 24 * time.Hour

	// URL JWT token expiry is one minute (might be exposed).
	defaultURLJWTExpiry = time.Minute
)

var (
	errInvalidAccessKeyID   = errors.New("The access key ID you provided does not exist in our records")
	errChangeCredNotAllowed = errors.New("Changing access key and secret key not allowed")
	errAuthentication       = errors.New("Authentication failed, check your access credentials")
	errNoAuthToken          = errors.New("JWT token missing")
)

func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string, error) {
	passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
	if err != nil {
		return "", err
	}

	serverCred := globalServerConfig.GetCredential()

	if serverCred.AccessKey != passedCredential.AccessKey {
		return "", errInvalidAccessKeyID
	}

	if !serverCred.Equal(passedCredential) {
		return "", errAuthentication
	}

	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
		ExpiresAt: UTCNow().Add(expiry).Unix(),
		Subject:   accessKey,
	})
	return jwt.SignedString([]byte(serverCred.SecretKey))
}

func authenticateNode(accessKey, secretKey string) (string, error) {
	return authenticateJWT(accessKey, secretKey, defaultInterNodeJWTExpiry)
}

func authenticateWeb(accessKey, secretKey string) (string, error) {
	return authenticateJWT(accessKey, secretKey, defaultJWTExpiry)
}

func authenticateURL(accessKey, secretKey string) (string, error) {
	return authenticateJWT(accessKey, secretKey, defaultURLJWTExpiry)
}

func keyFuncCallback(jwtToken *jwtgo.Token) (interface{}, error) {
	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
	}

	return []byte(globalServerConfig.GetCredential().SecretKey), nil
}

func isAuthTokenValid(tokenString string) bool {
	if tokenString == "" {
		return false
	}
	var claims jwtgo.StandardClaims
	jwtToken, err := jwtgo.ParseWithClaims(tokenString, &claims, keyFuncCallback)
	if err != nil {
		errorIf(err, "Unable to parse JWT token string")
		return false
	}
	if err = claims.Valid(); err != nil {
		errorIf(err, "Invalid claims in JWT token string")
		return false
	}
	return jwtToken.Valid && claims.Subject == globalServerConfig.GetCredential().AccessKey
}

func isHTTPRequestValid(req *http.Request) bool {
	return webRequestAuthenticate(req) == nil
}

// Check if the request is authenticated.
// Returns nil if the request is authenticated. errNoAuthToken if token missing.
// Returns errAuthentication for all other errors.
func webRequestAuthenticate(req *http.Request) error {
	var claims jwtgo.StandardClaims
	jwtToken, err := jwtreq.ParseFromRequestWithClaims(req, jwtreq.AuthorizationHeaderExtractor, &claims, keyFuncCallback)
	if err != nil {
		if err == jwtreq.ErrNoTokenInRequest {
			return errNoAuthToken
		}
		return errAuthentication
	}
	if err = claims.Valid(); err != nil {
		return errAuthentication
	}
	if claims.Subject != globalServerConfig.GetCredential().AccessKey {
		return errInvalidAccessKeyID
	}
	if !jwtToken.Valid {
		return errAuthentication
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/motemen/ghq/cmdutil"
	"github.com/motemen/ghq/logger"
	"github.com/urfave/cli/v2"
	"golang.org/x/sync/errgroup"
)

func doGet(c *cli.Context) error {
	var (
		args     = c.Args().Slice()
		andLook  = c.Bool("look")
		parallel = c.Bool("parallel")
	)
	g := &getter{
		update:    c.Bool("update"),
		shallow:   c.Bool("shallow"),
		ssh:       c.Bool("p"),
		vcs:       c.String("vcs"),
		silent:    c.Bool("silent"),
		branch:    c.String("branch"),
		recursive: !c.Bool("no-recursive"),
	}
	if parallel {
		// force silent in parallel import
		g.silent = true
	}

	var (
		firstArg string
		scr      scanner
	)
	if len(args) > 0 {
		scr = &sliceScanner{slice: args}
	} else {
		fi, err := os.Stdin.Stat()
		if err != nil {
			return err
		}
		// check if from pipe or not
		if fi.Mode()&os.ModeCharDevice != 0 || fi.Size() <= 0 {
			return fmt.Errorf("no project args specified. see `ghq get -h` for more details")
		}
		scr = bufio.NewScanner(os.Stdin)
	}
	eg := &errgroup.Group{}
	sem := make(chan struct{}, 6)
	for scr.Scan() {
		target := scr.Text()
		if firstArg != "" {
			firstArg = target
		}
		if parallel {
			eg.Go(func() error {
				sem <- struct{}{}
				defer func() { <-sem }()
				if err := g.get(target); err != nil {
					logger.Log("error", err.Error())
				}
				return nil
			})
		} else {
			if err := g.get(target); err != nil {
				return err
			}
		}
	}
	if err := scr.Err(); err != nil {
		return fmt.Errorf("While reading input: %s", err)
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	if andLook && firstArg != "" {
		return look(firstArg)
	}
	return nil
}

type sliceScanner struct {
	slice []string
	index int
}

func (s *sliceScanner) Scan() bool {
	s.index++
	return s.index <= len(s.slice)
}

func (s *sliceScanner) Text() string {
	return s.slice[s.index-1]
}

func (s *sliceScanner) Err() error {
	return nil
}

type scanner interface {
	Scan() bool
	Text() string
	Err() error
}

func detectShell() string {
	shell := os.Getenv("SHELL")
	if shell != "" {
		return shell
	}
	if runtime.GOOS == "windows" {
		return os.Getenv("COMSPEC")
	}
	return "/bin/sh"
}

func look(name string) error {
	var (
		reposFound []*LocalRepository
		mu         sync.Mutex
	)
	if err := walkAllLocalRepositories(func(repo *LocalRepository) {
		if repo.Matches(name) {
			mu.Lock()
			reposFound = append(reposFound, repo)
			mu.Unlock()
		}
	}); err != nil {
		return err
	}

	if len(reposFound) == 0 {
		if url, err := newURL(name, false, false); err == nil {
			repo, err := LocalRepositoryFromURL(url)
			if err != nil {
				return err
			}
			_, err = os.Stat(repo.FullPath)

			// if the directory exists
			if err == nil {
				reposFound = append(reposFound, repo)
			}
		}
	}

	switch len(reposFound) {
	case 0:
		return fmt.Errorf("No repository found")
	case 1:
		repo := reposFound[0]
		cmd := exec.Command(detectShell())
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = repo.FullPath
		cmd.Env = append(os.Environ(), "GHQ_LOOK="+filepath.ToSlash(repo.RelPath))
		return cmdutil.RunCommand(cmd, true)
	default:
		b := &strings.Builder{}
		b.WriteString("More than one repositories are found; Try more precise name\n")
		for _, repo := range reposFound {
			b.WriteString(fmt.Sprintf("       - %s\n", strings.Join(repo.PathParts, "/")))
		}
		return errors.New(b.String())
	}
}
<commit_msg>checks only stdin size<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"github.com/motemen/ghq/cmdutil"
	"github.com/motemen/ghq/logger"
	"github.com/urfave/cli/v2"
	"golang.org/x/sync/errgroup"
)

func doGet(c *cli.Context) error {
	var (
		args     = c.Args().Slice()
		andLook  = c.Bool("look")
		parallel = c.Bool("parallel")
	)
	g := &getter{
		update:    c.Bool("update"),
		shallow:   c.Bool("shallow"),
		ssh:       c.Bool("p"),
		vcs:       c.String("vcs"),
		silent:    c.Bool("silent"),
		branch:    c.String("branch"),
		recursive: !c.Bool("no-recursive"),
	}
	if parallel {
		// force silent in parallel import
		g.silent = true
	}

	var (
		firstArg string
		scr      scanner
	)
	if len(args) > 0 {
		scr = &sliceScanner{slice: args}
	} else {
		fi, err := os.Stdin.Stat()
		if err != nil {
			return err
		}
		if fi.Size() <= 0 {
			return fmt.Errorf("no target args specified. see `ghq get -h` for more details")
		}
		scr = bufio.NewScanner(os.Stdin)
	}
	eg := &errgroup.Group{}
	sem := make(chan struct{}, 6)
	for scr.Scan() {
		target := scr.Text()
		if firstArg != "" {
			firstArg = target
		}
		if parallel {
			eg.Go(func() error {
				sem <- struct{}{}
				defer func() { <-sem }()
				if err := g.get(target); err != nil {
					logger.Log("error", err.Error())
				}
				return nil
			})
		} else {
			if err := g.get(target); err != nil {
				return err
			}
		}
	}
	if err := scr.Err(); err != nil {
		return fmt.Errorf("While reading input: %s", err)
	}
	if err := eg.Wait(); err != nil {
		return err
	}
	if andLook && firstArg != "" {
		return look(firstArg)
	}
	return nil
}

type sliceScanner struct {
	slice []string
	index int
}

func (s *sliceScanner) Scan() bool {
	s.index++
	return s.index <= len(s.slice)
}

func (s *sliceScanner) Text() string {
	return s.slice[s.index-1]
}

func (s *sliceScanner) Err() error {
	return nil
}

type scanner interface {
	Scan() bool
	Text() string
	Err() error
}

func detectShell() string {
	shell := os.Getenv("SHELL")
	if shell != "" {
		return shell
	}
	if runtime.GOOS == "windows" {
		return os.Getenv("COMSPEC")
	}
	return "/bin/sh"
}

func look(name string) error {
	var (
		reposFound []*LocalRepository
		mu         sync.Mutex
	)
	if err := walkAllLocalRepositories(func(repo *LocalRepository) {
		if repo.Matches(name) {
			mu.Lock()
			reposFound = append(reposFound, repo)
			mu.Unlock()
		}
	}); err != nil {
		return err
	}

	if len(reposFound) == 0 {
		if url, err := newURL(name, false, false); err == nil {
			repo, err := LocalRepositoryFromURL(url)
			if err != nil {
				return err
			}
			_, err = os.Stat(repo.FullPath)

			// if the directory exists
			if err == nil {
				reposFound = append(reposFound, repo)
			}
		}
	}

	switch len(reposFound) {
	case 0:
		return fmt.Errorf("No repository found")
	case 1:
		repo := reposFound[0]
		cmd := exec.Command(detectShell())
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Dir = repo.FullPath
		cmd.Env = append(os.Environ(), "GHQ_LOOK="+filepath.ToSlash(repo.RelPath))
		return cmdutil.RunCommand(cmd, true)
	default:
		b := &strings.Builder{}
		b.WriteString("More than one repositories are found; Try more precise name\n")
		for _, repo := range reposFound {
			b.WriteString(fmt.Sprintf("       - %s\n", strings.Join(repo.PathParts, "/")))
		}
		return errors.New(b.String())
	}
}
<|endoftext|>


<commit_before>package sia

import (
	"errors"
	"fmt"
	"testing"
)

type testingEnvironment struct {
	wallets []*Wallet
	state   *State
}

// Creates a wallet and a state to use for testing.
func createEnvironment() (testEnv *testingEnvironment, err error) {
	testEnv = new(testingEnvironment)

	firstWallet, err := CreateWallet()
	if err != nil {
		return
	}
	testEnv.wallets = append(testEnv.wallets, firstWallet)

	testEnv.state = CreateGenesisState(testEnv.wallets[0].CoinAddress)

	if len(testEnv.state.ConsensusState.UnspentOutputs) != 1 {
		err = fmt.Errorf("Genesis state should have a single upspent output, has", len(testEnv.state.ConsensusState.UnspentOutputs))
		return
	}

	return
}

// Creates an empty block and applies it to the state.
func addEmptyBlock(testEnv *testingEnvironment) (err error) {
	// Make sure that the block will actually be empty.
	if len(testEnv.state.ConsensusState.TransactionList) != 0 {
		err = errors.New("cannot add an empty block without an empty transaction pool.")
		return
	}

	// Generate a valid empty block using GenerateBlock.
	emptyBlock := testEnv.state.GenerateBlock(testEnv.wallets[0].CoinAddress)
	if len(emptyBlock.Transactions) != 0 {
		err = errors.New("failed to make an empty block...")
		return
	}

	expectedOutputs := len(testEnv.state.ConsensusState.UnspentOutputs) + 1
	err = testEnv.state.AcceptBlock(*emptyBlock)
	if err != nil {
		return
	}
	if len(testEnv.state.ConsensusState.UnspentOutputs) != expectedOutputs {
		err = fmt.Errorf("Expecting", expectedOutputs, "outputs, got", len(testEnv.state.ConsensusState.UnspentOutputs), "outputs")
		return
	}

	return
}

// transactionPoolTests adds a few wallets to the test environment, creating
// transactions that fund each and probes the overall efficiency of the
// transaction pool structures.
func transactionPoolTests(testEnv *testingEnvironment) (err error) {
	// The current wallet design means that it will double spend on
	// sequential transactions - meaning that if you make two transactions
	// in the same block, the wallet will use the same input for each.
	// We'll fix this sooner rather than later, but for now the problem has
	// been left so we can focus on other things.

	// Create a new wallet for the test environment.
	wallet, err := CreateWallet()
	if err != nil {
		return
	}
	testEnv.wallets = append(testEnv.wallets, wallet)

	// Create a transaction to send to that wallet.
	transaction, err := testEnv.wallets[0].SpendCoins(Currency(3), testEnv.wallets[len(testEnv.wallets)-1].CoinAddress, testEnv.state)
	if err != nil {
		return
	}
	err = testEnv.state.AcceptTransaction(transaction)
	if err != nil {
		return
	}

	// Attempt to create a conflicting transaction and see if it is rejected from the pool.
	transaction.Outputs[0].SpendHash[0] = ^transaction.Outputs[0].SpendHash[0] // Change the output address
	transactionSigHash := transaction.SigHash(0)
	transaction.Signatures[0].Signature, err = SignBytes(transactionSigHash[:], testEnv.wallets[0].SecretKey)
	if err != nil {
		return
	}
	err = testEnv.state.AcceptTransaction(transaction)
	if err == nil {
		err = errors.New("Added a conflicting transaction to the transaction pool without error.")
		return
	}
	err = nil

	// Put a block through, which should clear the transaction pool
	// completely.

	return
}

// For now, this is really just a catch-all test. I'm not really sure how to
// modularize the various components =/
func TestBlockBuilding(t *testing.T) {
	// Initialize the testing evironment.
	testEnv, err := createEnvironment()
	if err != nil {
		t.Fatal(err)
	}

	// Add an empty block to the testing environment.
	err = addEmptyBlock(testEnv)
	if err != nil {
		t.Fatal(err)
	}

	// Create a few new wallets and send coins to each in a block.
	err = transactionPoolTests(testEnv)
	if err != nil {
		t.Fatal(err)
	}

	// Create a third block with transactions.

	// Create a thrid block containing the transaction, add it.

	// Create a block with multiple transactions, but one isn't valid.
	// This will see if the reverse code works correctly.
}
<commit_msg>add tests for generating blocks with a non-empty transaction pool<commit_after>package sia

import (
	"errors"
	"fmt"
	"testing"
)

type testingEnvironment struct {
	wallets []*Wallet
	state   *State
}

// Creates a wallet and a state to use for testing.
func createEnvironment() (testEnv *testingEnvironment, err error) {
	testEnv = new(testingEnvironment)

	firstWallet, err := CreateWallet()
	if err != nil {
		return
	}
	testEnv.wallets = append(testEnv.wallets, firstWallet)

	testEnv.state = CreateGenesisState(testEnv.wallets[0].CoinAddress)

	if len(testEnv.state.ConsensusState.UnspentOutputs) != 1 {
		err = fmt.Errorf("Genesis state should have a single upspent output, has", len(testEnv.state.ConsensusState.UnspentOutputs))
		return
	}

	return
}

// Creates an empty block and applies it to the state.
func addEmptyBlock(testEnv *testingEnvironment) (err error) {
	// Make sure that the block will actually be empty.
	if len(testEnv.state.ConsensusState.TransactionList) != 0 {
		err = errors.New("cannot add an empty block without an empty transaction pool.")
		return
	}

	// Generate a valid empty block using GenerateBlock.
	emptyBlock := testEnv.state.GenerateBlock(testEnv.wallets[0].CoinAddress)
	if len(emptyBlock.Transactions) != 0 {
		err = errors.New("failed to make an empty block...")
		return
	}

	expectedOutputs := len(testEnv.state.ConsensusState.UnspentOutputs) + 1
	err = testEnv.state.AcceptBlock(*emptyBlock)
	if err != nil {
		return
	}
	if len(testEnv.state.ConsensusState.UnspentOutputs) != expectedOutputs {
		err = fmt.Errorf("Expecting", expectedOutputs, "outputs, got", len(testEnv.state.ConsensusState.UnspentOutputs), "outputs")
		return
	}

	return
}

// transactionPoolTests adds a few wallets to the test environment, creating
// transactions that fund each and probes the overall efficiency of the
// transaction pool structures.
func transactionPoolTests(testEnv *testingEnvironment) (err error) {
	// The current wallet design means that it will double spend on
	// sequential transactions - meaning that if you make two transactions
	// in the same block, the wallet will use the same input for each.
	// We'll fix this sooner rather than later, but for now the problem has
	// been left so we can focus on other things.
	// Record the size of the transaction pool and the transaction list.

	// One thing we can do to increase the modularity of this function is
	// create a block at the beginning, and use the coinbase to create a
	// bunch of new wallets. This would also clear out the transaction pool
	// right at the beginning of the function.

	txnPoolLen := len(testEnv.state.ConsensusState.TransactionPool)
	txnListLen := len(testEnv.state.ConsensusState.TransactionList)

	// Create a new wallet for the test environment.
	wallet, err := CreateWallet()
	if err != nil {
		return
	}
	testEnv.wallets = append(testEnv.wallets, wallet)

	// Create a transaction to send to that wallet.
	transaction, err := testEnv.wallets[0].SpendCoins(Currency(3), testEnv.wallets[len(testEnv.wallets)-1].CoinAddress, testEnv.state)
	if err != nil {
		return
	}
	err = testEnv.state.AcceptTransaction(transaction)
	if err != nil {
		return
	}

	// Attempt to create a conflicting transaction and see if it is rejected from the pool.
	transaction.Outputs[0].SpendHash[0] = ^transaction.Outputs[0].SpendHash[0] // Change the output address
	transactionSigHash := transaction.SigHash(0)
	transaction.Signatures[0].Signature, err = SignBytes(transactionSigHash[:], testEnv.wallets[0].SecretKey) // Re-sign
	if err != nil {
		return
	}
	err = testEnv.state.AcceptTransaction(transaction)
	if err == nil {
		err = errors.New("Added a conflicting transaction to the transaction pool without error.")
		return
	}
	err = nil

	// The length of the transaction list should have grown by 1, and the
	// transaction pool should have grown by the number of outputs.
	if len(testEnv.state.ConsensusState.TransactionPool) != txnPoolLen+len(transaction.Inputs) {
		err = fmt.Errorf(
			"transaction pool did not grow by expected length. Started at %v and ended at %v but should have grown by %v",
			txnPoolLen,
			len(testEnv.state.ConsensusState.TransactionPool),
			len(transaction.Inputs),
		)
		return
	}
	if len(testEnv.state.ConsensusState.TransactionList) != txnListLen+1 {
		err = errors.New("transaction list did not grow by the expected length.")
		return
	}

	// Put a block through, which should clear the transaction pool
	// completely. Give the subsidy to the old wallet to replenish for
	// funding new wallets.
	transactionBlock := testEnv.state.GenerateBlock(testEnv.wallets[0].CoinAddress)
	if len(transactionBlock.Transactions) == 0 {
		err = errors.New("block created without accepting the transactions in the pool.")
		return
	}
	err = testEnv.state.AcceptBlock(*transactionBlock)
	if err != nil {
		return
	}

	// Check that the transaction pool has been cleared out.
	if len(testEnv.state.ConsensusState.TransactionPool) != 0 {
		err = errors.New("transaction pool not cleared out after getting a block.")
		return
	}
	if len(testEnv.state.ConsensusState.TransactionList) != 0 {
		err = errors.New("transaction list not cleared out after getting a block.")
		return
	}

	return
}

// For now, this is really just a catch-all test. I'm not really sure how to
// modularize the various components =/
func TestBlockBuilding(t *testing.T) {
	// Initialize the testing evironment.
	testEnv, err := createEnvironment()
	if err != nil {
		t.Fatal(err)
	}

	// Add an empty block to the testing environment.
	err = addEmptyBlock(testEnv)
	if err != nil {
		t.Fatal(err)
	}

	// Create a few new wallets and send coins to each in a block.
	err = transactionPoolTests(testEnv)
	if err != nil {
		t.Fatal(err)
	}

	// Create a third block with transactions.

	// Create a thrid block containing the transaction, add it.

	// Create a block with multiple transactions, but one isn't valid.
	// This will see if the reverse code works correctly.
}
<|endoftext|>


<commit_before>package schemamagic

import (
	"errors"
	"fmt"
	"strings"
)

// Column stores all the parameters of each column inside a table
type Column struct {
	Name            string
	Datatype        string
	PseudoDatatype  string // This is the name of the datatype that is used by PostgreSQL to store the mentioned Datatype. Eg.: time --> time without/with time zone, timestamp --> timestamp with/without time zone, etc.
	Action          string
	DefaultExists   bool
	DefaultValue    string
	IsUnique        bool
	IsPrimary       bool
	IsNotNull       bool
	IndexRequired   bool
	Comment         string
	SequenceRestart int64
}

// NewColumn initializes the Column with the default parameters
func NewColumn(c Column) Column {
	// col := new(Column)
	var col Column
	col.Name = c.Name
	col.Datatype = c.Datatype
	col.PseudoDatatype = c.PseudoDatatype
	if c.Action == "" {
		col.Action = "Add"
	} else {
		col.Action = c.Action
	}
	if c.DefaultExists {
		col.DefaultExists = true
	} else {
		col.DefaultExists = false
	}
	if c.DefaultValue == "" {
		col.DefaultValue = "'NULL'"
	} else {
		col.DefaultValue = c.DefaultValue
	}
	if c.IsUnique {
		col.IsUnique = true
	} else {
		col.IsUnique = false
	}
	if c.IsPrimary {
		col.IsPrimary = true
	} else {
		col.IsPrimary = false
	}
	if c.IsNotNull {
		// Set default IsNotNull to true
		col.IsNotNull = true
	} else {
		col.IsNotNull = false
	}
	if c.IndexRequired {
		col.IndexRequired = true
	} else {
		col.IndexRequired = false
	}
	col.Comment = c.Comment
	if c.SequenceRestart == 0 {
		col.SequenceRestart = 1
	} else {
		col.SequenceRestart = c.SequenceRestart
	}
	return col
}

// prepareSQLStatement prepares and returns the statement that needs to be executed by the table
func (c *Column) prepareSQLStatement(step int, tableName string, columnPresent bool) (string, error) {
	log.Debugln("Executing ", c.Name, " with step --> ", step)
	var statement string
	if step == 1 {
		// This is the step where the column is added without a default value
		// statement = "ALTER TABLE %s ADD %s %s"%(table_name, self.column_name, self.datatype)
		statement = fmt.Sprintf("ALTER TABLE %s ADD %s %s", tableName, c.Name, c.Datatype)
	} else if step == 2 {
		// This is the step where a default value is set for the column
		// statement = cursor.mogrify("ALTER TABLE %(table)s ALTER COLUMN %(column)s SET DEFAULT %(value)s", {"table" : AsIs(table_name), "column" : AsIs(self.column_name), "value" : AsIs(self.default_value)})
		if c.DefaultExists {
			statement = fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT %s", tableName, c.Name, c.DefaultValue)
		}
	} else if step == 3 {
		// This is the step where the default value is updated for all the existing rows
		// statement = cursor.mogrify("UPDATE %(table)s SET %(column)s = %(value)s", {"table" : AsIs(table_name), "column" : AsIs(self.column_name), "value" : AsIs(self.default_value)})
		if c.DefaultExists && !columnPresent {
			statement = fmt.Sprintf("UPDATE %s SET %s = %s", tableName, c.Name, c.DefaultValue)
		}
	} else if step == 4 {
		// This is the step where the sequence is altered, in case sequence_restart is > 0 and datatype is either bigserial or serial
		// statement = cursor.mogrify("ALTER SEQUENCE %(sequence_name)s RESTART WITH %(value)s", {"sequence_name" : AsIs(sequence_name), "value" : self.sequence_restart})
		if strings.Contains(c.Datatype, "serial") {
			statement = fmt.Sprintf("ALTER SEQUENCE %s RESTART WITH %d", tableName+"_"+c.Name+"_seq", c.SequenceRestart)
		}
	} else if step == 5 {
		// This is the step where a unique constraint is added, in case the column in unique
		if c.IsUnique {
			// statement = "ALTER TABLE %s ADD UNIQUE (%s)"%(table_name, self.column_name)
			constraint := Constraint{Name: fmt.Sprintf("%s_%s_unique", tableName, c.Name), Value: fmt.Sprintf("UNIQUE (%s)", c.Name)}
			statement = fmt.Sprintf("%s; %s", constraint.createDropRule(tableName), constraint.createAddRule(tableName))
			// statement = fmt.Sprintf("ALTER TABLE %s ADD UNIQUE (%s)", tableName, c.Name)
		}
	} else if step == 6 {
		// This is the step where a primary key constraint is added, in case the column is a primary key
		if c.IsPrimary {
			// statement = "ALTER TABLE %s ADD CONSTRAINT %s PRIMARY KEY(%s)"%(table_name, table_name + "_" +self.column_name, self.column_name)
			statement = fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT %s PRIMARY KEY(%s)", tableName, tableName+"_"+c.Name, c.Name)
		}
	} else if step == 7 {
		// This is the step where NOT NULL is applied to a particular column
		if c.IsNotNull {
			// statement = "ALTER TABLE %s ALTER COLUMN %s SET NOT NULL"%(table_name, self.column_name)
			statement = fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET NOT NULL", tableName, c.Name)
		}
	} else if step == 8 {
		// This is the step where the index is created on this column
		if c.IndexRequired {
			statement = fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_%s_index ON %s (%s)", tableName, c.Name, tableName, c.Name)
		}
	} else if step == 101 {
		// This is the step where the column's datatype is altered
		if strings.Contains(c.Datatype, "serial") {
			errorStatement := fmt.Sprintf("Can't modify datatype to SERIAL versions, while modifying \nTable --> %s \nColumn --> %s \nDatatype --> %s", tableName, c.Name, c.Datatype)
			err := errors.New(errorStatement)
			return "", err
		}
		// statement = "ALTER TABLE %s ALTER COLUMN %s TYPE %s USING %s::%s"%(table_name, self.column_name, self.datatype, self.column_name, altered_datatype)
		statement = fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s TYPE %s USING %s::%s", tableName, c.Name, c.Datatype, c.Name, c.Datatype)
	}

	log.Debugln("In prepareSQLStatement, statement is \n", statement)
	return statement, nil
}
<commit_msg>Columns now have an index type.<commit_after>package schemamagic

import (
	"errors"
	"fmt"
	"strings"
)

// Column stores all the parameters of each column inside a table
type Column struct {
	Name           string
	Datatype       string
	PseudoDatatype string // This is the name of the datatype that is used by PostgreSQL to store the mentioned Datatype. Eg.: time --> time without/with time zone, timestamp --> timestamp with/without time zone, etc.
	Action         string
	DefaultExists  bool
	DefaultValue   string
	IsUnique       bool
	IsPrimary      bool
	IsNotNull      bool
	IndexRequired  bool
	// Stores the Index Type: GIN, etc. Default will be empty, which is B-Tree (default index type in postgres)
	IndexType       string
	Comment         string
	SequenceRestart int64
}

// NewColumn initializes the Column with the default parameters
func NewColumn(c Column) Column {
	// col := new(Column)
	var col Column
	col.Name = c.Name
	col.Datatype = c.Datatype
	col.PseudoDatatype = c.PseudoDatatype
	if c.Action == "" {
		col.Action = "Add"
	} else {
		col.Action = c.Action
	}
	if c.DefaultExists {
		col.DefaultExists = true
	} else {
		col.DefaultExists = false
	}
	if c.DefaultValue == "" {
		col.DefaultValue = "'NULL'"
	} else {
		col.DefaultValue = c.DefaultValue
	}
	if c.IsUnique {
		col.IsUnique = true
	} else {
		col.IsUnique = false
	}
	if c.IsPrimary {
		col.IsPrimary = true
	} else {
		col.IsPrimary = false
	}
	if c.IsNotNull {
		// Set default IsNotNull to true
		col.IsNotNull = true
	} else {
		col.IsNotNull = false
	}
	if c.IndexRequired {
		col.IndexRequired = true
	} else {
		col.IndexRequired = false
	}
	if len(c.IndexType) > 0 && c.IndexRequired {
		col.IndexType = c.IndexType
	}
	col.Comment = c.Comment
	if c.SequenceRestart == 0 {
		col.SequenceRestart = 1
	} else {
		col.SequenceRestart = c.SequenceRestart
	}
	return col
}

// prepareSQLStatement prepares and returns the statement that needs to be executed by the table
func (c *Column) prepareSQLStatement(step int, tableName string, columnPresent bool) (string, error) {
	log.Debugln("Executing ", c.Name, " with step --> ", step)
	var statement string
	if step == 1 {
		// This is the step where the column is added without a default value
		// statement = "ALTER TABLE %s ADD %s %s"%(table_name, self.column_name, self.datatype)
		statement = fmt.Sprintf("ALTER TABLE %s ADD %s %s", tableName, c.Name, c.Datatype)
	} else if step == 2 {
		// This is the step where a default value is set for the column
		// statement = cursor.mogrify("ALTER TABLE %(table)s ALTER COLUMN %(column)s SET DEFAULT %(value)s", {"table" : AsIs(table_name), "column" : AsIs(self.column_name), "value" : AsIs(self.default_value)})
		if c.DefaultExists {
			statement = fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT %s", tableName, c.Name, c.DefaultValue)
		}
	} else if step == 3 {
		// This is the step where the default value is updated for all the existing rows
		// statement = cursor.mogrify("UPDATE %(table)s SET %(column)s = %(value)s", {"table" : AsIs(table_name), "column" : AsIs(self.column_name), "value" : AsIs(self.default_value)})
		if c.DefaultExists && !columnPresent {
			statement = fmt.Sprintf("UPDATE %s SET %s = %s", tableName, c.Name, c.DefaultValue)
		}
	} else if step == 4 {
		// This is the step where the sequence is altered, in case sequence_restart is > 0 and datatype is either bigserial or serial
		// statement = cursor.mogrify("ALTER SEQUENCE %(sequence_name)s RESTART WITH %(value)s", {"sequence_name" : AsIs(sequence_name), "value" : self.sequence_restart})
		if strings.Contains(c.Datatype, "serial") {
			statement = fmt.Sprintf("ALTER SEQUENCE %s RESTART WITH %d", tableName+"_"+c.Name+"_seq", c.SequenceRestart)
		}
	} else if step == 5 {
		// This is the step where a unique constraint is added, in case the column in unique
		if c.IsUnique {
			// statement = "ALTER TABLE %s ADD UNIQUE (%s)"%(table_name, self.column_name)
			constraint := Constraint{Name: fmt.Sprintf("%s_%s_unique", tableName, c.Name), Value: fmt.Sprintf("UNIQUE (%s)", c.Name)}
			statement = fmt.Sprintf("%s; %s", constraint.createDropRule(tableName), constraint.createAddRule(tableName))
			// statement = fmt.Sprintf("ALTER TABLE %s ADD UNIQUE (%s)", tableName, c.Name)
		}
	} else if step == 6 {
		// This is the step where a primary key constraint is added, in case the column is a primary key
		if c.IsPrimary {
			// statement = "ALTER TABLE %s ADD CONSTRAINT %s PRIMARY KEY(%s)"%(table_name, table_name + "_" +self.column_name, self.column_name)
			statement = fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT %s PRIMARY KEY(%s)", tableName, tableName+"_"+c.Name, c.Name)
		}
	} else if step == 7 {
		// This is the step where NOT NULL is applied to a particular column
		if c.IsNotNull {
			// statement = "ALTER TABLE %s ALTER COLUMN %s SET NOT NULL"%(table_name, self.column_name)
			statement = fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET NOT NULL", tableName, c.Name)
		}
	} else if step == 8 {
		// This is the step where the index is created on this column
		if c.IndexRequired {
			if len(c.IndexType) > 0 {
				statement = fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_%s_index ON %s USING %s(%s)", tableName, c.Name, tableName, c.IndexType, c.Name)
			} else {
				statement = fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s_%s_index ON %s (%s)", tableName, c.Name, tableName, c.Name)
			}
		}
	} else if step == 101 {
		// This is the step where the column's datatype is altered
		if strings.Contains(c.Datatype, "serial") {
			errorStatement := fmt.Sprintf("Can't modify datatype to SERIAL versions, while modifying \nTable --> %s \nColumn --> %s \nDatatype --> %s", tableName, c.Name, c.Datatype)
			err := errors.New(errorStatement)
			return "", err
		}
		// statement = "ALTER TABLE %s ALTER COLUMN %s TYPE %s USING %s::%s"%(table_name, self.column_name, self.datatype, self.column_name, altered_datatype)
		statement = fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s TYPE %s USING %s::%s", tableName, c.Name, c.Datatype, c.Name, c.Datatype)
	}

	log.Debugln("In prepareSQLStatement, statement is \n", statement)
	return statement, nil
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"fmt"
	"os/exec"
	"syscall"
)

/*
func execCommand(app string, arg ...string) {
	log.Info("CMD: " + app + " " + strings.Join(arg[:], " "))
	cmd := exec.Command(app, strings.Join(arg[:], " "))
	stdoutStderr, err := cmd.CombinedOutput()

	if err != nil {
		log.Error(err.Error())
		fmt.Printf("%s\n", string(stdoutStderr))
		log.Error("exit program")
		programExit(1)
	}

	fmt.Printf("%s\n", string(stdoutStderr))
}
*/

func ExecCommand(command string) (int, string, string) {
	return ExecCommandAllParams(command, true)
}

func ExecCommandWithoutErrCheck(command string) (int, string, string) {
	return ExecCommandAllParams(command, false)
}

func ExecCommandAllParams(command string, checkError bool) (int, string, string) {

	// run command
	log.Debug("CMD: " + command)
	cmd := exec.Command("/bin/sh", "-c", command)
	var stdoutBuf, stderrBuf bytes.Buffer
	cmd.Stdout = &stdoutBuf
	cmd.Stderr = &stderrBuf
	err := cmd.Run()

	// convert buffer to string
	stdoutStr := stdoutBuf.String()
	stderrStr := stderrBuf.String()

	// retrieve exit code
	waitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus)
	exitCode := waitStatus.ExitStatus()

	// logging
	if stdoutStr != "" {
		log.Trace(stdoutStr)
	}
	if stdoutStr != "" {
		log.Error(stderrStr)
	}
	if checkError && (err != nil || exitCode != 0) {
		log.Fatal(err)
	}
	log.Debug(fmt.Sprintf("EXIT CODE: %d", exitCode))

	return exitCode, stdoutStr, stderrStr
}
<commit_msg>improved logging /BB<commit_after>package utils

import (
	"bytes"
	"fmt"
	"os/exec"
	"syscall"
)

/*
func execCommand(app string, arg ...string) {
	log.Info("CMD: " + app + " " + strings.Join(arg[:], " "))
	cmd := exec.Command(app, strings.Join(arg[:], " "))
	stdoutStderr, err := cmd.CombinedOutput()

	if err != nil {
		log.Error(err.Error())
		fmt.Printf("%s\n", string(stdoutStderr))
		log.Error("exit program")
		programExit(1)
	}

	fmt.Printf("%s\n", string(stdoutStderr))
}
*/

func ExecCommand(command string) (int, string, string) {
	return ExecCommandAllParams(command, true)
}

func ExecCommandWithoutErrCheck(command string) (int, string, string) {
	return ExecCommandAllParams(command, false)
}

func ExecCommandAllParams(command string, checkError bool) (int, string, string) {

	// run command
	log.Debug("CMD: " + command)
	cmd := exec.Command("/bin/sh", "-c", command)
	var stdoutBuf, stderrBuf bytes.Buffer
	cmd.Stdout = &stdoutBuf
	cmd.Stderr = &stderrBuf
	err := cmd.Run()

	// convert buffer to string
	stdoutStr := stdoutBuf.String()
	stderrStr := stderrBuf.String()

	// retrieve exit code
	waitStatus := cmd.ProcessState.Sys().(syscall.WaitStatus)
	exitCode := waitStatus.ExitStatus()

	// logging
	if stdoutStr != "" {
		log.Trace(stdoutStr)
	}
	if stdoutStr != "" {
		log.Error(stderrStr)
	}
	log.Debug(fmt.Sprintf("EXIT CODE: %d", exitCode))
	if checkError && (err != nil || exitCode != 0) {
		log.Fatal(err)
	}

	return exitCode, stdoutStr, stderrStr
}
<|endoftext|>


<commit_before>package stream

import (
	"fmt"
	"os/exec"
	"sync"
)

// Command executes "command args...".
//
// The filter's input items are fed as standard input to the command,
// one line per input item. The standard output of the command is
// split into lines and the lines form the output of the filter (with
// trailing newlines removed).
func Command(command string, args ...string) Filter {
	return FilterFunc(func(arg Arg) error {
		cmd := exec.Command(command, args...)
		input, err := cmd.StdinPipe()
		if err != nil {
			return err
		}
		output, err := cmd.StdoutPipe()
		if err != nil {
			return err
		}
		if err := cmd.Start(); err != nil {
			return err
		}
		var ierr error // Records error writing to command input
		var wg sync.WaitGroup
		wg.Add(1)
		go func() {
			for s := range arg.In {
				_, ierr = fmt.Fprintln(input, s)
				if ierr != nil {
					input.Close()
					wg.Done()
					return
				}
			}
			ierr = input.Close()
			wg.Done()
		}()
		if err := splitIntoLines(output, arg); err != nil {
			wg.Wait()
			cmd.Wait()
			return err
		}
		err = cmd.Wait()
		wg.Wait()
		if err != nil {
			return err
		}
		return ierr
	})
}
<commit_msg>use defer as Nigel suggested<commit_after>package stream

import (
	"fmt"
	"os/exec"
	"sync"
)

// Command executes "command args...".
//
// The filter's input items are fed as standard input to the command,
// one line per input item. The standard output of the command is
// split into lines and the lines form the output of the filter (with
// trailing newlines removed).
func Command(command string, args ...string) Filter {
	return FilterFunc(func(arg Arg) error {
		cmd := exec.Command(command, args...)
		input, err := cmd.StdinPipe()
		if err != nil {
			return err
		}
		output, err := cmd.StdoutPipe()
		if err != nil {
			return err
		}
		if err := cmd.Start(); err != nil {
			return err
		}
		var ierr error // Records error writing to command input
		var wg sync.WaitGroup
		wg.Add(1)
		go func() {
			defer wg.Done()
			for s := range arg.In {
				_, ierr = fmt.Fprintln(input, s)
				if ierr != nil {
					input.Close()
					return
				}
			}
			ierr = input.Close()
		}()
		if err := splitIntoLines(output, arg); err != nil {
			wg.Wait()
			cmd.Wait()
			return err
		}
		err = cmd.Wait()
		wg.Wait()
		if err != nil {
			return err
		}
		return ierr
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package subscription // import "miniflux.app/reader/subscription"

import (
	"fmt"
	"io"
	"regexp"
	"strings"

	"miniflux.app/config"
	"miniflux.app/errors"
	"miniflux.app/http/client"
	"miniflux.app/reader/browser"
	"miniflux.app/reader/parser"
	"miniflux.app/url"

	"github.com/PuerkitoBio/goquery"
)

var (
	errUnreadableDoc    = "Unable to analyze this page: %v"
	youtubeChannelRegex = regexp.MustCompile(`youtube\.com/channel/(.*)`)
	youtubeVideoRegex   = regexp.MustCompile(`youtube\.com/watch\?v=(.*)`)
)

// FindSubscriptions downloads and try to find one or more subscriptions from an URL.
func FindSubscriptions(websiteURL, userAgent, cookie, username, password string, fetchViaProxy, allowSelfSignedCertificates bool) (Subscriptions, *errors.LocalizedError) {
	websiteURL = findYoutubeChannelFeed(websiteURL)
	websiteURL = parseYoutubeVideoPage(websiteURL)

	clt := client.NewClientWithConfig(websiteURL, config.Opts)
	clt.WithCredentials(username, password)
	clt.WithUserAgent(userAgent)
	clt.WithCookie(cookie)
	clt.AllowSelfSignedCertificates = allowSelfSignedCertificates

	if fetchViaProxy {
		clt.WithProxy()
	}

	response, err := browser.Exec(clt)
	if err != nil {
		return nil, err
	}

	body := response.BodyAsString()
	if format := parser.DetectFeedFormat(body); format != parser.FormatUnknown {
		var subscriptions Subscriptions
		subscriptions = append(subscriptions, &Subscription{
			Title: response.EffectiveURL,
			URL:   response.EffectiveURL,
			Type:  format,
		})

		return subscriptions, nil
	}

	subscriptions, err := parseWebPage(response.EffectiveURL, strings.NewReader(body))
	if err != nil || subscriptions != nil {
		return subscriptions, err
	}

	return tryWellKnownUrls(websiteURL, userAgent, cookie, username, password)
}

func parseWebPage(websiteURL string, data io.Reader) (Subscriptions, *errors.LocalizedError) {
	var subscriptions Subscriptions
	queries := map[string]string{
		"link[type='application/rss+xml']":  "rss",
		"link[type='application/atom+xml']": "atom",
		"link[type='application/json']":     "json",
	}

	doc, err := goquery.NewDocumentFromReader(data)
	if err != nil {
		return nil, errors.NewLocalizedError(errUnreadableDoc, err)
	}

	for query, kind := range queries {
		doc.Find(query).Each(func(i int, s *goquery.Selection) {
			subscription := new(Subscription)
			subscription.Type = kind

			if title, exists := s.Attr("title"); exists {
				subscription.Title = title
			} else {
				subscription.Title = "Feed"
			}

			if feedURL, exists := s.Attr("href"); exists {
				subscription.URL, _ = url.AbsoluteURL(websiteURL, feedURL)
			}

			if subscription.Title == "" {
				subscription.Title = subscription.URL
			}

			if subscription.URL != "" {
				subscriptions = append(subscriptions, subscription)
			}
		})
	}

	return subscriptions, nil
}

func findYoutubeChannelFeed(websiteURL string) string {
	matches := youtubeChannelRegex.FindStringSubmatch(websiteURL)

	if len(matches) == 2 {
		return fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, matches[1])
	}
	return websiteURL
}

func parseYoutubeVideoPage(websiteURL string) string {
	if !youtubeVideoRegex.MatchString(websiteURL) {
		return websiteURL
	}

	clt := client.NewClientWithConfig(websiteURL, config.Opts)
	response, browserErr := browser.Exec(clt)
	if browserErr != nil {
		return websiteURL
	}

	doc, docErr := goquery.NewDocumentFromReader(response.Body)
	if docErr != nil {
		return websiteURL
	}

	if channelID, exists := doc.Find(`meta[itemprop="channelId"]`).First().Attr("content"); exists {
		return fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, channelID)
	}

	return websiteURL
}

func tryWellKnownUrls(websiteURL, userAgent, cookie, username, password string) (Subscriptions, *errors.LocalizedError) {
	var subscriptions Subscriptions
	knownURLs := map[string]string{
		"/atom.xml": "atom",
		"/feed.xml": "atom",
		"/feed/":    "atom",
		"/rss.xml":  "rss",
		"/rss/":     "rss",
	}

	lastCharacter := websiteURL[len(websiteURL)-1:]
	if lastCharacter == "/" {
		websiteURL = websiteURL[:len(websiteURL)-1]
	}

	for knownURL, kind := range knownURLs {
		fullURL, err := url.AbsoluteURL(websiteURL, knownURL)
		if err != nil {
			continue
		}

		clt := client.NewClientWithConfig(fullURL, config.Opts)
		clt.WithCredentials(username, password)
		clt.WithUserAgent(userAgent)
		clt.WithCookie(cookie)

		// Some websites redirects unknown URLs to the home page.
		// As result, the list of known URLs is returned to the subscription list.
		// We don't want the user to choose between invalid feed URLs.
		clt.WithoutRedirects()

		response, err := clt.Get()
		if err != nil {
			continue
		}

		if response != nil && response.StatusCode == 200 {
			subscription := new(Subscription)
			subscription.Type = kind
			subscription.Title = fullURL
			subscription.URL = fullURL
			if subscription.URL != "" {
				subscriptions = append(subscriptions, subscription)
			}
		}
	}

	return subscriptions, nil
}
<commit_msg>Fix finding JSON feeds with new MIME type<commit_after>// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package subscription // import "miniflux.app/reader/subscription"

import (
	"fmt"
	"io"
	"regexp"
	"strings"

	"miniflux.app/config"
	"miniflux.app/errors"
	"miniflux.app/http/client"
	"miniflux.app/reader/browser"
	"miniflux.app/reader/parser"
	"miniflux.app/url"

	"github.com/PuerkitoBio/goquery"
)

var (
	errUnreadableDoc    = "Unable to analyze this page: %v"
	youtubeChannelRegex = regexp.MustCompile(`youtube\.com/channel/(.*)`)
	youtubeVideoRegex   = regexp.MustCompile(`youtube\.com/watch\?v=(.*)`)
)

// FindSubscriptions downloads and try to find one or more subscriptions from an URL.
func FindSubscriptions(websiteURL, userAgent, cookie, username, password string, fetchViaProxy, allowSelfSignedCertificates bool) (Subscriptions, *errors.LocalizedError) {
	websiteURL = findYoutubeChannelFeed(websiteURL)
	websiteURL = parseYoutubeVideoPage(websiteURL)

	clt := client.NewClientWithConfig(websiteURL, config.Opts)
	clt.WithCredentials(username, password)
	clt.WithUserAgent(userAgent)
	clt.WithCookie(cookie)
	clt.AllowSelfSignedCertificates = allowSelfSignedCertificates

	if fetchViaProxy {
		clt.WithProxy()
	}

	response, err := browser.Exec(clt)
	if err != nil {
		return nil, err
	}

	body := response.BodyAsString()
	if format := parser.DetectFeedFormat(body); format != parser.FormatUnknown {
		var subscriptions Subscriptions
		subscriptions = append(subscriptions, &Subscription{
			Title: response.EffectiveURL,
			URL:   response.EffectiveURL,
			Type:  format,
		})

		return subscriptions, nil
	}

	subscriptions, err := parseWebPage(response.EffectiveURL, strings.NewReader(body))
	if err != nil || subscriptions != nil {
		return subscriptions, err
	}

	return tryWellKnownUrls(websiteURL, userAgent, cookie, username, password)
}

func parseWebPage(websiteURL string, data io.Reader) (Subscriptions, *errors.LocalizedError) {
	var subscriptions Subscriptions
	queries := map[string]string{
		"link[type='application/rss+xml']":   "rss",
		"link[type='application/atom+xml']":  "atom",
		"link[type='application/json']":      "json",
		"link[type='application/feed+json']": "json",
	}

	doc, err := goquery.NewDocumentFromReader(data)
	if err != nil {
		return nil, errors.NewLocalizedError(errUnreadableDoc, err)
	}

	for query, kind := range queries {
		doc.Find(query).Each(func(i int, s *goquery.Selection) {
			subscription := new(Subscription)
			subscription.Type = kind

			if title, exists := s.Attr("title"); exists {
				subscription.Title = title
			} else {
				subscription.Title = "Feed"
			}

			if feedURL, exists := s.Attr("href"); exists {
				subscription.URL, _ = url.AbsoluteURL(websiteURL, feedURL)
			}

			if subscription.Title == "" {
				subscription.Title = subscription.URL
			}

			if subscription.URL != "" {
				subscriptions = append(subscriptions, subscription)
			}
		})
	}

	return subscriptions, nil
}

func findYoutubeChannelFeed(websiteURL string) string {
	matches := youtubeChannelRegex.FindStringSubmatch(websiteURL)

	if len(matches) == 2 {
		return fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, matches[1])
	}
	return websiteURL
}

func parseYoutubeVideoPage(websiteURL string) string {
	if !youtubeVideoRegex.MatchString(websiteURL) {
		return websiteURL
	}

	clt := client.NewClientWithConfig(websiteURL, config.Opts)
	response, browserErr := browser.Exec(clt)
	if browserErr != nil {
		return websiteURL
	}

	doc, docErr := goquery.NewDocumentFromReader(response.Body)
	if docErr != nil {
		return websiteURL
	}

	if channelID, exists := doc.Find(`meta[itemprop="channelId"]`).First().Attr("content"); exists {
		return fmt.Sprintf(`https://www.youtube.com/feeds/videos.xml?channel_id=%s`, channelID)
	}

	return websiteURL
}

func tryWellKnownUrls(websiteURL, userAgent, cookie, username, password string) (Subscriptions, *errors.LocalizedError) {
	var subscriptions Subscriptions
	knownURLs := map[string]string{
		"/atom.xml": "atom",
		"/feed.xml": "atom",
		"/feed/":    "atom",
		"/rss.xml":  "rss",
		"/rss/":     "rss",
	}

	lastCharacter := websiteURL[len(websiteURL)-1:]
	if lastCharacter == "/" {
		websiteURL = websiteURL[:len(websiteURL)-1]
	}

	for knownURL, kind := range knownURLs {
		fullURL, err := url.AbsoluteURL(websiteURL, knownURL)
		if err != nil {
			continue
		}

		clt := client.NewClientWithConfig(fullURL, config.Opts)
		clt.WithCredentials(username, password)
		clt.WithUserAgent(userAgent)
		clt.WithCookie(cookie)

		// Some websites redirects unknown URLs to the home page.
		// As result, the list of known URLs is returned to the subscription list.
		// We don't want the user to choose between invalid feed URLs.
		clt.WithoutRedirects()

		response, err := clt.Get()
		if err != nil {
			continue
		}

		if response != nil && response.StatusCode == 200 {
			subscription := new(Subscription)
			subscription.Type = kind
			subscription.Title = fullURL
			subscription.URL = fullURL
			if subscription.URL != "" {
				subscriptions = append(subscriptions, subscription)
			}
		}
	}

	return subscriptions, nil
}
<|endoftext|>


<commit_before>package rtfdoc

import "fmt"

// AddParagraph return new instance of Paragraph
func (doc *Document) AddParagraph() *Paragraph {
	p := Paragraph{
		align:   AlignCenter,
		indent:  "\\fl360",
		content: nil,
		generalSettings: generalSettings{
			colorTable: doc.colorTable,
			fontColor:  doc.fontColor,
		},
		allowedWidth: doc.maxWidth,
	}
	p.updateMaxWidth()
	doc.content = append(doc.content, &p)
	return &p
}

func (par *Paragraph) updateMaxWidth() *Paragraph {
	par.maxWidth = par.allowedWidth
	return par
}

func (par Paragraph) compose() string {
	indentStr := fmt.Sprintf("\\fi%d \\li%d \\ri%d",
		par.indentFirstLine,
		par.indentLeftIndent,
		par.indentRightIndent)
	res := fmt.Sprintf("\n\\pard \\q%s { %s ", par.align, indentStr)
	if par.isTable {
		res += "\\intbl"
	}
	// res += fmt.Sprintf(" \\q%s", par.align)

	for _, c := range par.content {
		res += c.compose()
	}
	// res += "\n\\par}"
	res += "}"
	if !par.isTable {
		res += "\\par"
	}
	return res
}

// SetIndentFirstLine function sets first line indent in twips
func (par *Paragraph) SetIndentFirstLine(value int) *Paragraph {
	par.indentFirstLine = value
	return par
}

// SetIndentRight function sets right indent in twips
func (par *Paragraph) SetIndentRight(value int) *Paragraph {
	par.indentRightIndent = value
	return par
}

// SetIndentLeft function sets left indent in twips
func (par *Paragraph) SetIndentLeft(value int) *Paragraph {
	par.indentLeftIndent = value
	return par
}

// SetAlign sets Paragraph align (c/center, l/left, r/right, j/justify)
func (par *Paragraph) SetAlign(align string) *Paragraph {
	for _, i := range []string{
		AlignCenter,
		AlignLeft,
		AlignRight,
		AlignJustify,
		AlignDistribute,
	} {
		if i == align {
			par.align = i
		}
	}

	return par
}
<commit_msg>remove space<commit_after>package rtfdoc

import "fmt"

// AddParagraph return new instance of Paragraph
func (doc *Document) AddParagraph() *Paragraph {
	p := Paragraph{
		align:   AlignCenter,
		indent:  "\\fl360",
		content: nil,
		generalSettings: generalSettings{
			colorTable: doc.colorTable,
			fontColor:  doc.fontColor,
		},
		allowedWidth: doc.maxWidth,
	}
	p.updateMaxWidth()
	doc.content = append(doc.content, &p)
	return &p
}

func (par *Paragraph) updateMaxWidth() *Paragraph {
	par.maxWidth = par.allowedWidth
	return par
}

func (par Paragraph) compose() string {
	indentStr := fmt.Sprintf("\\fi%d \\li%d \\ri%d",
		par.indentFirstLine,
		par.indentLeftIndent,
		par.indentRightIndent)
	res := fmt.Sprintf("\n\\pard \\q%s {%s ", par.align, indentStr)
	if par.isTable {
		res += "\\intbl"
	}
	// res += fmt.Sprintf(" \\q%s", par.align)

	for _, c := range par.content {
		res += c.compose()
	}
	// res += "\n\\par}"
	res += "}"
	if !par.isTable {
		res += "\\par"
	}
	return res
}

// SetIndentFirstLine function sets first line indent in twips
func (par *Paragraph) SetIndentFirstLine(value int) *Paragraph {
	par.indentFirstLine = value
	return par
}

// SetIndentRight function sets right indent in twips
func (par *Paragraph) SetIndentRight(value int) *Paragraph {
	par.indentRightIndent = value
	return par
}

// SetIndentLeft function sets left indent in twips
func (par *Paragraph) SetIndentLeft(value int) *Paragraph {
	par.indentLeftIndent = value
	return par
}

// SetAlign sets Paragraph align (c/center, l/left, r/right, j/justify)
func (par *Paragraph) SetAlign(align string) *Paragraph {
	for _, i := range []string{
		AlignCenter,
		AlignLeft,
		AlignRight,
		AlignJustify,
		AlignDistribute,
	} {
		if i == align {
			par.align = i
		}
	}

	return par
}
<|endoftext|>


<commit_before>package reflector

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

// Utility to test performance of some typical operations, run with:
// N=1000000 go test -v ./... -run=TestPerformance
// Iy you change anything here, change TestPerformancePlain too!
func TestPerformance(t *testing.T) {
	n, _ := strconv.ParseInt(os.Getenv("N"), 10, 64)
	if n <= 0 {
		n = 1000
	}
	started := time.Now()
	for i := 0; i < int(n); i++ {
		p := &Person{}
		obj := New(p)

		err := obj.Field("Number").Set(i)
		if err != nil {
			panic("Should not error")
		}
		if p.Number != i {
			panic("Should be " + string(i))
		}
		number, err := obj.Field("Number").Get()
		if err != nil {
			panic("Number is valid")
		}
		if number.(int) != i {
			panic("Should be " + string(i))
		}
		res, err := obj.Method("Add").Call(1, 2, 3)
		if err != nil {
			panic("shouldn't be an error")
		}
		if res.IsError() {
			panic("method shouldn't return an error")
		}
		if len(res.Result) != 1 && res.Result[0].(int) != 6 {
			panic("result should be 6")
		}
	}

	assert.Equal(t, 1, metadataCached, "Only 1 metadata must be cached")
	assert.Equal(t, 1, len(metadataCache), "Only 1 metadata must be cached")

	ended := time.Now()
	fmt.Println("WITH REFLECTION")
	fmt.Println("    n=", n)
	fmt.Println("    started:", started.Format("2006-01-02 15:04:05.123"))
	fmt.Println("    ended:", ended.Format("2006-01-02 15:04:05.123"))
	fmt.Printf("    duration: %fs\n", ended.Sub(started).Seconds())
}

func TestPerformancePlain(t *testing.T) {
	n, _ := strconv.ParseInt(os.Getenv("N"), 10, 64)
	if n <= 0 {
		n = 1000
	}
	started := time.Now()
	for i := 0; i < int(n); i++ {
		p := &Person{}

		p.Number = i
		number := p.Number
		if number != i {
			panic("Should be " + string(i))
		}
		res := p.Add(1, 2, 3)
		if res != 6 {
			panic("result should be 6")
		}
	}

	ended := time.Now()
	fmt.Println("WITHOUT REFLECTION")
	fmt.Println("    n=", n)
	fmt.Println("    started:", started.Format("2006-01-02 15:04:05.123"))
	fmt.Println("    ended:", ended.Format("2006-01-02 15:04:05.123"))
	fmt.Printf("    duration: %fs\n", ended.Sub(started).Seconds())
}
<commit_msg>clean performance tests<commit_after>package reflector

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func stopwatch(n int, title string) func() {
	start := time.Now()
	return func() {
		fmt.Println(title)
		fmt.Printf("%12s= %d\n", "n", n)
		const dateFormat = "2006-01-02 15:04:05.123"
		fmt.Printf("%12s: %s\n", "started", start.Format(dateFormat))
		fmt.Printf("%12s: %fs\n", "duration", time.Since(start).Seconds())
	}
}

func performanceN(envN string) int {
	n, _ := strconv.ParseInt(envN, 10, 64)
	const defaultN = 1000
	if n < 1 {
		n = defaultN
	}
	return int(n)
}

// Utility to test performance of some typical operations, run with:
// N=1000000 go test -v ./... -run=TestPerformance
// Iy you change anything here, change TestPerformance_plain too!
func TestPerformance_reflection(t *testing.T) {
	n := performanceN(os.Getenv("N"))
	defer stopwatch(n, "WITH REFLECTION")()
	for i := 0; i < n; i++ {
		p := &Person{}
		obj := New(p)

		err := obj.Field("Number").Set(i)
		if err != nil {
			t.Fatal("Should not error")
		}
		if p.Number != i {
			t.Fatalf("Should be %d", i)
		}
		number, err := obj.Field("Number").Get()
		if err != nil {
			t.Fatal("Number is valid")
		}
		if number.(int) != i {
			t.Fatalf("Should be %d", i)
		}
		res, err := obj.Method("Add").Call(1, 2, 3)
		if err != nil {
			t.Fatal("shouldn't be an error")
		}
		if res.IsError() {
			t.Fatal("method shouldn't return an error")
		}
		if len(res.Result) != 1 && res.Result[0].(int) != 6 {
			t.Fatal("result should be 6")
		}
	}

	assert.Equal(t, 1, metadataCached, "Only 1 metadata must be cached")
	assert.Equal(t, 1, len(metadataCache), "Only 1 metadata must be cached")
}

func TestPerformance_plain(t *testing.T) {
	n := performanceN(os.Getenv("N"))
	defer stopwatch(n, "WITHOUT REFLECTION")()
	for i := 0; i < n; i++ {
		p := &Person{}

		p.Number = i
		number := p.Number
		if number != i {
			t.Fatalf("Should be %d", i)
		}
		res := p.Add(1, 2, 3)
		if res != 6 {
			t.Fatal("result should be 6")
		}
	}
}
<|endoftext|>


<commit_before>package horizon

import (
	"github.com/stellar/go-horizon/test"
	"log"
)

func NewTestApp() *App {
	app, err := NewApp(Config{DatabaseUrl: test.DatabaseUrl()})

	if err != nil {
		log.Panic(err)
	}

	return app
}
<commit_msg>Log db queries in test suite<commit_after>package horizon

import (
	"github.com/stellar/go-horizon/test"
	"log"
)

func NewTestApp() *App {
	app, err := NewApp(Config{DatabaseUrl: test.DatabaseUrl()})

	if err != nil {
		log.Panic(err)
	}

	app.historyDb.LogMode(true)

	return app
}
<|endoftext|>


<commit_before>// Copyright ©2012 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package kdtree

import (
	"fmt"
	"math/rand"
	"sort"
)

// Partition partitions list such that all elements less than the value at pivot prior to the
// call are placed before that element and all elements greater than that value are placed after it.
// The final location of the element at pivot prior to the call is returned.
func Partition(list sort.Interface, pivot int) int {
	var index, last int
	if last = list.Len() - 1; last < 0 {
		return -1
	}
	list.Swap(pivot, last)
	for i := 0; i < last; i++ {
		if !list.Less(last, i) {
			list.Swap(index, i)
			index++
		}
	}
	list.Swap(last, index)
	return index
}

// A SortSlicer satisfies the sort.Interface and is able to slice itself.
type SortSlicer interface {
	sort.Interface
	Slice(start, end int) SortSlicer
}

// Select partitions list such that all elements less than the kth largest element are
// placed placed before k in the resulting list and all elements greater than it are placed
// after the position k.
func Select(list SortSlicer, k int) int {
	var (
		start int
		end   = list.Len()
	)
	if k >= end {
		if k == 0 {
			return 0
		}
		panic(fmt.Sprintf("kdtree: index out of range"))
	}
	if start == end-1 {
		return k
	}

	for {
		if start == end {
			panic("kdtree: internal inconsistency")
		}
		sub := list.Slice(start, end)
		pivot := Partition(sub, rand.Intn(sub.Len()))
		switch {
		case pivot == k:
			return k
		case k < pivot:
			end = pivot + start
		default:
			k -= pivot
			start += pivot
		}
	}

	panic("cannot reach")
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

// MedianOfMedians returns the index to the median value of the medians of groups of 5 consecutive elements.
func MedianOfMedians(list SortSlicer) int {
	n := list.Len() / 5
	for i := 0; i < n; i++ {
		left := i * 5
		sub := list.Slice(left, min(left+5, list.Len()-1))
		Select(sub, 2)
		list.Swap(i, left+2)
	}
	Select(list.Slice(0, min(n, list.Len()-1)), min(list.Len(), n/2))
	return n / 2
}

// RandomOfMedians returns the index to the median value of up to n randomly chosen elements in list.
func MedianOfRandoms(list SortSlicer, n int) int {
	if l := list.Len(); n <= l {
		for i := 0; i < n; i++ {
			list.Swap(i, rand.Intn(n))
		}
	} else {
		n = l
	}
	Select(list.Slice(0, n), n/2)
	return n / 2
}
<commit_msg>Doc bug<commit_after>// Copyright ©2012 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package kdtree

import (
	"fmt"
	"math/rand"
	"sort"
)

// Partition partitions list such that all elements less than the value at pivot prior to the
// call are placed before that element and all elements greater than that value are placed after it.
// The final location of the element at pivot prior to the call is returned.
func Partition(list sort.Interface, pivot int) int {
	var index, last int
	if last = list.Len() - 1; last < 0 {
		return -1
	}
	list.Swap(pivot, last)
	for i := 0; i < last; i++ {
		if !list.Less(last, i) {
			list.Swap(index, i)
			index++
		}
	}
	list.Swap(last, index)
	return index
}

// A SortSlicer satisfies the sort.Interface and is able to slice itself.
type SortSlicer interface {
	sort.Interface
	Slice(start, end int) SortSlicer
}

// Select partitions list such that all elements less than the kth largest element are
// placed placed before k in the resulting list and all elements greater than it are placed
// after the position k.
func Select(list SortSlicer, k int) int {
	var (
		start int
		end   = list.Len()
	)
	if k >= end {
		if k == 0 {
			return 0
		}
		panic(fmt.Sprintf("kdtree: index out of range"))
	}
	if start == end-1 {
		return k
	}

	for {
		if start == end {
			panic("kdtree: internal inconsistency")
		}
		sub := list.Slice(start, end)
		pivot := Partition(sub, rand.Intn(sub.Len()))
		switch {
		case pivot == k:
			return k
		case k < pivot:
			end = pivot + start
		default:
			k -= pivot
			start += pivot
		}
	}

	panic("cannot reach")
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

// MedianOfMedians returns the index to the median value of the medians of groups of 5 consecutive elements.
func MedianOfMedians(list SortSlicer) int {
	n := list.Len() / 5
	for i := 0; i < n; i++ {
		left := i * 5
		sub := list.Slice(left, min(left+5, list.Len()-1))
		Select(sub, 2)
		list.Swap(i, left+2)
	}
	Select(list.Slice(0, min(n, list.Len()-1)), min(list.Len(), n/2))
	return n / 2
}

// MedianOfRandoms returns the index to the median value of up to n randomly chosen elements in list.
func MedianOfRandoms(list SortSlicer, n int) int {
	if l := list.Len(); n <= l {
		for i := 0; i < n; i++ {
			list.Swap(i, rand.Intn(n))
		}
	} else {
		n = l
	}
	Select(list.Slice(0, n), n/2)
	return n / 2
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
)

type Server struct {
	Parse Parse
}

func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	notification, err := s.Parse(r)
	if err != nil {
		log.Printf("Unable to parse request: %s\n", err)
		return
	}

	repo := notification.RepositoryUrl()

	fmt.Fprintln(w, repo)
	log.Printf("Received notification for repository %q", repo)

	if repositoryConfig, found := config.FindRepositoryConfig(notification); found {
		log.Printf("Executing command %q in %q", repositoryConfig.Command, repositoryConfig.Dir)
		cmd := exec.Command("/bin/sh", "-c", repositoryConfig.Command)
		cmd.Dir = repositoryConfig.Dir
		out, err := cmd.CombinedOutput()
		log.Printf("Command output: %q", string(out))
		if err != nil {
			log.Printf("Command exited with error: %s\n", err)
		}
	} else {
		log.Printf("Repo/branch not configured.")
	}

}

var configFileName string
var config Config

func init() {
	flag.StringVar(&configFileName, "c", "/etc/hookreceiver.conf.d", "Config file or directory name")
}

func main() {
	// Call realMain instead of doing the work here so we can use
	// `defer` statements within the function and have them work properly.
	// (defers aren't called with os.Exit)

	flag.Parse()
	os.Exit(realMain())
}

// realMain is executed from main and returns the exit status to exit with.
func realMain() int {
	var err error
	config, err = readConfig(configFileName)
	if err != nil {
		log.Fatal(err)
		return 1
	}

	http.Handle("/hooks/bitbucket/", Server{Parse: BitbucketParse})
	if err := http.ListenAndServe(config.Addr, nil); err != nil {
		log.Fatal(err)
		return 1
	}

	return 0
}
<commit_msg>Log branches<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
)

type Server struct {
	Parse Parse
}

func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	notification, err := s.Parse(r)
	if err != nil {
		log.Printf("Unable to parse request: %s\n", err)
		return
	}

	repo := notification.RepositoryUrl()
	branches := notification.Branches()

	fmt.Fprintln(w, repo)
	log.Printf("Received notification for repository %q branches %q", repo, branches)

	if repositoryConfig, found := config.FindRepositoryConfig(notification); found {
		log.Printf("Executing command %q in %q", repositoryConfig.Command, repositoryConfig.Dir)
		cmd := exec.Command("/bin/sh", "-c", repositoryConfig.Command)
		cmd.Dir = repositoryConfig.Dir
		out, err := cmd.CombinedOutput()
		log.Printf("Command output: %q", string(out))
		if err != nil {
			log.Printf("Command exited with error: %s\n", err)
		}
	} else {
		log.Printf("Repo/branch not configured.")
	}

}

var configFileName string
var config Config

func init() {
	flag.StringVar(&configFileName, "c", "/etc/hookreceiver.conf.d", "Config file or directory name")
}

func main() {
	// Call realMain instead of doing the work here so we can use
	// `defer` statements within the function and have them work properly.
	// (defers aren't called with os.Exit)

	flag.Parse()
	os.Exit(realMain())
}

// realMain is executed from main and returns the exit status to exit with.
func realMain() int {
	var err error
	config, err = readConfig(configFileName)
	if err != nil {
		log.Fatal(err)
		return 1
	}

	http.Handle("/hooks/bitbucket/", Server{Parse: BitbucketParse})
	if err := http.ListenAndServe(config.Addr, nil); err != nil {
		log.Fatal(err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type containerPutReq struct {
	Architecture string            `json:"architecture"`
	Config       map[string]string `json:"config"`
	Devices      shared.Devices    `json:"devices"`
	Ephemeral    bool              `json:"ephemeral"`
	Profiles     []string          `json:"profiles"`
	Restore      string            `json:"restore"`
}

/*
 * Update configuration, or, if 'restore:snapshot-name' is present, restore
 * the named snapshot
 */
func containerPut(d *Daemon, r *http.Request) Response {
	// Get the container
	name := mux.Vars(r)["name"]
	c, err := containerLoadByName(d, name)
	if err != nil {
		return NotFound
	}

	// Validate the ETag
	etag := []interface{}{c.Architecture(), c.LocalConfig(), c.LocalDevices(), c.IsEphemeral(), c.Profiles()}
	err = etagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	configRaw := containerPutReq{}
	if err := json.NewDecoder(r.Body).Decode(&configRaw); err != nil {
		return BadRequest(err)
	}

	architecture, err := shared.ArchitectureId(configRaw.Architecture)
	if err != nil {
		architecture = 0
	}

	var do = func(*operation) error { return nil }

	if configRaw.Restore == "" {
		// Update container configuration
		do = func(op *operation) error {
			args := containerArgs{
				Architecture: architecture,
				Config:       configRaw.Config,
				Devices:      configRaw.Devices,
				Ephemeral:    configRaw.Ephemeral,
				Profiles:     configRaw.Profiles}

			// FIXME: should set to true when not migrating
			err = c.Update(args, false)
			if err != nil {
				return err
			}

			return nil
		}
	} else {
		// Snapshot Restore
		do = func(op *operation) error {
			return containerSnapRestore(d, name, configRaw.Restore)
		}
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operationCreate(operationClassTask, resources, nil, do, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerSnapRestore(d *Daemon, name string, snap string) error {
	// normalize snapshot name
	if !shared.IsSnapshot(snap) {
		snap = name + shared.SnapshotDelimiter + snap
	}

	shared.Log.Info(
		"RESTORE => Restoring snapshot",
		log.Ctx{
			"snapshot":  snap,
			"container": name})

	c, err := containerLoadByName(d, name)
	if err != nil {
		shared.Log.Error(
			"RESTORE => loadcontainerLXD() failed",
			log.Ctx{
				"container": name,
				"err":       err})
		return err
	}

	source, err := containerLoadByName(d, snap)
	if err != nil {
		switch err {
		case sql.ErrNoRows:
			return fmt.Errorf("snapshot %s does not exist", snap)
		default:
			return err
		}
	}

	if err := c.Restore(source); err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/container_put: switch to new logging functions<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type containerPutReq struct {
	Architecture string            `json:"architecture"`
	Config       map[string]string `json:"config"`
	Devices      shared.Devices    `json:"devices"`
	Ephemeral    bool              `json:"ephemeral"`
	Profiles     []string          `json:"profiles"`
	Restore      string            `json:"restore"`
}

/*
 * Update configuration, or, if 'restore:snapshot-name' is present, restore
 * the named snapshot
 */
func containerPut(d *Daemon, r *http.Request) Response {
	// Get the container
	name := mux.Vars(r)["name"]
	c, err := containerLoadByName(d, name)
	if err != nil {
		return NotFound
	}

	// Validate the ETag
	etag := []interface{}{c.Architecture(), c.LocalConfig(), c.LocalDevices(), c.IsEphemeral(), c.Profiles()}
	err = etagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	configRaw := containerPutReq{}
	if err := json.NewDecoder(r.Body).Decode(&configRaw); err != nil {
		return BadRequest(err)
	}

	architecture, err := shared.ArchitectureId(configRaw.Architecture)
	if err != nil {
		architecture = 0
	}

	var do = func(*operation) error { return nil }

	if configRaw.Restore == "" {
		// Update container configuration
		do = func(op *operation) error {
			args := containerArgs{
				Architecture: architecture,
				Config:       configRaw.Config,
				Devices:      configRaw.Devices,
				Ephemeral:    configRaw.Ephemeral,
				Profiles:     configRaw.Profiles}

			// FIXME: should set to true when not migrating
			err = c.Update(args, false)
			if err != nil {
				return err
			}

			return nil
		}
	} else {
		// Snapshot Restore
		do = func(op *operation) error {
			return containerSnapRestore(d, name, configRaw.Restore)
		}
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operationCreate(operationClassTask, resources, nil, do, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerSnapRestore(d *Daemon, name string, snap string) error {
	// normalize snapshot name
	if !shared.IsSnapshot(snap) {
		snap = name + shared.SnapshotDelimiter + snap
	}

	shared.LogInfo(
		"RESTORE => Restoring snapshot",
		log.Ctx{
			"snapshot":  snap,
			"container": name})

	c, err := containerLoadByName(d, name)
	if err != nil {
		shared.LogError(
			"RESTORE => loadcontainerLXD() failed",
			log.Ctx{
				"container": name,
				"err":       err})
		return err
	}

	source, err := containerLoadByName(d, snap)
	if err != nil {
		switch err {
		case sql.ErrNoRows:
			return fmt.Errorf("snapshot %s does not exist", snap)
		default:
			return err
		}
	}

	if err := c.Restore(source); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"database/sql"
	"encoding/hex"
	"fmt"
	"io"
	"os/exec"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/crypto/scrypt"
	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/shared"
)

var daemonConfigLock sync.Mutex
var daemonConfig map[string]*daemonConfigKey

type daemonConfigKey struct {
	valueType    string
	defaultValue string
	validValues  []string
	currentValue string
	hiddenValue  bool

	validator func(d *Daemon, key string, value string) error
	setter    func(d *Daemon, key string, value string) (string, error)
	trigger   func(d *Daemon, key string, value string)
}

func (k *daemonConfigKey) name() string {
	name := ""

	// Look for a matching entry in daemonConfig
	daemonConfigLock.Lock()
	for key, value := range daemonConfig {
		if value == k {
			name = key
			break
		}
	}
	daemonConfigLock.Unlock()

	return name
}

func (k *daemonConfigKey) Validate(d *Daemon, value string) error {
	// Handle unsetting
	if value == "" {
		value = k.defaultValue

		if k.validator != nil {
			err := k.validator(d, k.name(), value)
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Validate booleans
	if k.valueType == "bool" && !shared.StringInSlice(strings.ToLower(value), []string{"true", "false", "1", "0", "yes", "no", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	// Validate integers
	if k.valueType == "int" {
		_, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			return err
		}
	}

	// Check against valid values
	if k.validValues != nil && !shared.StringInSlice(value, k.validValues) {
		return fmt.Errorf("Invalid value, only the following values are allowed: %s", k.validValues)
	}

	// Run external validation function
	if k.validator != nil {
		err := k.validator(d, k.name(), value)
		if err != nil {
			return err
		}
	}

	return nil
}

func (k *daemonConfigKey) Set(d *Daemon, value string) error {
	var name string

	// Check if we are actually changing things
	oldValue := k.currentValue
	if oldValue == value {
		return nil
	}

	// Validate the new value
	err := k.Validate(d, value)
	if err != nil {
		return err
	}

	// Run external setting function
	if k.setter != nil {
		value, err = k.setter(d, k.name(), value)
		if err != nil {
			return err
		}
	}

	// Get the configuration key and make sure daemonConfig is sane
	name = k.name()
	if name == "" {
		return fmt.Errorf("Corrupted configuration cache")
	}

	// Actually apply the change
	daemonConfigLock.Lock()
	k.currentValue = value
	daemonConfigLock.Unlock()

	err = dbConfigValueSet(d.db, name, value)
	if err != nil {
		return err
	}

	return nil
}

func (k *daemonConfigKey) Get() string {
	value := k.currentValue

	// Get the default value if not set
	if value == "" {
		value = k.defaultValue
	}

	return value
}

func (k *daemonConfigKey) GetBool() bool {
	value := k.currentValue

	// Get the default value if not set
	if value == "" {
		value = k.defaultValue
	}

	// Convert to boolean
	return shared.IsTrue(value)
}

func (k *daemonConfigKey) GetInt64() int64 {
	value := k.currentValue

	// Get the default value if not set
	if value == "" {
		value = k.defaultValue
	}

	// Convert to int64
	ret, _ := strconv.ParseInt(value, 10, 64)
	return ret
}

func daemonConfigInit(db *sql.DB) error {
	// Set all the keys
	daemonConfig = map[string]*daemonConfigKey{
		"core.https_address":             {valueType: "string", setter: daemonConfigSetAddress},
		"core.https_allowed_headers":     {valueType: "string"},
		"core.https_allowed_methods":     {valueType: "string"},
		"core.https_allowed_origin":      {valueType: "string"},
		"core.https_allowed_credentials": {valueType: "bool"},
		"core.proxy_http":                {valueType: "string", setter: daemonConfigSetProxy},
		"core.proxy_https":               {valueType: "string", setter: daemonConfigSetProxy},
		"core.proxy_ignore_hosts":        {valueType: "string", setter: daemonConfigSetProxy},
		"core.trust_password":            {valueType: "string", hiddenValue: true, setter: daemonConfigSetPassword},

		"images.auto_update_cached":    {valueType: "bool", defaultValue: "true"},
		"images.auto_update_interval":  {valueType: "int", defaultValue: "6"},
		"images.compression_algorithm": {valueType: "string", validator: daemonConfigValidateCompression, defaultValue: "gzip"},
		"images.remote_cache_expiry":   {valueType: "int", defaultValue: "10", trigger: daemonConfigTriggerExpiry},

		// Keys deprecated since the implementation of the storage api.
		"storage.lvm_fstype":           {valueType: "string", defaultValue: "ext4", validValues: []string{"ext4", "xfs"}, validator: storageDeprecatedKeys},
		"storage.lvm_mount_options":    {valueType: "string", defaultValue: "discard", validator: storageDeprecatedKeys},
		"storage.lvm_thinpool_name":    {valueType: "string", defaultValue: "LXDPool", validator: storageDeprecatedKeys},
		"storage.lvm_vg_name":          {valueType: "string", validator: storageDeprecatedKeys},
		"storage.lvm_volume_size":      {valueType: "string", defaultValue: "10GiB", validator: storageDeprecatedKeys},
		"storage.zfs_pool_name":        {valueType: "string", validator: storageDeprecatedKeys},
		"storage.zfs_remove_snapshots": {valueType: "bool", validator: storageDeprecatedKeys},
		"storage.zfs_use_refquota":     {valueType: "bool", validator: storageDeprecatedKeys},
	}

	// Load the values from the DB
	dbValues, err := dbConfigValuesGet(db)
	if err != nil {
		return err
	}

	daemonConfigLock.Lock()
	for k, v := range dbValues {
		_, ok := daemonConfig[k]
		if !ok {
			shared.LogError("Found invalid configuration key in database", log.Ctx{"key": k})
		}

		daemonConfig[k].currentValue = v
	}
	daemonConfigLock.Unlock()

	return nil
}

func daemonConfigRender() map[string]interface{} {
	config := map[string]interface{}{}

	// Turn the config into a JSON-compatible map
	for k, v := range daemonConfig {
		value := v.Get()
		if value != v.defaultValue {
			if v.hiddenValue {
				config[k] = true
			} else {
				config[k] = value
			}
		}
	}

	return config
}

func daemonConfigSetPassword(d *Daemon, key string, value string) (string, error) {
	// Nothing to do on unset
	if value == "" {
		return value, nil
	}

	// Hash the password
	buf := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", err
	}

	hash, err := scrypt.Key([]byte(value), buf, 1<<14, 8, 1, 64)
	if err != nil {
		return "", err
	}

	buf = append(buf, hash...)
	value = hex.EncodeToString(buf)

	return value, nil
}

func daemonConfigSetAddress(d *Daemon, key string, value string) (string, error) {
	// Update the current https address
	err := d.UpdateHTTPsPort(value)
	if err != nil {
		return "", err
	}

	return value, nil
}

func daemonConfigSetProxy(d *Daemon, key string, value string) (string, error) {
	// Get the current config
	config := map[string]string{}
	config["core.proxy_https"] = daemonConfig["core.proxy_https"].Get()
	config["core.proxy_http"] = daemonConfig["core.proxy_http"].Get()
	config["core.proxy_ignore_hosts"] = daemonConfig["core.proxy_ignore_hosts"].Get()

	// Apply the change
	config[key] = value

	// Update the cached proxy function
	d.proxy = shared.ProxyFromConfig(
		config["core.proxy_https"],
		config["core.proxy_http"],
		config["core.proxy_ignore_hosts"],
	)

	// Clear the simplestreams cache as it's tied to the old proxy config
	imageStreamCacheLock.Lock()
	for k := range imageStreamCache {
		delete(imageStreamCache, k)
	}
	imageStreamCacheLock.Unlock()

	return value, nil
}

func daemonConfigTriggerExpiry(d *Daemon, key string, value string) {
	// Trigger an image pruning run
	d.pruneChan <- true
}

func daemonConfigValidateCompression(d *Daemon, key string, value string) error {
	if value == "none" {
		return nil
	}

	_, err := exec.LookPath(value)
	return err
}

func storageDeprecatedKeys(d *Daemon, key string, value string) error {
	if value == "" {
		return nil
	}

	return fmt.Errorf("Setting the key \"%s\" is deprecated in favor of storage pool configuration.", key)
}
<commit_msg>Allow unsetting deprecated keys with default<commit_after>package main

import (
	"crypto/rand"
	"database/sql"
	"encoding/hex"
	"fmt"
	"io"
	"os/exec"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/crypto/scrypt"
	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/shared"
)

var daemonConfigLock sync.Mutex
var daemonConfig map[string]*daemonConfigKey

type daemonConfigKey struct {
	valueType    string
	defaultValue string
	validValues  []string
	currentValue string
	hiddenValue  bool

	validator func(d *Daemon, key string, value string) error
	setter    func(d *Daemon, key string, value string) (string, error)
	trigger   func(d *Daemon, key string, value string)
}

func (k *daemonConfigKey) name() string {
	name := ""

	// Look for a matching entry in daemonConfig
	daemonConfigLock.Lock()
	for key, value := range daemonConfig {
		if value == k {
			name = key
			break
		}
	}
	daemonConfigLock.Unlock()

	return name
}

func (k *daemonConfigKey) Validate(d *Daemon, value string) error {
	// Handle unsetting
	if value == "" {
		value = k.defaultValue

		if k.validator != nil {
			err := k.validator(d, k.name(), value)
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Validate booleans
	if k.valueType == "bool" && !shared.StringInSlice(strings.ToLower(value), []string{"true", "false", "1", "0", "yes", "no", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	// Validate integers
	if k.valueType == "int" {
		_, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			return err
		}
	}

	// Check against valid values
	if k.validValues != nil && !shared.StringInSlice(value, k.validValues) {
		return fmt.Errorf("Invalid value, only the following values are allowed: %s", k.validValues)
	}

	// Run external validation function
	if k.validator != nil {
		err := k.validator(d, k.name(), value)
		if err != nil {
			return err
		}
	}

	return nil
}

func (k *daemonConfigKey) Set(d *Daemon, value string) error {
	var name string

	// Check if we are actually changing things
	oldValue := k.currentValue
	if oldValue == value {
		return nil
	}

	// Validate the new value
	err := k.Validate(d, value)
	if err != nil {
		return err
	}

	// Run external setting function
	if k.setter != nil {
		value, err = k.setter(d, k.name(), value)
		if err != nil {
			return err
		}
	}

	// Get the configuration key and make sure daemonConfig is sane
	name = k.name()
	if name == "" {
		return fmt.Errorf("Corrupted configuration cache")
	}

	// Actually apply the change
	daemonConfigLock.Lock()
	k.currentValue = value
	daemonConfigLock.Unlock()

	err = dbConfigValueSet(d.db, name, value)
	if err != nil {
		return err
	}

	return nil
}

func (k *daemonConfigKey) Get() string {
	value := k.currentValue

	// Get the default value if not set
	if value == "" {
		value = k.defaultValue
	}

	return value
}

func (k *daemonConfigKey) GetBool() bool {
	value := k.currentValue

	// Get the default value if not set
	if value == "" {
		value = k.defaultValue
	}

	// Convert to boolean
	return shared.IsTrue(value)
}

func (k *daemonConfigKey) GetInt64() int64 {
	value := k.currentValue

	// Get the default value if not set
	if value == "" {
		value = k.defaultValue
	}

	// Convert to int64
	ret, _ := strconv.ParseInt(value, 10, 64)
	return ret
}

func daemonConfigInit(db *sql.DB) error {
	// Set all the keys
	daemonConfig = map[string]*daemonConfigKey{
		"core.https_address":             {valueType: "string", setter: daemonConfigSetAddress},
		"core.https_allowed_headers":     {valueType: "string"},
		"core.https_allowed_methods":     {valueType: "string"},
		"core.https_allowed_origin":      {valueType: "string"},
		"core.https_allowed_credentials": {valueType: "bool"},
		"core.proxy_http":                {valueType: "string", setter: daemonConfigSetProxy},
		"core.proxy_https":               {valueType: "string", setter: daemonConfigSetProxy},
		"core.proxy_ignore_hosts":        {valueType: "string", setter: daemonConfigSetProxy},
		"core.trust_password":            {valueType: "string", hiddenValue: true, setter: daemonConfigSetPassword},

		"images.auto_update_cached":    {valueType: "bool", defaultValue: "true"},
		"images.auto_update_interval":  {valueType: "int", defaultValue: "6"},
		"images.compression_algorithm": {valueType: "string", validator: daemonConfigValidateCompression, defaultValue: "gzip"},
		"images.remote_cache_expiry":   {valueType: "int", defaultValue: "10", trigger: daemonConfigTriggerExpiry},

		// Keys deprecated since the implementation of the storage api.
		"storage.lvm_fstype":           {valueType: "string", defaultValue: "ext4", validValues: []string{"ext4", "xfs"}, validator: storageDeprecatedKeys},
		"storage.lvm_mount_options":    {valueType: "string", defaultValue: "discard", validator: storageDeprecatedKeys},
		"storage.lvm_thinpool_name":    {valueType: "string", defaultValue: "LXDPool", validator: storageDeprecatedKeys},
		"storage.lvm_vg_name":          {valueType: "string", validator: storageDeprecatedKeys},
		"storage.lvm_volume_size":      {valueType: "string", defaultValue: "10GiB", validator: storageDeprecatedKeys},
		"storage.zfs_pool_name":        {valueType: "string", validator: storageDeprecatedKeys},
		"storage.zfs_remove_snapshots": {valueType: "bool", validator: storageDeprecatedKeys},
		"storage.zfs_use_refquota":     {valueType: "bool", validator: storageDeprecatedKeys},
	}

	// Load the values from the DB
	dbValues, err := dbConfigValuesGet(db)
	if err != nil {
		return err
	}

	daemonConfigLock.Lock()
	for k, v := range dbValues {
		_, ok := daemonConfig[k]
		if !ok {
			shared.LogError("Found invalid configuration key in database", log.Ctx{"key": k})
		}

		daemonConfig[k].currentValue = v
	}
	daemonConfigLock.Unlock()

	return nil
}

func daemonConfigRender() map[string]interface{} {
	config := map[string]interface{}{}

	// Turn the config into a JSON-compatible map
	for k, v := range daemonConfig {
		value := v.Get()
		if value != v.defaultValue {
			if v.hiddenValue {
				config[k] = true
			} else {
				config[k] = value
			}
		}
	}

	return config
}

func daemonConfigSetPassword(d *Daemon, key string, value string) (string, error) {
	// Nothing to do on unset
	if value == "" {
		return value, nil
	}

	// Hash the password
	buf := make([]byte, 32)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", err
	}

	hash, err := scrypt.Key([]byte(value), buf, 1<<14, 8, 1, 64)
	if err != nil {
		return "", err
	}

	buf = append(buf, hash...)
	value = hex.EncodeToString(buf)

	return value, nil
}

func daemonConfigSetAddress(d *Daemon, key string, value string) (string, error) {
	// Update the current https address
	err := d.UpdateHTTPsPort(value)
	if err != nil {
		return "", err
	}

	return value, nil
}

func daemonConfigSetProxy(d *Daemon, key string, value string) (string, error) {
	// Get the current config
	config := map[string]string{}
	config["core.proxy_https"] = daemonConfig["core.proxy_https"].Get()
	config["core.proxy_http"] = daemonConfig["core.proxy_http"].Get()
	config["core.proxy_ignore_hosts"] = daemonConfig["core.proxy_ignore_hosts"].Get()

	// Apply the change
	config[key] = value

	// Update the cached proxy function
	d.proxy = shared.ProxyFromConfig(
		config["core.proxy_https"],
		config["core.proxy_http"],
		config["core.proxy_ignore_hosts"],
	)

	// Clear the simplestreams cache as it's tied to the old proxy config
	imageStreamCacheLock.Lock()
	for k := range imageStreamCache {
		delete(imageStreamCache, k)
	}
	imageStreamCacheLock.Unlock()

	return value, nil
}

func daemonConfigTriggerExpiry(d *Daemon, key string, value string) {
	// Trigger an image pruning run
	d.pruneChan <- true
}

func daemonConfigValidateCompression(d *Daemon, key string, value string) error {
	if value == "none" {
		return nil
	}

	_, err := exec.LookPath(value)
	return err
}

func storageDeprecatedKeys(d *Daemon, key string, value string) error {
	if value == "" || daemonConfig[key].defaultValue == value {
		return nil
	}

	return fmt.Errorf("Setting the key \"%s\" is deprecated in favor of storage pool configuration.", key)
}
<|endoftext|>


<commit_before>package events

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/gorilla/websocket"
	"github.com/lxc/lxd/shared/logger"
)

// serverCommon represents an instance of a comon event server.
type serverCommon struct {
	debug   bool
	verbose bool
	lock    sync.Mutex
}

// listenerCommon describes a common event listener.
type listenerCommon struct {
	*websocket.Conn

	messageTypes []string
	ctx          context.Context
	ctxCancel    func()
	id           string
	lock         sync.Mutex
	location     string
	pongsPending uint

	// If true, this listener won't get events forwarded from other
	// nodes. It only used by listeners created internally by LXD nodes
	// connecting to other LXD nodes to get their local events only.
	localOnly bool
}

func (e *listenerCommon) heartbeat() {
	logger.Debug("Event listener server handler started", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})

	defer e.Close()

	pingInterval := time.Second * 5
	e.pongsPending = 0

	e.SetPongHandler(func(msg string) error {
		e.lock.Lock()
		e.pongsPending = 0
		e.lock.Unlock()
		return nil
	})

	// Run a blocking reader to detect if the remote side is closed.
	// We don't expect to get anything from the remote side, so this should remain blocked until disconnected.
	go func() {
		e.Conn.NextReader()
		e.Close()
	}()

	for {
		if e.IsClosed() {
			return
		}

		e.lock.Lock()
		if e.pongsPending > 2 {
			e.lock.Unlock()
			logger.Warn("Hearbeat for event listener handler timed out", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})
			return
		}
		err := e.WriteControl(websocket.PingMessage, []byte("keepalive"), time.Now().Add(5*time.Second))
		if err != nil {
			e.lock.Unlock()
			return
		}

		e.pongsPending++
		e.lock.Unlock()

		select {
		case <-time.After(pingInterval):
		case <-e.ctx.Done():
			return
		}
	}
}

// IsClosed returns true if the listener is closed.
func (e *listenerCommon) IsClosed() bool {
	return e.ctx.Err() != nil
}

// ID returns the listener ID.
func (e *listenerCommon) ID() string {
	return e.id
}

// Wait waits for a message on its active channel or the context is cancelled, then returns.
func (e *listenerCommon) Wait(ctx context.Context) {
	select {
	case <-ctx.Done():
	case <-e.ctx.Done():
	}
}

// Close Disconnects the listener.
func (e *listenerCommon) Close() {
	e.lock.Lock()
	defer e.lock.Unlock()

	if e.IsClosed() {
		return
	}

	logger.Debug("Event listener server handler stopped", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})

	e.Conn.Close()
	e.ctxCancel()
}

// WriteJSON message to the connection.
func (e *listenerCommon) WriteJSON(v interface{}) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteJSON(v)
}

// WriteMessage to the connection.
func (e *listenerCommon) WriteMessage(messageType int, data []byte) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteMessage(messageType, data)
}
<commit_msg>lxd/events/common: listenerCommon<commit_after>package events

import (
	"context"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/gorilla/websocket"
	"github.com/lxc/lxd/shared/logger"
)

// serverCommon represents an instance of a comon event server.
type serverCommon struct {
	debug   bool
	verbose bool
	lock    sync.Mutex
}

// listenerCommon describes a common event listener.
type listenerCommon struct {
	*websocket.Conn

	messageTypes []string
	ctx          context.Context
	ctxCancel    func()
	id           string
	lock         sync.Mutex
	pongsPending uint

	// If true, this listener won't get events forwarded from other
	// nodes. It only used by listeners created internally by LXD nodes
	// connecting to other LXD nodes to get their local events only.
	localOnly bool
}

func (e *listenerCommon) heartbeat() {
	logger.Debug("Event listener server handler started", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})

	defer e.Close()

	pingInterval := time.Second * 5
	e.pongsPending = 0

	e.SetPongHandler(func(msg string) error {
		e.lock.Lock()
		e.pongsPending = 0
		e.lock.Unlock()
		return nil
	})

	// Run a blocking reader to detect if the remote side is closed.
	// We don't expect to get anything from the remote side, so this should remain blocked until disconnected.
	go func() {
		e.Conn.NextReader()
		e.Close()
	}()

	for {
		if e.IsClosed() {
			return
		}

		e.lock.Lock()
		if e.pongsPending > 2 {
			e.lock.Unlock()
			logger.Warn("Hearbeat for event listener handler timed out", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})
			return
		}
		err := e.WriteControl(websocket.PingMessage, []byte("keepalive"), time.Now().Add(5*time.Second))
		if err != nil {
			e.lock.Unlock()
			return
		}

		e.pongsPending++
		e.lock.Unlock()

		select {
		case <-time.After(pingInterval):
		case <-e.ctx.Done():
			return
		}
	}
}

// IsClosed returns true if the listener is closed.
func (e *listenerCommon) IsClosed() bool {
	return e.ctx.Err() != nil
}

// ID returns the listener ID.
func (e *listenerCommon) ID() string {
	return e.id
}

// Wait waits for a message on its active channel or the context is cancelled, then returns.
func (e *listenerCommon) Wait(ctx context.Context) {
	select {
	case <-ctx.Done():
	case <-e.ctx.Done():
	}
}

// Close Disconnects the listener.
func (e *listenerCommon) Close() {
	e.lock.Lock()
	defer e.lock.Unlock()

	if e.IsClosed() {
		return
	}

	logger.Debug("Event listener server handler stopped", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})

	e.Conn.Close()
	e.ctxCancel()
}

// WriteJSON message to the connection.
func (e *listenerCommon) WriteJSON(v interface{}) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteJSON(v)
}

// WriteMessage to the connection.
func (e *listenerCommon) WriteMessage(messageType int, data []byte) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteMessage(messageType, data)
}
<|endoftext|>