<commit_before>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 2*time.Hour + 18*time.Minute
	MaxScanSleep     = 6 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 5 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 25
)

var (
	MaxReliability     = types.NewCurrency64(50) // Given the scanning defaults, about 1 week of survival.
	DefaultReliability = types.NewCurrency64(20) // Given the scanning defaults, about 3 days of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// addHostToScanPool creates a gofunc that adds a host to the scan pool. If the
// scan pool is currently full, the blocking gofunc will not cause a deadlock.
// The gofunc is created inside of this function to eliminate the burden of
// needing to remember to call 'go addHostToScanPool'.
func (hdb *HostDB) scanHostEntry(entry *hostEntry) {
	go func() {
		hdb.scanPool <- entry
	}()
}

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queued host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, modules.RPCSettings)
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = calculateHostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
			}
		}
		hdb.mu.Unlock(id)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				hdb.scanHostEntry(host.hostEntry)
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				hdb.scanHostEntry(random[i])
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<commit_msg>set hostdb to scan network more frequently<commit_after>package hostdb

// scan.go contians the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"crypto/rand"
	"math/big"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	DefaultScanSleep = 1*time.Hour + 37*time.Minute
	MaxScanSleep     = 4 * time.Hour
	MinScanSleep     = 1 * time.Hour

	MaxActiveHosts              = 500
	InactiveHostCheckupQuantity = 250

	maxSettingsLen = 1024

	hostRequestTimeout = 5 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 25
)

var (
	MaxReliability     = types.NewCurrency64(225) // Given the scanning defaults, about 3 weeks of survival.
	DefaultReliability = types.NewCurrency64(75)  // Given the scanning defaults, about 1 week of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// addHostToScanPool creates a gofunc that adds a host to the scan pool. If the
// scan pool is currently full, the blocking gofunc will not cause a deadlock.
// The gofunc is created inside of this function to eliminate the burden of
// needing to remember to call 'go addHostToScanPool'.
func (hdb *HostDB) scanHostEntry(entry *hostEntry) {
	go func() {
		hdb.scanPool <- entry
	}()
}

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		return
	}
	entry.reliability = entry.reliability.Sub(penalty)

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		delete(hdb.activeHosts, entry.IPAddress)
		node.removeNode()
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHost tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	for hostEntry := range hdb.scanPool {
		// Request settings from the queued host entry.
		var settings modules.HostSettings
		err := func() error {
			conn, err := net.DialTimeout("tcp", string(hostEntry.IPAddress), hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, modules.RPCSettings)
			if err != nil {
				return err
			}
			return encoding.ReadObject(conn, &settings, maxSettingsLen)
		}()

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		id := hdb.mu.Lock()
		{
			if err != nil {
				hdb.decrementReliability(hostEntry.IPAddress, UnreachablePenalty)
				hdb.mu.Unlock(id)
				continue
			}

			// Update the host settings, reliability, and weight. The old IPAddress
			// must be preserved.
			settings.IPAddress = hostEntry.HostSettings.IPAddress
			hostEntry.HostSettings = settings
			hostEntry.reliability = MaxReliability
			hostEntry.weight = calculateHostWeight(*hostEntry)

			// If the host is not already in the database and 'MaxActiveHosts' has not
			// been reached, add the host to the database.
			_, exists1 := hdb.activeHosts[hostEntry.IPAddress]
			_, exists2 := hdb.allHosts[hostEntry.IPAddress]
			if !exists1 && exists2 && len(hdb.activeHosts) < MaxActiveHosts {
				hdb.insertNode(hostEntry)
			}
		}
		hdb.mu.Unlock(id)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	for {
		// Log the current number of hosts.
		hdb.log.Printf("INFO: Starting a host scan. Active Hosts: %v. All hosts: %v.", len(hdb.activeHosts), len(hdb.allHosts))

		// Determine who to scan. At most 'MaxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		id := hdb.mu.Lock()
		{
			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				hdb.scanHostEntry(host.hostEntry)
			}

			// Assemble all of the inactive hosts into a single array.
			var random []*hostEntry
			for _, entry := range hdb.allHosts {
				entry2, exists := hdb.activeHosts[entry.IPAddress]
				if !exists {
					random = append(random, entry)
				} else {
					if build.DEBUG {
						if entry2.hostEntry != entry {
							panic("allHosts + activeHosts mismatch!")
						}
					}
				}
			}

			// Randomize the slice by swapping each element with an element
			// that hasn't been visited yet.
			for i := 0; i < len(random); i++ {
				N, err := rand.Int(rand.Reader, big.NewInt(int64(len(random)-i)))
				if err != nil {
					if build.DEBUG {
						panic(err)
					}
				} else {
					break
				}

				n := int(N.Int64()) + i
				tmp := random[i]
				random[i] = random[n]
				random[n] = tmp
			}

			// Select the first InactiveHostCheckupQuantity hosts from the
			// shuffled list and scan them.
			n := InactiveHostCheckupQuantity
			if len(random) < InactiveHostCheckupQuantity {
				n = len(random)
			}
			for i := 0; i < n; i++ {
				hdb.scanHostEntry(random[i])
			}
		}
		hdb.mu.Unlock(id)

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(MaxScanSleep))
		minBig := big.NewInt(int64(MinScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			if build.DEBUG {
				panic(err)
			} else {
				// If there's an error, sleep for the default amount of time.
				defaultBig := big.NewInt(int64(DefaultScanSleep))
				randSleep = defaultBig.Sub(defaultBig, minBig)
			}
		}
		time.Sleep(time.Duration(randSleep.Int64()) + MinScanSleep) // this means the MaxScanSleep is actual Max+Min.
	}
}
<|endoftext|>


<commit_before>package wallet

import (
	"crypto/rand"
	"errors"
	"runtime"
	"sync"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
	"github.com/NebulousLabs/bolt"
)

var (
	errKnownSeed = errors.New("seed is already known")
)

type (
	// uniqueID is a unique id randomly generated and put at the front of every
	// persistence object. It is used to make sure that a different encryption
	// key can be used for every persistence object.
	uniqueID [crypto.EntropySize]byte

	// seedFile stores an encrypted wallet seed on disk.
	seedFile struct {
		UID                    uniqueID
		EncryptionVerification crypto.Ciphertext
		Seed                   crypto.Ciphertext
	}
)

// generateSpendableKey creates the keys and unlock conditions for seed at a
// given index.
func generateSpendableKey(seed modules.Seed, index uint64) spendableKey {
	sk, pk := crypto.GenerateKeyPairDeterministic(crypto.HashAll(seed, index))
	return spendableKey{
		UnlockConditions: types.UnlockConditions{
			PublicKeys:         []types.SiaPublicKey{types.Ed25519PublicKey(pk)},
			SignaturesRequired: 1,
		},
		SecretKeys: []crypto.SecretKey{sk},
	}
}

// generateKeys generates n keys from seed, starting from index start.
func generateKeys(seed modules.Seed, start, n uint64) []spendableKey {
	// generate in parallel, one goroutine per core.
	keys := make([]spendableKey, n)
	var wg sync.WaitGroup
	wg.Add(runtime.NumCPU())
	for cpu := 0; cpu < runtime.NumCPU(); cpu++ {
		go func(offset uint64) {
			defer wg.Done()
			for i := offset; i < n; i += uint64(runtime.NumCPU()) {
				// NOTE: don't bother trying to optimize generateSpendableKey;
				// profiling shows that ed25519 key generation consumes far
				// more CPU time than encoding or hashing.
				keys[i] = generateSpendableKey(seed, start+i)
			}
		}(uint64(cpu))
	}
	wg.Wait()
	return keys
}

// createSeedFile creates and encrypts a seedFile.
func createSeedFile(masterKey crypto.TwofishKey, seed modules.Seed) (seedFile, error) {
	var sf seedFile
	_, err := rand.Read(sf.UID[:])
	if err != nil {
		return seedFile{}, err
	}
	sek := uidEncryptionKey(masterKey, sf.UID)
	sf.EncryptionVerification, err = sek.EncryptBytes(verificationPlaintext)
	if err != nil {
		return seedFile{}, err
	}
	sf.Seed, err = sek.EncryptBytes(seed[:])
	if err != nil {
		return seedFile{}, err
	}
	return sf, nil
}

// decryptSeedFile decrypts a seed file using the encryption key.
func decryptSeedFile(masterKey crypto.TwofishKey, sf seedFile) (seed modules.Seed, err error) {
	// Verify that the provided master key is the correct key.
	decryptionKey := uidEncryptionKey(masterKey, sf.UID)
	err = verifyEncryption(decryptionKey, sf.EncryptionVerification)
	if err != nil {
		return modules.Seed{}, err
	}

	// Decrypt and return the seed.
	plainSeed, err := decryptionKey.DecryptBytes(sf.Seed)
	if err != nil {
		return modules.Seed{}, err
	}
	copy(seed[:], plainSeed)
	return seed, nil
}

// integrateSeed generates n spendableKeys from the seed and loads them into
// the wallet.
func (w *Wallet) integrateSeed(seed modules.Seed, n uint64) {
	for _, sk := range generateKeys(seed, 0, n) {
		w.keys[sk.UnlockConditions.UnlockHash()] = sk
	}
}

// loadSeed integrates a recovery seed into the wallet.
func (w *Wallet) loadSeed(masterKey crypto.TwofishKey, seed modules.Seed) error {
	// Because the recovery seed does not have a UID, duplication must be
	// prevented by comparing with the list of decrypted seeds. This can only
	// occur while the wallet is unlocked.
	if !w.unlocked {
		return modules.ErrLockedWallet
	}
	if seed == w.primarySeed {
		return errKnownSeed
	}
	for _, wSeed := range w.seeds {
		if seed == wSeed {
			return errKnownSeed
		}
	}

	err := w.db.Update(func(tx *bolt.Tx) error {
		err := checkMasterKey(tx, masterKey)
		if err != nil {
			return err
		}

		// create a seedFile for the seed
		sf, err := createSeedFile(masterKey, seed)
		if err != nil {
			return err
		}

		// add the seedFile
		return tx.Bucket(bucketSeedFiles).Put(sf.UID[:], encoding.Marshal(sf))
	})
	if err != nil {
		return err
	}

	// load the seed's keys
	w.integrateSeed(seed, modules.PublicKeysPerSeed)
	w.seeds = append(w.seeds, seed)
	return nil
}

// nextPrimarySeedAddress fetches the next address from the primary seed.
func (w *Wallet) nextPrimarySeedAddress(tx *bolt.Tx) (types.UnlockConditions, error) {
	// Check that the wallet has been unlocked.
	if !w.unlocked {
		return types.UnlockConditions{}, modules.ErrLockedWallet
	}

	// Fetch and increment the seed progress.
	progress, err := dbGetPrimarySeedProgress(tx)
	if err != nil {
		return types.UnlockConditions{}, err
	}
	if err = dbPutPrimarySeedProgress(tx, progress+1); err != nil {
		return types.UnlockConditions{}, err
	}
	// Integrate the next key into the wallet, and return the unlock
	// conditions.
	spendableKey := generateSpendableKey(w.primarySeed, progress)
	w.keys[spendableKey.UnlockConditions.UnlockHash()] = spendableKey
	return spendableKey.UnlockConditions, nil
}

// AllSeeds returns a list of all seeds known to and used by the wallet.
func (w *Wallet) AllSeeds() ([]modules.Seed, error) {
	w.mu.Lock()
	defer w.mu.Unlock()
	if !w.unlocked {
		return nil, modules.ErrLockedWallet
	}
	return append([]modules.Seed{w.primarySeed}, w.seeds...), nil
}

// PrimarySeed returns the decrypted primary seed of the wallet, as well as
// the number of addresses that the seed can be safely used to generate.
func (w *Wallet) PrimarySeed() (modules.Seed, uint64, error) {
	w.mu.Lock()
	defer w.mu.Unlock()
	if !w.unlocked {
		return modules.Seed{}, 0, modules.ErrLockedWallet
	}
	// TODO: going to the db is slow; consider caching progress. On the other
	// hand, PrimarySeed isn't a frequently called method, so caching may be
	// overkill.
	var progress uint64
	err := w.db.View(func(tx *bolt.Tx) error {
		var err error
		progress, err = dbGetPrimarySeedProgress(tx)
		return err
	})
	if err != nil {
		return modules.Seed{}, 0, err
	}

	// addresses remaining is maxScanKeys-progress; generating more keys than
	// that risks not being able to recover them when using SweepSeed or
	// InitFromSeed.
	remaining := maxScanKeys - progress
	if progress > maxScanKeys {
		remaining = 0
	}
	return w.primarySeed, remaining, nil
}

// NextAddress returns an unlock hash that is ready to receive siacoins or
// siafunds. The address is generated using the primary address seed.
func (w *Wallet) NextAddress() (types.UnlockConditions, error) {
	if err := w.tg.Add(); err != nil {
		return types.UnlockConditions{}, err
	}
	defer w.tg.Done()
	w.mu.Lock()
	defer w.mu.Unlock()

	// TODO: going to the db is slow; consider creating 100 addresses at a
	// time.
	var uc types.UnlockConditions
	err := w.db.Update(func(tx *bolt.Tx) error {
		var err error
		uc, err = w.nextPrimarySeedAddress(tx)
		return err
	})
	return uc, err
}

// LoadSeed will track all of the addresses generated by the input seed,
// reclaiming any funds that were lost due to a deleted file or lost encryption
// key. An error will be returned if the seed has already been integrated with
// the wallet.
func (w *Wallet) LoadSeed(masterKey crypto.TwofishKey, seed modules.Seed) error {
	if err := w.tg.Add(); err != nil {
		return err
	}
	defer w.tg.Done()
	w.mu.Lock()
	defer w.mu.Unlock()
	return w.loadSeed(masterKey, seed)
}

// SweepSeed scans the blockchain for outputs generated from seed and create a
// transaction that transfers them to the wallet. Note that this incurs a
// transaction fee. It returns the total value of the outputs, minus the fee.
// TODO: support SiafundOutputs too
func (w *Wallet) SweepSeed(seed modules.Seed) (payout types.Currency, err error) {
	if err = w.tg.Add(); err != nil {
		return
	}
	defer w.tg.Done()

	if !w.cs.Synced() {
		return types.Currency{}, errors.New("cannot sweep until blockchain is synced")
	}

	// get an address to spend into
	var uc types.UnlockConditions
	err = w.db.Update(func(tx *bolt.Tx) error {
		var err error
		uc, err = w.nextPrimarySeedAddress(tx)
		return err
	})
	if err != nil {
		return
	}

	// scan blockchain for outputs, filtering out 'dust' (outputs that cost
	// more in fees than they are worth)
	s := newSeedScanner(seed)
	_, maxFee := w.tpool.FeeEstimation()
	const outputSize = 350 // approx. size in bytes of an output and accompanying signature
	s.dustThreshold = maxFee.Mul64(outputSize)
	if err = s.scan(w.cs); err != nil {
		return
	}

	// construct a transaction that spends the outputs
	// TODO: this may result in transactions that are too large.
	var txn types.Transaction
	var swept types.Currency // total value of swept outputs
	for _, output := range s.siacoinOutputs {
		// construct an input for the output
		sk := generateSpendableKey(seed, output.seedIndex)
		txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{
			ParentID:         types.SiacoinOutputID(output.id),
			UnlockConditions: sk.UnlockConditions, // TODO: check timelock, etc.
		})
		// add a signature for the input
		swept = swept.Add(output.value)
	}

	// estimate the transaction size. NOTE: this equation doesn't account for
	// other fields in the transaction, but since we are multiplying by
	// maxFee, lowballing is ok
	estTxnSize := len(s.siacoinOutputs) * outputSize
	estFee := maxFee.Mul64(uint64(estTxnSize))
	if estFee.Cmp(swept) >= 0 {
		return types.Currency{}, errors.New("transaction fee exceeds value of swept outputs")
	}
	txn.MinerFees = append(txn.MinerFees, estFee)

	// add the recipient output, equal to the sum of the inputs minus the
	// transaction fee
	payout = swept.Sub(estFee)
	txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{
		Value:      payout,
		UnlockHash: uc.UnlockHash(),
	})

	// add signatures
	for _, output := range s.siacoinOutputs {
		sk := generateSpendableKey(seed, output.seedIndex)
		addSignatures(&txn, types.FullCoveredFields, sk.UnlockConditions, crypto.Hash(output.id), sk)
	}

	// submit the transaction
	txnSet := []types.Transaction{txn}
	err = w.tpool.AcceptTransactionSet(txnSet)
	return
}
<commit_msg>refactor SweepSeed to use txnBuilder<commit_after>package wallet

import (
	"crypto/rand"
	"errors"
	"runtime"
	"sync"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
	"github.com/NebulousLabs/bolt"
)

var (
	errKnownSeed = errors.New("seed is already known")
)

type (
	// uniqueID is a unique id randomly generated and put at the front of every
	// persistence object. It is used to make sure that a different encryption
	// key can be used for every persistence object.
	uniqueID [crypto.EntropySize]byte

	// seedFile stores an encrypted wallet seed on disk.
	seedFile struct {
		UID                    uniqueID
		EncryptionVerification crypto.Ciphertext
		Seed                   crypto.Ciphertext
	}
)

// generateSpendableKey creates the keys and unlock conditions for seed at a
// given index.
func generateSpendableKey(seed modules.Seed, index uint64) spendableKey {
	sk, pk := crypto.GenerateKeyPairDeterministic(crypto.HashAll(seed, index))
	return spendableKey{
		UnlockConditions: types.UnlockConditions{
			PublicKeys:         []types.SiaPublicKey{types.Ed25519PublicKey(pk)},
			SignaturesRequired: 1,
		},
		SecretKeys: []crypto.SecretKey{sk},
	}
}

// generateKeys generates n keys from seed, starting from index start.
func generateKeys(seed modules.Seed, start, n uint64) []spendableKey {
	// generate in parallel, one goroutine per core.
	keys := make([]spendableKey, n)
	var wg sync.WaitGroup
	wg.Add(runtime.NumCPU())
	for cpu := 0; cpu < runtime.NumCPU(); cpu++ {
		go func(offset uint64) {
			defer wg.Done()
			for i := offset; i < n; i += uint64(runtime.NumCPU()) {
				// NOTE: don't bother trying to optimize generateSpendableKey;
				// profiling shows that ed25519 key generation consumes far
				// more CPU time than encoding or hashing.
				keys[i] = generateSpendableKey(seed, start+i)
			}
		}(uint64(cpu))
	}
	wg.Wait()
	return keys
}

// createSeedFile creates and encrypts a seedFile.
func createSeedFile(masterKey crypto.TwofishKey, seed modules.Seed) (seedFile, error) {
	var sf seedFile
	_, err := rand.Read(sf.UID[:])
	if err != nil {
		return seedFile{}, err
	}
	sek := uidEncryptionKey(masterKey, sf.UID)
	sf.EncryptionVerification, err = sek.EncryptBytes(verificationPlaintext)
	if err != nil {
		return seedFile{}, err
	}
	sf.Seed, err = sek.EncryptBytes(seed[:])
	if err != nil {
		return seedFile{}, err
	}
	return sf, nil
}

// decryptSeedFile decrypts a seed file using the encryption key.
func decryptSeedFile(masterKey crypto.TwofishKey, sf seedFile) (seed modules.Seed, err error) {
	// Verify that the provided master key is the correct key.
	decryptionKey := uidEncryptionKey(masterKey, sf.UID)
	err = verifyEncryption(decryptionKey, sf.EncryptionVerification)
	if err != nil {
		return modules.Seed{}, err
	}

	// Decrypt and return the seed.
	plainSeed, err := decryptionKey.DecryptBytes(sf.Seed)
	if err != nil {
		return modules.Seed{}, err
	}
	copy(seed[:], plainSeed)
	return seed, nil
}

// integrateSeed generates n spendableKeys from the seed and loads them into
// the wallet.
func (w *Wallet) integrateSeed(seed modules.Seed, n uint64) {
	for _, sk := range generateKeys(seed, 0, n) {
		w.keys[sk.UnlockConditions.UnlockHash()] = sk
	}
}

// loadSeed integrates a recovery seed into the wallet.
func (w *Wallet) loadSeed(masterKey crypto.TwofishKey, seed modules.Seed) error {
	// Because the recovery seed does not have a UID, duplication must be
	// prevented by comparing with the list of decrypted seeds. This can only
	// occur while the wallet is unlocked.
	if !w.unlocked {
		return modules.ErrLockedWallet
	}
	if seed == w.primarySeed {
		return errKnownSeed
	}
	for _, wSeed := range w.seeds {
		if seed == wSeed {
			return errKnownSeed
		}
	}

	err := w.db.Update(func(tx *bolt.Tx) error {
		err := checkMasterKey(tx, masterKey)
		if err != nil {
			return err
		}

		// create a seedFile for the seed
		sf, err := createSeedFile(masterKey, seed)
		if err != nil {
			return err
		}

		// add the seedFile
		return tx.Bucket(bucketSeedFiles).Put(sf.UID[:], encoding.Marshal(sf))
	})
	if err != nil {
		return err
	}

	// load the seed's keys
	w.integrateSeed(seed, modules.PublicKeysPerSeed)
	w.seeds = append(w.seeds, seed)
	return nil
}

// nextPrimarySeedAddress fetches the next address from the primary seed.
func (w *Wallet) nextPrimarySeedAddress(tx *bolt.Tx) (types.UnlockConditions, error) {
	// Check that the wallet has been unlocked.
	if !w.unlocked {
		return types.UnlockConditions{}, modules.ErrLockedWallet
	}

	// Fetch and increment the seed progress.
	progress, err := dbGetPrimarySeedProgress(tx)
	if err != nil {
		return types.UnlockConditions{}, err
	}
	if err = dbPutPrimarySeedProgress(tx, progress+1); err != nil {
		return types.UnlockConditions{}, err
	}
	// Integrate the next key into the wallet, and return the unlock
	// conditions.
	spendableKey := generateSpendableKey(w.primarySeed, progress)
	w.keys[spendableKey.UnlockConditions.UnlockHash()] = spendableKey
	return spendableKey.UnlockConditions, nil
}

// AllSeeds returns a list of all seeds known to and used by the wallet.
func (w *Wallet) AllSeeds() ([]modules.Seed, error) {
	w.mu.Lock()
	defer w.mu.Unlock()
	if !w.unlocked {
		return nil, modules.ErrLockedWallet
	}
	return append([]modules.Seed{w.primarySeed}, w.seeds...), nil
}

// PrimarySeed returns the decrypted primary seed of the wallet, as well as
// the number of addresses that the seed can be safely used to generate.
func (w *Wallet) PrimarySeed() (modules.Seed, uint64, error) {
	w.mu.Lock()
	defer w.mu.Unlock()
	if !w.unlocked {
		return modules.Seed{}, 0, modules.ErrLockedWallet
	}
	// TODO: going to the db is slow; consider caching progress. On the other
	// hand, PrimarySeed isn't a frequently called method, so caching may be
	// overkill.
	var progress uint64
	err := w.db.View(func(tx *bolt.Tx) error {
		var err error
		progress, err = dbGetPrimarySeedProgress(tx)
		return err
	})
	if err != nil {
		return modules.Seed{}, 0, err
	}

	// addresses remaining is maxScanKeys-progress; generating more keys than
	// that risks not being able to recover them when using SweepSeed or
	// InitFromSeed.
	remaining := maxScanKeys - progress
	if progress > maxScanKeys {
		remaining = 0
	}
	return w.primarySeed, remaining, nil
}

// NextAddress returns an unlock hash that is ready to receive siacoins or
// siafunds. The address is generated using the primary address seed.
func (w *Wallet) NextAddress() (types.UnlockConditions, error) {
	if err := w.tg.Add(); err != nil {
		return types.UnlockConditions{}, err
	}
	defer w.tg.Done()
	w.mu.Lock()
	defer w.mu.Unlock()

	// TODO: going to the db is slow; consider creating 100 addresses at a
	// time.
	var uc types.UnlockConditions
	err := w.db.Update(func(tx *bolt.Tx) error {
		var err error
		uc, err = w.nextPrimarySeedAddress(tx)
		return err
	})
	return uc, err
}

// LoadSeed will track all of the addresses generated by the input seed,
// reclaiming any funds that were lost due to a deleted file or lost encryption
// key. An error will be returned if the seed has already been integrated with
// the wallet.
func (w *Wallet) LoadSeed(masterKey crypto.TwofishKey, seed modules.Seed) error {
	if err := w.tg.Add(); err != nil {
		return err
	}
	defer w.tg.Done()
	w.mu.Lock()
	defer w.mu.Unlock()
	return w.loadSeed(masterKey, seed)
}

// SweepSeed scans the blockchain for outputs generated from seed and create a
// transaction that transfers them to the wallet. Note that this incurs a
// transaction fee. It returns the total value of the outputs, minus the fee.
// TODO: support SiafundOutputs too
func (w *Wallet) SweepSeed(seed modules.Seed) (payout types.Currency, err error) {
	if err = w.tg.Add(); err != nil {
		return
	}
	defer w.tg.Done()

	if !w.cs.Synced() {
		return types.Currency{}, errors.New("cannot sweep until blockchain is synced")
	}

	// get an address to spend into
	var uc types.UnlockConditions
	err = w.db.Update(func(tx *bolt.Tx) error {
		var err error
		uc, err = w.nextPrimarySeedAddress(tx)
		return err
	})
	if err != nil {
		return
	}

	// scan blockchain for outputs, filtering out 'dust' (outputs that cost
	// more in fees than they are worth)
	s := newSeedScanner(seed)
	_, maxFee := w.tpool.FeeEstimation()
	const outputSize = 350 // approx. size in bytes of an output and accompanying signature
	s.dustThreshold = maxFee.Mul64(outputSize)
	if err = s.scan(w.cs); err != nil {
		return
	}

	// construct a transaction that spends the outputs
	// TODO: this may result in transactions that are too large.
	tb := w.StartTransaction()
	var swept types.Currency // total value of swept outputs
	for _, output := range s.siacoinOutputs {
		// construct an input for the output
		sk := generateSpendableKey(seed, output.seedIndex)
		tb.AddSiacoinInput(types.SiacoinInput{
			ParentID:         types.SiacoinOutputID(output.id),
			UnlockConditions: sk.UnlockConditions, // TODO: check timelock, etc.
		})
		// add a signature for the input
		swept = swept.Add(output.value)
	}

	// estimate the transaction size. NOTE: this equation doesn't account for
	// other fields in the transaction, but since we are multiplying by
	// maxFee, lowballing is ok
	estTxnSize := len(s.siacoinOutputs) * outputSize
	estFee := maxFee.Mul64(uint64(estTxnSize))
	if estFee.Cmp(swept) >= 0 {
		return types.Currency{}, errors.New("transaction fee exceeds value of swept outputs")
	}
	tb.AddMinerFee(estFee)

	// add the recipient output, equal to the sum of the inputs minus the
	// transaction fee
	payout = swept.Sub(estFee)
	tb.AddSiacoinOutput(types.SiacoinOutput{
		Value:      payout,
		UnlockHash: uc.UnlockHash(),
	})

	// add signatures (manually, since tb doesn't have access to the signing key)
	txn, parents := tb.View()
	for _, output := range s.siacoinOutputs {
		sk := generateSpendableKey(seed, output.seedIndex)
		addSignatures(&txn, types.FullCoveredFields, sk.UnlockConditions, crypto.Hash(output.id), sk)
	}

	// submit the transaction
	txnSet := append(parents, txn)
	err = w.tpool.AcceptTransactionSet(txnSet)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net/http"
	"testing"
	"time"
)

const requestTimeout = time.Second * 5

var (
	edgeHost    = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")
	originPort  = flag.Int("originPort", 8080, "Origin port to listen on for requests")
	insecureTLS = flag.Bool("insecureTLS", false, "Whether to check server certificates")

	client       *http.Transport
	originServer *CDNServeMux
)

// Setup clients and servers.
func init() {

	flag.Parse()

	tlsOptions := &tls.Config{}
	if *insecureTLS {
		tlsOptions.InsecureSkipVerify = true
	}

	client = &http.Transport{
		ResponseHeaderTimeout: requestTimeout,
		TLSClientConfig:       tlsOptions,
	}
	originServer = StartServer(*originPort)

	log.Println("Confirming that CDN has successfully probed Origin")
	err := confirmOriginIsEnabled(originServer, *edgeHost)
	if err != nil {
		log.Fatal(err)
	}
}

func TestHelpers(t *testing.T) {
	testHelpersCDNServeMuxHandlers(t, originServer)
	testHelpersCDNServeMuxProbes(t, originServer)
}

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("Request should not have made it to origin")
	})

	sourceUrl := fmt.Sprintf("http://%s/foo/bar", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/foo/bar", *edgeHost)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}
}

// Should send request to origin by default
func TestRequestsGoToOriginByDefault(t *testing.T) {
	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("EnsureOriginServed", uuid)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Errorf("Status code expected 200, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("EnsureOriginServed"); d != uuid {
		t.Errorf("EnsureOriginServed header has not come from Origin: expected %q, got %q", uuid, d)
	}

}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve a known static error page if cannot serve a page
// from origin, stale or any mirror.
// NB: ideally this should be a page that we control that has a mechanism
//     to alert us that it has been served.
func TestErrorPageIsServedWhenNoBackendAvailable(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}

// ---------------------------------------------------------
// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header itself rather than passing the Age header from origin.
func TestAgeHeaderIsSetByProviderNotOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestXCacheHeaderContainsHitMissFromBothProviderAndOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Served-By header giving information on the node and location served from.
func TestXServedByHeaderContainsANodeIdAndLocation(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the provider not origin
func TestXCacheHitsContainsProviderHitCountForThisObject(t *testing.T) {
	t.Error("Not implemented")
}
<commit_msg>Add complete test for cached responses<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"testing"
	"time"
)

const requestTimeout = time.Second * 5

var (
	edgeHost    = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")
	originPort  = flag.Int("originPort", 8080, "Origin port to listen on for requests")
	insecureTLS = flag.Bool("insecureTLS", false, "Whether to check server certificates")

	client       *http.Transport
	originServer *CDNServeMux
)

// Setup clients and servers.
func init() {

	flag.Parse()

	tlsOptions := &tls.Config{}
	if *insecureTLS {
		tlsOptions.InsecureSkipVerify = true
	}

	client = &http.Transport{
		ResponseHeaderTimeout: requestTimeout,
		TLSClientConfig:       tlsOptions,
	}
	originServer = StartServer(*originPort)

	log.Println("Confirming that CDN has successfully probed Origin")
	err := confirmOriginIsEnabled(originServer, *edgeHost)
	if err != nil {
		log.Fatal(err)
	}
}

func TestHelpers(t *testing.T) {
	testHelpersCDNServeMuxHandlers(t, originServer)
	testHelpersCDNServeMuxProbes(t, originServer)
}

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("Request should not have made it to origin")
	})

	sourceUrl := fmt.Sprintf("http://%s/foo/bar", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/foo/bar", *edgeHost)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}
}

// Should send request to origin by default
func TestRequestsGoToOriginByDefault(t *testing.T) {
	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("EnsureOriginServed", uuid)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Errorf("Status code expected 200, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("EnsureOriginServed"); d != uuid {
		t.Errorf("EnsureOriginServed header has not come from Origin: expected %q, got %q", uuid, d)
	}

}

// Should cache first response and return it on second request without
// hitting origin again.
func TestFirstResponseCached(t *testing.T) {
	const bodyExpected = "first request"
	const requestsExpectedCount = 1
	requestsReceivedCount := 0

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if requestsReceivedCount == 0 {
			w.Write([]byte(bodyExpected))
		} else {
			w.Write([]byte("subsequent request"))
		}

		requestsReceivedCount++
	})

	url := fmt.Sprintf("https://%s/%s", *edgeHost, NewUUID())
	req, _ := http.NewRequest("GET", url, nil)

	for i := 0; i < 2; i++ {
		resp, err := client.RoundTrip(req)
		if err != nil {
			t.Fatal(err)
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatal(err)
		}
		if string(body) != bodyExpected {
			t.Errorf("Incorrect response body. Expected %q, got %q", bodyExpected, body)
		}
	}

	if requestsReceivedCount > requestsExpectedCount {
		t.Errorf("originServer got too many requests. Expected %d requests, got %d", requestsExpectedCount, requestsReceivedCount)
	}
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve a known static error page if cannot serve a page
// from origin, stale or any mirror.
// NB: ideally this should be a page that we control that has a mechanism
//     to alert us that it has been served.
func TestErrorPageIsServedWhenNoBackendAvailable(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}

// ---------------------------------------------------------
// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header itself rather than passing the Age header from origin.
func TestAgeHeaderIsSetByProviderNotOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestXCacheHeaderContainsHitMissFromBothProviderAndOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Served-By header giving information on the node and location served from.
func TestXServedByHeaderContainsANodeIdAndLocation(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the provider not origin
func TestXCacheHitsContainsProviderHitCountForThisObject(t *testing.T) {
	t.Error("Not implemented")
}
<|endoftext|>


<commit_before>package radosAPI

import (
	"encoding/json"
	"errors"
	"net/url"
	"time"

	"github.com/QuentinPerez/go-encodeUrl"
)

// UsageConfig usage request
type UsageConfig struct {
	UID         string     `url:"uid,ifStringIsNotEmpty"`     // The user for which the information is requested. If not specified will apply to all users
	Start       *time.Time `url:"start,ifTimeIsNotNilCeph"`   // Date and (optional) time that specifies the start time of the requested data
	End         *time.Time `url:"end,ifTimeIsNotNilCeph"`     // Date and (optional) time that specifies the end time of the requested data (non-inclusive)
	ShowEntries bool       `url:"show-entries,ifBoolIsFalse"` // Specifies whether data entries should be returned.
	ShowSummary bool       `url:"show-summary,ifBoolIsFalse"` // Specifies whether data summary should be returned
	RemoveAll   bool       `url:"remove-all,ifBoolIsTrue"`    // Required when uid is not specified, in order to acknowledge multi user data removal.
}

// GetUsage requests bandwidth usage information.
//
// !! caps: usage=read !!
//
// @UID
// @Start
// @End
// @ShowEntries
// @ShowSummary
//
func (api *API) GetUsage(conf UsageConfig) (*Usage, error) {
	var (
		ret    = &Usage{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.get("/admin/usage", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// DeleteUsage removes usage information. With no dates specified, removes all usage information
//
// !! caps: usage=write !!
//
// @UID
// @Start
// @End
// @RemoveAll
//
func (api *API) DeleteUsage(conf UsageConfig) error {
	var (
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return errs[0]
	}
	values.Add("format", "json")
	_, _, err := api.delete("/admin/usage", values)
	if err != nil {
		return err
	}
	return nil
}

// GetUser gets user information. If no user is specified returns the list of all users along with suspension information
//
// !! caps: users=read !!
//
// @uid
//
func (api *API) GetUser(uid ...string) (*User, error) {
	ret := &User{}
	values := url.Values{}

	values.Add("format", "json")
	if len(uid) != 0 {
		values.Add("uid", uid[0])
	}
	body, _, err := api.get("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UserConfig user request
type UserConfig struct {
	UID         string `url:"uid,ifStringIsNotEmpty"`          // The user ID to be created
	DisplayName string `url:"display-name,ifStringIsNotEmpty"` // The display name of the user to be created
	Email       string `url:"email,ifStringIsNotEmpty"`        // The email address associated with the user
	KeyType     string `url:"key-type,ifStringIsNotEmpty"`     // Key type to be generated, options are: swift, s3 (default)
	AccessKey   string `url:"access-key,ifStringIsNotEmpty"`   // Specify access key
	SecretKey   string `url:"secret-key,ifStringIsNotEmpty"`   // Specify secret key
	UserCaps    string `url:"user-caps,ifStringIsNotEmpty"`    // User capabilities
	GenerateKey bool   `url:"generate-key,ifBoolIsTrue"`       // Generate a new key pair and add to the existing keyring
	MaxBuckets  *int   `url:"max-buckets,itoaIfNotNil"`        // Specify the maximum number of buckets the user can own
	Suspended   bool   `url:"suspended,ifBoolIsTrue"`          // Specify whether the user should be suspended
	PurgeData   bool   `url:"suspended,ifBoolIsTrue"`          // When specified the buckets and objects belonging to the user will also be removed
}

// CreateUser creates a new user. By Default, a S3 key pair will be created automatically and returned in the response.
// If only one of access-key or secret-key is provided, the omitted key will be automatically generated.
// By default, a generated key is added to the keyring without replacing an existing key pair.
// If access-key is specified and refers to an existing key owned by the user then it will be modified
//
// !! caps: users=write !!
//
// @UID
// @DisplayName
// @Email
// @KeyType
// @AccessKey
// @SecretKey
// @UserCaps
// @GenerateKey
// @MaxBuckets
// @Suspended
//
func (api *API) CreateUser(conf UserConfig) (*User, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}
	if conf.DisplayName == "" {
		return nil, errors.New("DisplayName field is required")
	}

	var (
		ret    = &User{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.put("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UpdateUser modifies a user
//
// !! caps: users=write !!
//
// @UID
// @DisplayName
// @Email
// @KeyType
// @AccessKey
// @SecretKey
// @UserCaps
// @GenerateKey
// @MaxBuckets
// @Suspended
//
func (api *API) UpdateUser(conf UserConfig) (*User, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}

	var (
		ret    = &User{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.post("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// RemoveUser removes an existing user.
//
// !! caps: users=write !!
//
// @UID
// @PurgeData
//
func (api *API) RemoveUser(conf UserConfig) error {
	if conf.UID == "" {
		return errors.New("UID field is required")
	}
	var (
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return errs[0]
	}
	values.Add("format", "json")
	_, _, err := api.delete("/admin/user", values)
	if err != nil {
		return err
	}
	return nil
}

// SubUserConfig subuser request
type SubUserConfig struct {
	UID            string `url:"uid,ifStringIsNotEmpty"`        // The user ID under which a subuser is to be created
	SubUser        string `url:"subuser,ifStringIsNotEmpty"`    // Specify the subuser ID to be created
	KeyType        string `url:"key-type,ifStringIsNotEmpty"`   // Key type to be generated, options are: swift (default), s3
	Access         string `url:"access,ifStringIsNotEmpty"`     // Set access permissions for sub-user, should be one of read, write, readwrite, full
	Secret         string `url:"secret,ifStringIsNotEmpty"`     // Specify secret key
	SecretKey      string `url:"secret-key,ifStringIsNotEmpty"` // Specify secret key
	GenerateSecret bool   `url:"generate-secret,ifBoolIsTrue"`  // Generate the secret key
	PurgeKeys      bool   `url:"purge-keys,ifBoolIsTrue"`       // Remove keys belonging to the subuser
}

// CreateSubUser creates a new subuser (primarily useful for clients using the Swift API).
// Note that either gen-subuser or subuser is required for a valid request.
// Note that in general for a subuser to be useful, it must be granted permissions by specifying access.
// As with user creation if subuser is specified without secret, then a secret key will be automatically generated.
//
// !! caps:	users=write !!
//
// @UID
// @SubUser
// @KeyType
// @Access
// @SecretKey
// @GenerateSecret
//
func (api *API) CreateSubUser(conf SubUserConfig) (*SubUsers, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}

	var (
		ret    = &SubUsers{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.put("/admin/user", values, "subuser")
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UpdateSubUser modifies an existing subuser
//
// !! caps:	users=write !!
//
// @UID
// @SubUser
// @KeyType
// @Access
// @Secret
// @GenerateSecret
//
func (api *API) UpdateSubUser(conf SubUserConfig) (*SubUsers, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}
	if conf.SubUser == "" {
		return nil, errors.New("SubUser field is required")
	}

	var (
		ret    = &SubUsers{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.post("/admin/user", values, "subuser")
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}
<commit_msg>Add RemoveSubUser<commit_after>package radosAPI

import (
	"encoding/json"
	"errors"
	"net/url"
	"time"

	"github.com/QuentinPerez/go-encodeUrl"
)

// UsageConfig usage request
type UsageConfig struct {
	UID         string     `url:"uid,ifStringIsNotEmpty"`     // The user for which the information is requested. If not specified will apply to all users
	Start       *time.Time `url:"start,ifTimeIsNotNilCeph"`   // Date and (optional) time that specifies the start time of the requested data
	End         *time.Time `url:"end,ifTimeIsNotNilCeph"`     // Date and (optional) time that specifies the end time of the requested data (non-inclusive)
	ShowEntries bool       `url:"show-entries,ifBoolIsFalse"` // Specifies whether data entries should be returned.
	ShowSummary bool       `url:"show-summary,ifBoolIsFalse"` // Specifies whether data summary should be returned
	RemoveAll   bool       `url:"remove-all,ifBoolIsTrue"`    // Required when uid is not specified, in order to acknowledge multi user data removal.
}

// GetUsage requests bandwidth usage information.
//
// !! caps: usage=read !!
//
// @UID
// @Start
// @End
// @ShowEntries
// @ShowSummary
//
func (api *API) GetUsage(conf UsageConfig) (*Usage, error) {
	var (
		ret    = &Usage{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.get("/admin/usage", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// DeleteUsage removes usage information. With no dates specified, removes all usage information
//
// !! caps: usage=write !!
//
// @UID
// @Start
// @End
// @RemoveAll
//
func (api *API) DeleteUsage(conf UsageConfig) error {
	var (
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return errs[0]
	}
	values.Add("format", "json")
	_, _, err := api.delete("/admin/usage", values)
	if err != nil {
		return err
	}
	return nil
}

// GetUser gets user information. If no user is specified returns the list of all users along with suspension information
//
// !! caps: users=read !!
//
// @uid
//
func (api *API) GetUser(uid ...string) (*User, error) {
	ret := &User{}
	values := url.Values{}

	values.Add("format", "json")
	if len(uid) != 0 {
		values.Add("uid", uid[0])
	}
	body, _, err := api.get("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UserConfig user request
type UserConfig struct {
	UID         string `url:"uid,ifStringIsNotEmpty"`          // The user ID to be created
	DisplayName string `url:"display-name,ifStringIsNotEmpty"` // The display name of the user to be created
	Email       string `url:"email,ifStringIsNotEmpty"`        // The email address associated with the user
	KeyType     string `url:"key-type,ifStringIsNotEmpty"`     // Key type to be generated, options are: swift, s3 (default)
	AccessKey   string `url:"access-key,ifStringIsNotEmpty"`   // Specify access key
	SecretKey   string `url:"secret-key,ifStringIsNotEmpty"`   // Specify secret key
	UserCaps    string `url:"user-caps,ifStringIsNotEmpty"`    // User capabilities
	GenerateKey bool   `url:"generate-key,ifBoolIsTrue"`       // Generate a new key pair and add to the existing keyring
	MaxBuckets  *int   `url:"max-buckets,itoaIfNotNil"`        // Specify the maximum number of buckets the user can own
	Suspended   bool   `url:"suspended,ifBoolIsTrue"`          // Specify whether the user should be suspended
	PurgeData   bool   `url:"suspended,ifBoolIsTrue"`          // When specified the buckets and objects belonging to the user will also be removed
}

// CreateUser creates a new user. By Default, a S3 key pair will be created automatically and returned in the response.
// If only one of access-key or secret-key is provided, the omitted key will be automatically generated.
// By default, a generated key is added to the keyring without replacing an existing key pair.
// If access-key is specified and refers to an existing key owned by the user then it will be modified
//
// !! caps: users=write !!
//
// @UID
// @DisplayName
// @Email
// @KeyType
// @AccessKey
// @SecretKey
// @UserCaps
// @GenerateKey
// @MaxBuckets
// @Suspended
//
func (api *API) CreateUser(conf UserConfig) (*User, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}
	if conf.DisplayName == "" {
		return nil, errors.New("DisplayName field is required")
	}

	var (
		ret    = &User{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.put("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UpdateUser modifies a user
//
// !! caps: users=write !!
//
// @UID
// @DisplayName
// @Email
// @KeyType
// @AccessKey
// @SecretKey
// @UserCaps
// @GenerateKey
// @MaxBuckets
// @Suspended
//
func (api *API) UpdateUser(conf UserConfig) (*User, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}

	var (
		ret    = &User{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.post("/admin/user", values)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// RemoveUser removes an existing user.
//
// !! caps: users=write !!
//
// @UID
// @PurgeData
//
func (api *API) RemoveUser(conf UserConfig) error {
	if conf.UID == "" {
		return errors.New("UID field is required")
	}
	var (
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return errs[0]
	}
	values.Add("format", "json")
	_, _, err := api.delete("/admin/user", values)
	if err != nil {
		return err
	}
	return nil
}

// SubUserConfig subuser request
type SubUserConfig struct {
	UID            string `url:"uid,ifStringIsNotEmpty"`        // The user ID under which a subuser is to be created
	SubUser        string `url:"subuser,ifStringIsNotEmpty"`    // Specify the subuser ID to be created
	KeyType        string `url:"key-type,ifStringIsNotEmpty"`   // Key type to be generated, options are: swift (default), s3
	Access         string `url:"access,ifStringIsNotEmpty"`     // Set access permissions for sub-user, should be one of read, write, readwrite, full
	Secret         string `url:"secret,ifStringIsNotEmpty"`     // Specify secret key
	SecretKey      string `url:"secret-key,ifStringIsNotEmpty"` // Specify secret key
	GenerateSecret bool   `url:"generate-secret,ifBoolIsTrue"`  // Generate the secret key
	PurgeKeys      bool   `url:"purge-keys,ifBoolIsTrue"`       // Remove keys belonging to the subuser
}

// CreateSubUser creates a new subuser (primarily useful for clients using the Swift API).
// Note that either gen-subuser or subuser is required for a valid request.
// Note that in general for a subuser to be useful, it must be granted permissions by specifying access.
// As with user creation if subuser is specified without secret, then a secret key will be automatically generated.
//
// !! caps:	users=write !!
//
// @UID
// @SubUser
// @KeyType
// @Access
// @SecretKey
// @GenerateSecret
//
func (api *API) CreateSubUser(conf SubUserConfig) (*SubUsers, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}

	var (
		ret    = &SubUsers{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.put("/admin/user", values, "subuser")
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// UpdateSubUser modifies an existing subuser
//
// !! caps:	users=write !!
//
// @UID
// @SubUser
// @KeyType
// @Access
// @Secret
// @GenerateSecret
//
func (api *API) UpdateSubUser(conf SubUserConfig) (*SubUsers, error) {
	if conf.UID == "" {
		return nil, errors.New("UID field is required")
	}
	if conf.SubUser == "" {
		return nil, errors.New("SubUser field is required")
	}

	var (
		ret    = &SubUsers{}
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return nil, errs[0]
	}
	values.Add("format", "json")
	body, _, err := api.post("/admin/user", values, "subuser")
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &ret); err != nil {
		return nil, err
	}
	return ret, nil
}

// RemoveSubUser remove an existing subuser
//
// !! caps:	users=write !!
//
// @UID
// @SubUser
// @PurgeKeys
//
func (api *API) RemoveSubUser(conf SubUserConfig) error {
	if conf.UID == "" {
		return errors.New("UID field is required")
	}
	if conf.SubUser == "" {
		return errors.New("SubUser field is required")
	}
	var (
		values = url.Values{}
		errs   []error
	)

	values, errs = encurl.Translate(conf)
	if len(errs) > 0 {
		return errs[0]
	}
	values.Add("format", "json")
	_, _, err := api.delete("/admin/user", values, "subuser")
	if err != nil {
		return err
	}
	return nil

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"testing"
	"net/http"
)

var edgeHostName = flag.String("edge", "www.gov.uk", "Hostname of edge")

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	sourceUrl := fmt.Sprintf("http://%s/", *edgeHostName)
	destUrl := fmt.Sprintf("https://%s/", *edgeHostName)

	client := &http.Transport{}
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Error(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %s", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}

	t.Error("Not implemented test to confirm that it doesn't hit origin")
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}
<commit_msg>Empty tests for stale and mirror failover.<commit_after>package main

import (
	"flag"
	"fmt"
	"testing"
	"net/http"
)

var edgeHostName = flag.String("edge", "www.gov.uk", "Hostname of edge")

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	sourceUrl := fmt.Sprintf("http://%s/", *edgeHostName)
	destUrl := fmt.Sprintf("https://%s/", *edgeHostName)

	client := &http.Transport{}
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Error(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %s", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}

	t.Error("Not implemented test to confirm that it doesn't hit origin")
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}
<|endoftext|>


<commit_before>package auth

import (
	"net/http"

	"github.com/jmoiron/sqlx"
	"github.com/sirupsen/logrus"
	"github.com/skygeario/skygear-server/pkg/auth/dependency"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/provider/anonymous"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/provider/password"
	coreAudit "github.com/skygeario/skygear-server/pkg/core/audit"
	coreAuth "github.com/skygeario/skygear-server/pkg/core/auth"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/logging"
	"github.com/skygeario/skygear-server/pkg/server/audit"
)

type DependencyMap struct{}

func NewDependencyMap() DependencyMap {
	return DependencyMap{}
}

func openDB(tConfig config.TenantConfiguration) func() (*sqlx.DB, error) {
	return func() (*sqlx.DB, error) {
		return sqlx.Open("postgres", tConfig.DBConnectionStr)
	}
}

// Provide provides dependency instance by name
// nolint: gocyclo
func (m DependencyMap) Provide(dependencyName string, r *http.Request) interface{} {
	switch dependencyName {
	case "AuthContextGetter":
		return coreAuth.NewContextGetterWithContext(r.Context())
	case "TxContext":
		tConfig := config.GetTenantConfig(r)
		return db.NewTxContextWithContext(r.Context(), openDB(tConfig))
	case "TokenStore":
		tConfig := config.GetTenantConfig(r)
		return coreAuth.NewDefaultTokenStore(r.Context(), tConfig)
	case "AuthInfoStore":
		tConfig := config.GetTenantConfig(r)
		return coreAuth.NewDefaultAuthInfoStore(r.Context(), tConfig)
	case "AuthDataChecker":
		return &dependency.DefaultAuthDataChecker{
			//TODO:
			// from tConfig
			AuthRecordKeys: [][]string{[]string{"email"}, []string{"username"}},
		}
	case "PasswordChecker":
		return &audit.PasswordChecker{
			// TODO:
			// from tConfig
			PwMinLength: 6,
		}
	case "PasswordAuthProvider":
		tConfig := config.GetTenantConfig(r)
		maskFormatter := logging.CreateMaskFormatter(sensitiveLoggerValues(r), &logrus.TextFormatter{})
		return password.NewProvider(
			db.NewSQLBuilder("auth", tConfig.AppName),
			db.NewSQLExecutor(r.Context(), db.NewContextWithContext(r.Context(), openDB(tConfig))),
			logging.CreateLogger(r, "provider_password", maskFormatter),
		)
	case "AnonymousAuthProvider":
		tConfig := config.GetTenantConfig(r)
		return anonymous.NewProvider(
			db.NewSQLBuilder("auth", tConfig.AppName),
			db.NewSQLExecutor(r.Context(), db.NewContextWithContext(r.Context(), openDB(tConfig))),
			logging.CreateLogger(r, "provider_anonymous"),
		)
	case "HandlerLogger":
		maskFormatter := logging.CreateMaskFormatter(sensitiveLoggerValues(r), &logrus.TextFormatter{})
		return logging.CreateLogger(r, "handler", maskFormatter)
	case "UserProfileStore":
		tConfig := config.GetTenantConfig(r)
		switch tConfig.UserProfile.ImplName {
		default:
			panic("unrecgonized user profile store implementation: " + tConfig.UserProfile.ImplName)
		case "":
			return nil
			// case "skygear":
			// 	return XXX
		}
	case "RoleStore":
		tConfig := config.GetTenantConfig(r)
		return coreAuth.NewDefaultRoleStore(r.Context(), tConfig)
	case "AuditTrail":
		tConfig := config.GetTenantConfig(r)
		trail, err := coreAudit.NewTrail(tConfig.UserAudit.Enabled, tConfig.UserAudit.TrailHandlerURL, r)
		if err != nil {
			panic(err)
		}
		return trail
	default:
		return nil
	}
}

func sensitiveLoggerValues(r *http.Request) []string {
	tConfig := config.GetTenantConfig(r)
	return tConfig.DefaultSensitiveLoggerValues()
}
<commit_msg>Create mask logger formatter for AnonymousAuthProvider<commit_after>package auth

import (
	"net/http"

	"github.com/jmoiron/sqlx"
	"github.com/sirupsen/logrus"
	"github.com/skygeario/skygear-server/pkg/auth/dependency"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/provider/anonymous"
	"github.com/skygeario/skygear-server/pkg/auth/dependency/provider/password"
	coreAudit "github.com/skygeario/skygear-server/pkg/core/audit"
	coreAuth "github.com/skygeario/skygear-server/pkg/core/auth"
	"github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/logging"
	"github.com/skygeario/skygear-server/pkg/server/audit"
)

type DependencyMap struct{}

func NewDependencyMap() DependencyMap {
	return DependencyMap{}
}

func openDB(tConfig config.TenantConfiguration) func() (*sqlx.DB, error) {
	return func() (*sqlx.DB, error) {
		return sqlx.Open("postgres", tConfig.DBConnectionStr)
	}
}

// Provide provides dependency instance by name
// nolint: gocyclo
func (m DependencyMap) Provide(dependencyName string, r *http.Request) interface{} {
	switch dependencyName {
	case "AuthContextGetter":
		return coreAuth.NewContextGetterWithContext(r.Context())
	case "TxContext":
		tConfig := config.GetTenantConfig(r)
		return db.NewTxContextWithContext(r.Context(), openDB(tConfig))
	case "TokenStore":
		tConfig := config.GetTenantConfig(r)
		return coreAuth.NewDefaultTokenStore(r.Context(), tConfig)
	case "AuthInfoStore":
		tConfig := config.GetTenantConfig(r)
		return coreAuth.NewDefaultAuthInfoStore(r.Context(), tConfig)
	case "AuthDataChecker":
		return &dependency.DefaultAuthDataChecker{
			//TODO:
			// from tConfig
			AuthRecordKeys: [][]string{[]string{"email"}, []string{"username"}},
		}
	case "PasswordChecker":
		return &audit.PasswordChecker{
			// TODO:
			// from tConfig
			PwMinLength: 6,
		}
	case "PasswordAuthProvider":
		tConfig := config.GetTenantConfig(r)
		return password.NewProvider(
			db.NewSQLBuilder("auth", tConfig.AppName),
			db.NewSQLExecutor(r.Context(), db.NewContextWithContext(r.Context(), openDB(tConfig))),
			logging.CreateLogger(r, "provider_password", createLoggerMaskFormatter(r)),
		)
	case "AnonymousAuthProvider":
		tConfig := config.GetTenantConfig(r)
		return anonymous.NewProvider(
			db.NewSQLBuilder("auth", tConfig.AppName),
			db.NewSQLExecutor(r.Context(), db.NewContextWithContext(r.Context(), openDB(tConfig))),
			logging.CreateLogger(r, "provider_anonymous", createLoggerMaskFormatter(r)),
		)
	case "HandlerLogger":
		return logging.CreateLogger(r, "handler", createLoggerMaskFormatter(r))
	case "UserProfileStore":
		tConfig := config.GetTenantConfig(r)
		switch tConfig.UserProfile.ImplName {
		default:
			panic("unrecgonized user profile store implementation: " + tConfig.UserProfile.ImplName)
		case "":
			return nil
			// case "skygear":
			// 	return XXX
		}
	case "RoleStore":
		tConfig := config.GetTenantConfig(r)
		return coreAuth.NewDefaultRoleStore(r.Context(), tConfig)
	case "AuditTrail":
		tConfig := config.GetTenantConfig(r)
		trail, err := coreAudit.NewTrail(tConfig.UserAudit.Enabled, tConfig.UserAudit.TrailHandlerURL, r)
		if err != nil {
			panic(err)
		}
		return trail
	default:
		return nil
	}
}

func createLoggerMaskFormatter(r *http.Request) logrus.Formatter {
	tConfig := config.GetTenantConfig(r)
	return logging.CreateMaskFormatter(tConfig.DefaultSensitiveLoggerValues(), &logrus.TextFormatter{})
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chart

import "strings"

// APIVersionV1 is the API version number for version 1.
const APIVersionV1 = "v1"

// APIVersionV2 is the API version number for version 2.
const APIVersionV2 = "v2"

// Chart is a helm package that contains metadata, a default config, zero or more
// optionally parameterizable templates, and zero or more charts (dependencies).
type Chart struct {
	// Raw contains the raw contents of the files originally contained in the chart archive.
	//
	// This should not be used except in special cases like `helm show values`,
	// where we want to display the raw values, comments and all.
	Raw []*File `json:"-"`
	// Metadata is the contents of the Chartfile.
	Metadata *Metadata `json:"metadata"`
	// LocK is the contents of Chart.lock.
	Lock *Lock `json:"lock"`
	// Templates for this chart.
	Templates []*File `json:"templates"`
	// Values are default config for this chart.
	Values map[string]interface{} `json:"values"`
	// Schema is an optional JSON schema for imposing structure on Values
	Schema []byte `json:"schema"`
	// Files are miscellaneous files in a chart archive,
	// e.g. README, LICENSE, etc.
	Files []*File `json:"files"`

	parent       *Chart
	dependencies []*Chart
}

// SetDependencies replaces the chart dependencies.
func (ch *Chart) SetDependencies(charts ...*Chart) {
	ch.dependencies = nil
	ch.AddDependency(charts...)
}

// Name returns the name of the chart.
func (ch *Chart) Name() string {
	if ch.Metadata == nil {
		return ""
	}
	return ch.Metadata.Name
}

// AddDependency determines if the chart is a subchart.
func (ch *Chart) AddDependency(charts ...*Chart) {
	for i, x := range charts {
		charts[i].parent = ch
		ch.dependencies = append(ch.dependencies, x)
	}
}

// Root finds the root chart.
func (ch *Chart) Root() *Chart {
	if ch.IsRoot() {
		return ch
	}
	return ch.Parent().Root()
}

// Dependencies are the charts that this chart depends on.
func (ch *Chart) Dependencies() []*Chart { return ch.dependencies }

// IsRoot determines if the chart is the root chart.
func (ch *Chart) IsRoot() bool { return ch.parent == nil }

// Parent returns a subchart's parent chart.
func (ch *Chart) Parent() *Chart { return ch.parent }

// ChartPath returns the full path to this chart in dot notation.
func (ch *Chart) ChartPath() string {
	if !ch.IsRoot() {
		return ch.Parent().ChartPath() + "." + ch.Name()
	}
	return ch.Name()
}

// ChartFullPath returns the full path to this chart.
func (ch *Chart) ChartFullPath() string {
	if !ch.IsRoot() {
		return ch.Parent().ChartFullPath() + "/charts/" + ch.Name()
	}
	return ch.Name()
}

// Validate validates the metadata.
func (ch *Chart) Validate() error {
	return ch.Metadata.Validate()
}

// AppVersion returns the appversion of the chart.
func (ch *Chart) AppVersion() string {
	if ch.Metadata == nil {
		return ""
	}
	return ch.Metadata.AppVersion
}

// CRDs returns a list of File objects in the 'crds/' directory of a Helm chart.
func (ch *Chart) CRDs() []*File {
	files := []*File{}
	// Find all resources in the crds/ directory
	for _, f := range ch.Files {
		if strings.HasPrefix(f.Name, "crds/") {
			files = append(files, f)
		}
	}
	// Get CRDs from dependencies, too.
	for _, dep := range ch.Dependencies() {
		files = append(files, dep.CRDs()...)
	}
	return files
}
<commit_msg>Fix typo<commit_after>/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chart

import "strings"

// APIVersionV1 is the API version number for version 1.
const APIVersionV1 = "v1"

// APIVersionV2 is the API version number for version 2.
const APIVersionV2 = "v2"

// Chart is a helm package that contains metadata, a default config, zero or more
// optionally parameterizable templates, and zero or more charts (dependencies).
type Chart struct {
	// Raw contains the raw contents of the files originally contained in the chart archive.
	//
	// This should not be used except in special cases like `helm show values`,
	// where we want to display the raw values, comments and all.
	Raw []*File `json:"-"`
	// Metadata is the contents of the Chartfile.
	Metadata *Metadata `json:"metadata"`
	// Lock is the contents of Chart.lock.
	Lock *Lock `json:"lock"`
	// Templates for this chart.
	Templates []*File `json:"templates"`
	// Values are default config for this chart.
	Values map[string]interface{} `json:"values"`
	// Schema is an optional JSON schema for imposing structure on Values
	Schema []byte `json:"schema"`
	// Files are miscellaneous files in a chart archive,
	// e.g. README, LICENSE, etc.
	Files []*File `json:"files"`

	parent       *Chart
	dependencies []*Chart
}

// SetDependencies replaces the chart dependencies.
func (ch *Chart) SetDependencies(charts ...*Chart) {
	ch.dependencies = nil
	ch.AddDependency(charts...)
}

// Name returns the name of the chart.
func (ch *Chart) Name() string {
	if ch.Metadata == nil {
		return ""
	}
	return ch.Metadata.Name
}

// AddDependency determines if the chart is a subchart.
func (ch *Chart) AddDependency(charts ...*Chart) {
	for i, x := range charts {
		charts[i].parent = ch
		ch.dependencies = append(ch.dependencies, x)
	}
}

// Root finds the root chart.
func (ch *Chart) Root() *Chart {
	if ch.IsRoot() {
		return ch
	}
	return ch.Parent().Root()
}

// Dependencies are the charts that this chart depends on.
func (ch *Chart) Dependencies() []*Chart { return ch.dependencies }

// IsRoot determines if the chart is the root chart.
func (ch *Chart) IsRoot() bool { return ch.parent == nil }

// Parent returns a subchart's parent chart.
func (ch *Chart) Parent() *Chart { return ch.parent }

// ChartPath returns the full path to this chart in dot notation.
func (ch *Chart) ChartPath() string {
	if !ch.IsRoot() {
		return ch.Parent().ChartPath() + "." + ch.Name()
	}
	return ch.Name()
}

// ChartFullPath returns the full path to this chart.
func (ch *Chart) ChartFullPath() string {
	if !ch.IsRoot() {
		return ch.Parent().ChartFullPath() + "/charts/" + ch.Name()
	}
	return ch.Name()
}

// Validate validates the metadata.
func (ch *Chart) Validate() error {
	return ch.Metadata.Validate()
}

// AppVersion returns the appversion of the chart.
func (ch *Chart) AppVersion() string {
	if ch.Metadata == nil {
		return ""
	}
	return ch.Metadata.AppVersion
}

// CRDs returns a list of File objects in the 'crds/' directory of a Helm chart.
func (ch *Chart) CRDs() []*File {
	files := []*File{}
	// Find all resources in the crds/ directory
	for _, f := range ch.Files {
		if strings.HasPrefix(f.Name, "crds/") {
			files = append(files, f)
		}
	}
	// Get CRDs from dependencies, too.
	for _, dep := range ch.Dependencies() {
		files = append(files, dep.CRDs()...)
	}
	return files
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	kubecmd "k8s.io/kubernetes/pkg/kubectl/cmd"

	"github.com/openshift/origin/pkg/cmd/cli/cmd"
	"github.com/openshift/origin/pkg/cmd/cli/policy"
	"github.com/openshift/origin/pkg/cmd/cli/secrets"
	"github.com/openshift/origin/pkg/cmd/flagtypes"
	"github.com/openshift/origin/pkg/cmd/templates"
	cmdutil "github.com/openshift/origin/pkg/cmd/util"
	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/openshift/origin/pkg/version"
)

const cliLong = `
Developer and Administrator Client

This client exposes commands for managing your applications, as well as lower level
tools to interact with each component of your system.

To create a new application, you can use the example app source. Login to your server and then
run new-app:

  $ %[1]s login
  $ %[1]s new-app openshift/ruby-20-centos7~https://github.com/openshift/ruby-hello-world.git

This will create an application based on the Docker image 'openshift/ruby-20-centos7' that builds
the source code at 'github.com/openshift/ruby-hello-world.git'. To start the build, run

  $ %[1]s start-build ruby-hello-world --follow

Once your application is deployed, use the status, get, and describe commands to see more about
the created components:

  $ %[1]s status
  $ %[1]s describe deploymentconfig ruby-hello-world
  $ %[1]s get pods

You'll be able to view the deployed application on the IP and port of the service that new-app
created for you.

You can easily switch between multiple projects using '%[1]s project <projectname>'.`

func NewCommandCLI(name, fullName string, out io.Writer) *cobra.Command {
	in := os.Stdin
	errout := os.Stderr

	// Main command
	cmds := &cobra.Command{
		Use:   name,
		Short: "Command line tools for managing applications",
		Long:  fmt.Sprintf(cliLong, fullName),
		Run:   cmdutil.DefaultSubCommandRun(out),
		BashCompletionFunction: bashCompletionFunc,
	}

	f := clientcmd.New(cmds.PersistentFlags())

	loginCmd := cmd.NewCmdLogin(fullName, f, in, out)
	groups := templates.CommandGroups{
		{
			Message: "Basic Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdTypes(fullName, f, out),
				loginCmd,
				cmd.NewCmdRequestProject("new-project", fullName+" new-project", fullName+" login", fullName+" project", f, out),
				cmd.NewCmdNewApplication(fullName, f, out),
				cmd.NewCmdStatus(cmd.StatusRecommendedName, fullName+" "+cmd.StatusRecommendedName, f, out),
				cmd.NewCmdProject(fullName+" project", f, out),
			},
		},
		{
			Message: "Build and Deploy Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdStartBuild(fullName, f, out),
				cmd.NewCmdBuildLogs(fullName, f, out),
				cmd.NewCmdDeploy(fullName, f, out),
				cmd.NewCmdRollback(fullName, f, out),
				cmd.NewCmdNewBuild(fullName, f, out),
				cmd.NewCmdCancelBuild(fullName, f, out),
				cmd.NewCmdImportImage(fullName, f, out),
				cmd.NewCmdScale(fullName, f, out),
				cmd.NewCmdTag(fullName, f, out),
			},
		},
		{
			Message: "Application Modification Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdGet(fullName, f, out),
				cmd.NewCmdDescribe(fullName, f, out),
				cmd.NewCmdEdit(fullName, f, out),
				cmd.NewCmdEnv(fullName, f, in, out),
				cmd.NewCmdVolume(fullName, f, out, errout),
				cmd.NewCmdLabel(fullName, f, out),
				cmd.NewCmdAnnotate(fullName, f, out),
				cmd.NewCmdExpose(fullName, f, out),
				cmd.NewCmdStop(fullName, f, out),
				cmd.NewCmdDelete(fullName, f, out),
			},
		},
		{
			Message: "Troubleshooting and Debugging Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogs(fullName, f, out),
				cmd.NewCmdRsh(fullName, f, in, out, errout),
				cmd.NewCmdExec(fullName, f, in, out, errout),
				cmd.NewCmdPortForward(fullName, f),
				cmd.NewCmdProxy(fullName, f, out),
			},
		},
		{
			Message: "Advanced Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdCreate(fullName, f, out),
				cmd.NewCmdReplace(fullName, f, out),
				cmd.NewCmdPatch(fullName, f, out),
				cmd.NewCmdProcess(fullName, f, out),
				cmd.NewCmdExport(fullName, f, in, out),
				cmd.NewCmdRun(fullName, f, in, out, errout),
				cmd.NewCmdAttach(fullName, f, in, out, errout),
				policy.NewCmdPolicy(policy.PolicyRecommendedName, fullName+" "+policy.PolicyRecommendedName, f, out),
				secrets.NewCmdSecrets(secrets.SecretsRecommendedName, fullName+" "+secrets.SecretsRecommendedName, f, out, fullName+" edit"),
			},
		},
		{
			Message: "Settings Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogout("logout", fullName+" logout", fullName+" login", f, in, out),
				cmd.NewCmdConfig(fullName, "config"),
				cmd.NewCmdWhoAmI(cmd.WhoAmIRecommendedCommandName, fullName+" "+cmd.WhoAmIRecommendedCommandName, f, out),
			},
		},
	}
	groups.Add(cmds)
	templates.ActsAsRootCommand(cmds, groups...).
		ExposeFlags(loginCmd, "certificate-authority", "insecure-skip-tls-verify")

	if name == fullName {
		cmds.AddCommand(version.NewVersionCommand(fullName))
	}
	cmds.AddCommand(cmd.NewCmdOptions(out))

	return cmds
}

// NewCmdKubectl provides exactly the functionality from Kubernetes,
// but with support for OpenShift resources
func NewCmdKubectl(name string, out io.Writer) *cobra.Command {
	flags := pflag.NewFlagSet("", pflag.ContinueOnError)
	f := clientcmd.New(flags)
	cmds := kubecmd.NewKubectlCommand(f.Factory, os.Stdin, out, os.Stderr)
	cmds.Aliases = []string{"kubectl"}
	cmds.Use = name
	cmds.Short = "Kubernetes cluster management via kubectl"
	cmds.Long = cmds.Long + `

This command exposes the exact semantics of the Kubernetes command line client with additional
support for application lifecycles.`
	flags.VisitAll(func(flag *pflag.Flag) {
		if f := cmds.PersistentFlags().Lookup(flag.Name); f == nil {
			cmds.PersistentFlags().AddFlag(flag)
		} else {
			glog.V(5).Infof("already registered flag %s", flag.Name)
		}
	})
	templates.ActsAsRootCommand(cmds)
	cmds.AddCommand(cmd.NewCmdOptions(out))
	return cmds
}

// CommandFor returns the appropriate command for this base name,
// or the OpenShift CLI command.
func CommandFor(basename string) *cobra.Command {
	var cmd *cobra.Command

	out := os.Stdout

	// Make case-insensitive and strip executable suffix if present
	if runtime.GOOS == "windows" {
		basename = strings.ToLower(basename)
		basename = strings.TrimSuffix(basename, ".exe")
	}

	switch basename {
	case "kubectl":
		cmd = NewCmdKubectl(basename, out)
	default:
		cmd = NewCommandCLI(basename, basename, out)
	}

	if cmd.UsageFunc() == nil {
		templates.ActsAsRootCommand(cmd)
	}
	flagtypes.GLog(cmd.PersistentFlags())

	return cmd
}

// applyToCreate injects the deprecation notice about for 'apply' command into
// 'create' command.
// TODO: Remove this once we get rid of 'apply' in all documentation/etc.
func applyToCreate(dst *cobra.Command) *cobra.Command {
	dst.Aliases = append(dst.Aliases, "apply")
	oldRun := dst.Run
	dst.Run = func(c *cobra.Command, args []string) {
		calledApply := false
		calledApply = calledApply || len(os.Args) >= 2 && os.Args[1] == "apply" // `oc apply`
		calledApply = calledApply || len(os.Args) >= 3 && os.Args[2] == "apply" // `openshift cli apply`
		if calledApply {
			glog.Errorf("DEPRECATED: The 'apply' command is now deprecated, use 'create' instead.")
		}
		oldRun(c, args)
	}
	return dst
}
<commit_msg>Fix docs of oc<commit_after>package cli

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	kubecmd "k8s.io/kubernetes/pkg/kubectl/cmd"

	"github.com/openshift/origin/pkg/cmd/cli/cmd"
	"github.com/openshift/origin/pkg/cmd/cli/policy"
	"github.com/openshift/origin/pkg/cmd/cli/secrets"
	"github.com/openshift/origin/pkg/cmd/flagtypes"
	"github.com/openshift/origin/pkg/cmd/templates"
	cmdutil "github.com/openshift/origin/pkg/cmd/util"
	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/openshift/origin/pkg/version"
)

const cliLong = `
Developer and Administrator Client

This client exposes commands for managing your applications, as well as lower level
tools to interact with each component of your system.

To create a new application, you can use the example app source. Login to your server and then
run new-app:

  $ %[1]s login
  $ %[1]s new-app openshift/ruby-20-centos7~https://github.com/openshift/ruby-hello-world.git

This will create an application based on the Docker image 'openshift/ruby-20-centos7' that builds
the source code at 'github.com/openshift/ruby-hello-world.git'. A build will start automatically and
a deployment will start as soon as the build finishes.

Once your application is deployed, use the status, get, and describe commands to see more about
the created components:

  $ %[1]s status
  $ %[1]s describe deploymentconfig ruby-hello-world
  $ %[1]s get pods

You'll be able to view the deployed application on the IP and port of the service that new-app
created for you.

You can easily switch between multiple projects using '%[1]s project <projectname>'.`

func NewCommandCLI(name, fullName string, out io.Writer) *cobra.Command {
	in := os.Stdin
	errout := os.Stderr

	// Main command
	cmds := &cobra.Command{
		Use:   name,
		Short: "Command line tools for managing applications",
		Long:  fmt.Sprintf(cliLong, fullName),
		Run:   cmdutil.DefaultSubCommandRun(out),
		BashCompletionFunction: bashCompletionFunc,
	}

	f := clientcmd.New(cmds.PersistentFlags())

	loginCmd := cmd.NewCmdLogin(fullName, f, in, out)
	groups := templates.CommandGroups{
		{
			Message: "Basic Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdTypes(fullName, f, out),
				loginCmd,
				cmd.NewCmdRequestProject("new-project", fullName+" new-project", fullName+" login", fullName+" project", f, out),
				cmd.NewCmdNewApplication(fullName, f, out),
				cmd.NewCmdStatus(cmd.StatusRecommendedName, fullName+" "+cmd.StatusRecommendedName, f, out),
				cmd.NewCmdProject(fullName+" project", f, out),
			},
		},
		{
			Message: "Build and Deploy Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdStartBuild(fullName, f, out),
				cmd.NewCmdBuildLogs(fullName, f, out),
				cmd.NewCmdDeploy(fullName, f, out),
				cmd.NewCmdRollback(fullName, f, out),
				cmd.NewCmdNewBuild(fullName, f, out),
				cmd.NewCmdCancelBuild(fullName, f, out),
				cmd.NewCmdImportImage(fullName, f, out),
				cmd.NewCmdScale(fullName, f, out),
				cmd.NewCmdTag(fullName, f, out),
			},
		},
		{
			Message: "Application Modification Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdGet(fullName, f, out),
				cmd.NewCmdDescribe(fullName, f, out),
				cmd.NewCmdEdit(fullName, f, out),
				cmd.NewCmdEnv(fullName, f, in, out),
				cmd.NewCmdVolume(fullName, f, out, errout),
				cmd.NewCmdLabel(fullName, f, out),
				cmd.NewCmdAnnotate(fullName, f, out),
				cmd.NewCmdExpose(fullName, f, out),
				cmd.NewCmdStop(fullName, f, out),
				cmd.NewCmdDelete(fullName, f, out),
			},
		},
		{
			Message: "Troubleshooting and Debugging Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogs(fullName, f, out),
				cmd.NewCmdRsh(fullName, f, in, out, errout),
				cmd.NewCmdExec(fullName, f, in, out, errout),
				cmd.NewCmdPortForward(fullName, f),
				cmd.NewCmdProxy(fullName, f, out),
			},
		},
		{
			Message: "Advanced Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdCreate(fullName, f, out),
				cmd.NewCmdReplace(fullName, f, out),
				cmd.NewCmdPatch(fullName, f, out),
				cmd.NewCmdProcess(fullName, f, out),
				cmd.NewCmdExport(fullName, f, in, out),
				cmd.NewCmdRun(fullName, f, in, out, errout),
				cmd.NewCmdAttach(fullName, f, in, out, errout),
				policy.NewCmdPolicy(policy.PolicyRecommendedName, fullName+" "+policy.PolicyRecommendedName, f, out),
				secrets.NewCmdSecrets(secrets.SecretsRecommendedName, fullName+" "+secrets.SecretsRecommendedName, f, out, fullName+" edit"),
			},
		},
		{
			Message: "Settings Commands:",
			Commands: []*cobra.Command{
				cmd.NewCmdLogout("logout", fullName+" logout", fullName+" login", f, in, out),
				cmd.NewCmdConfig(fullName, "config"),
				cmd.NewCmdWhoAmI(cmd.WhoAmIRecommendedCommandName, fullName+" "+cmd.WhoAmIRecommendedCommandName, f, out),
			},
		},
	}
	groups.Add(cmds)
	templates.ActsAsRootCommand(cmds, groups...).
		ExposeFlags(loginCmd, "certificate-authority", "insecure-skip-tls-verify")

	if name == fullName {
		cmds.AddCommand(version.NewVersionCommand(fullName))
	}
	cmds.AddCommand(cmd.NewCmdOptions(out))

	return cmds
}

// NewCmdKubectl provides exactly the functionality from Kubernetes,
// but with support for OpenShift resources
func NewCmdKubectl(name string, out io.Writer) *cobra.Command {
	flags := pflag.NewFlagSet("", pflag.ContinueOnError)
	f := clientcmd.New(flags)
	cmds := kubecmd.NewKubectlCommand(f.Factory, os.Stdin, out, os.Stderr)
	cmds.Aliases = []string{"kubectl"}
	cmds.Use = name
	cmds.Short = "Kubernetes cluster management via kubectl"
	cmds.Long = cmds.Long + `

This command exposes the exact semantics of the Kubernetes command line client with additional
support for application lifecycles.`
	flags.VisitAll(func(flag *pflag.Flag) {
		if f := cmds.PersistentFlags().Lookup(flag.Name); f == nil {
			cmds.PersistentFlags().AddFlag(flag)
		} else {
			glog.V(5).Infof("already registered flag %s", flag.Name)
		}
	})
	templates.ActsAsRootCommand(cmds)
	cmds.AddCommand(cmd.NewCmdOptions(out))
	return cmds
}

// CommandFor returns the appropriate command for this base name,
// or the OpenShift CLI command.
func CommandFor(basename string) *cobra.Command {
	var cmd *cobra.Command

	out := os.Stdout

	// Make case-insensitive and strip executable suffix if present
	if runtime.GOOS == "windows" {
		basename = strings.ToLower(basename)
		basename = strings.TrimSuffix(basename, ".exe")
	}

	switch basename {
	case "kubectl":
		cmd = NewCmdKubectl(basename, out)
	default:
		cmd = NewCommandCLI(basename, basename, out)
	}

	if cmd.UsageFunc() == nil {
		templates.ActsAsRootCommand(cmd)
	}
	flagtypes.GLog(cmd.PersistentFlags())

	return cmd
}

// applyToCreate injects the deprecation notice about for 'apply' command into
// 'create' command.
// TODO: Remove this once we get rid of 'apply' in all documentation/etc.
func applyToCreate(dst *cobra.Command) *cobra.Command {
	dst.Aliases = append(dst.Aliases, "apply")
	oldRun := dst.Run
	dst.Run = func(c *cobra.Command, args []string) {
		calledApply := false
		calledApply = calledApply || len(os.Args) >= 2 && os.Args[1] == "apply" // `oc apply`
		calledApply = calledApply || len(os.Args) >= 3 && os.Args[2] == "apply" // `openshift cli apply`
		if calledApply {
			glog.Errorf("DEPRECATED: The 'apply' command is now deprecated, use 'create' instead.")
		}
		oldRun(c, args)
	}
	return dst
}
<|endoftext|>