<commit_before>package dfp

import (
	"encoding/json"
	"net/http"

	sa "github.com/secureauthcorp/saidp-sdk-go"
)

/*
**********************************************************************
*   @author jhickman@secureauth.com
*
*  Copyright (c) 2017, SecureAuth
*  All rights reserved.
*
*    Redistribution and use in source and binary forms, with or without modification,
*    are permitted provided that the following conditions are met:
*
*    1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
*
*    2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
*    in the documentation and/or other materials provided with the distribution.
*
*    3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived
*    from this software without specific prior written permission.
*
*    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
*    THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
*    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
*    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
*    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
*    EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**********************************************************************
 */

const (
	jsEndpoint      = "/api/v1/dfp/js"
	valEndpoint     = "/api/v1/dfp/validate"
	confirmEndpoint = "/api/v1/dfp/confirm"
)

// Response :
//	Response struct that will be populated after the post request.
type Response struct {
	FingerprintID   string         `json:"fingerprint_id,omitempty"`
	FingerprintName string         `json:"fingerprint_name,omitempty"`
	Score           string         `json:"score,omitempty"`
	MatchScore      string         `json:"match_score,omitempty"`
	UpdateScore     string         `json:"update_score,omitempty"`
	Status          string         `json:"status,omitempty"`
	Message         string         `json:"message,omitempty"`
	UserID          string         `json:"user_id,omitempty"`
	Source          string         `json:"src,omitempty"`
	HTTPResponse    *http.Response `json:"-,omitempty"`
}

// Request :
//	Request struct to build the required post parameters.
// Fields:
//	[Required] UserID: the username that you wish to validate a fingerprint for.
//	HostAddress: the IP Address of the user.
//	FingerprintID: used to validate known fingerprint or confirm a fingerprint.
//	Fingerprint: fingerprint value struct required to validate a fingerprint.
type Request struct {
	UserID        string      `json:"user_id,omitempty"`
	HostAddress   string      `json:"host_address,omitempty"`
	FingerprintID string      `json:"fingerprint_id,omitempty"`
	Fingerprint   Fingerprint `json:"fingerprint,omitempty"`
}

// Fingerprint :
//	Details for the fingerprint makeup.
// Fields:
//	All fields should be populated from the Json string returned by the imported javascript.
type Fingerprint struct {
	Fonts          string `json:"fonts,omitempty"`
	Plugins        string `json:"plugins,omitempty"`
	Timezone       string `json:"timezone,omitempty"`
	Video          string `json:"video,omitempty"`
	LocalStorage   string `json:"local_storage,omitempty"`
	SessionStorage string `json:"session_storage,omitempty"`
	IeUserData     string `json:"ie_user_data,omitempty"`
	CookieEnabled  string `json:"cookie_enabled,omitempty"`
	UserAgent      string `json:"user_agent,omitempty"`
	Accept         string `json:"accept,omitempty"`
	AcceptCharset  string `json:"accept_charset,omitempty"`
	AcceptEncoding string `json:"accept_encoding,omitempty"`
	AcceptLang     string `json:"accept_language,omitempty"`
}

// Get :
//	Executes a get to the dfp/js endpoint.
// Parameters:
//	[Required] r: struct used to perform get request.
//	[Required] c: passing in the client containing authorization and host information.
//	endpoint: the endpoint for the get request.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) Get(c *sa.Client, endpoint string) (*Response, error) {
	httpRequest, err := c.BuildGetRequest(endpoint)
	if err != nil {
		return nil, err
	}
	httpResponse, err := c.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	dfpResponse := new(Response)
	if err := json.NewDecoder(httpResponse.Body).Decode(dfpResponse); err != nil {
		return nil, err
	}
	dfpResponse.HTTPResponse = httpResponse
	httpResponse.Body.Close()
	return dfpResponse, nil
}

// Post :
//	Executes a post to the dfp endpoint.
// Parameters:
// 	[Required] r: should have all required fields of the struct populated before using.
// 	[Required] c: passing in the client containing authorization and host information.
//	[Required] endpoint: the endpoint for the post request.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) Post(c *sa.Client, endpoint string) (*Response, error) {
	jsonRequest, err := json.Marshal(r)
	if err != nil {
		return nil, err
	}
	httpRequest, err := c.BuildPostRequest(endpoint, string(jsonRequest))
	if err != nil {
		return nil, err
	}
	httpResponse, err := c.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	dfpResponse := new(Response)
	if err := json.NewDecoder(httpResponse.Body).Decode(dfpResponse); err != nil {
		return nil, err
	}
	dfpResponse.HTTPResponse = httpResponse
	httpResponse.Body.Close()
	return dfpResponse, nil
}

// GetDfpJs :
//	Helper function for Get request to retrieve the fingerprint javascript source.
// Parameters:
//	[Required] c: passing in the client containing authorization and host information.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) GetDfpJs(c *sa.Client) (*Response, error) {
	dpfResponse, err := r.Get(c, jsEndpoint)
	if err != nil {
		return nil, err
	}
	return dpfResponse, nil
}

// ValidateDfp :
//	Helper function for posting to the dfp validate endpoint.
// Parameters:
//	[Required] c: passing in the client containing authorization and host information.
//	[Required] userID: the username of the user you wish to validate a dfp for.
//	[Required] hostAddress: the ip address of the user's device.
//	fingerprintId: if it is a known fingerprint, provide the fingerprint id to validate against.
//	[Required] fingerprint: the json string returned by the javascript dfp script.
//	[Required] accept: accept header of the users request.
//	[Required] acceptCharset: the accept_charset header of the users request.
//	[Required] acceptEncoding: the accept_encoding header of the users request.
//	[Required] acceptLanguage: the accept_language header of the users request.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) ValidateDfp(c *sa.Client, userID string, hostAddress string, fingerprintId string, fingerprint string, accept string, acceptCharset string, acceptEncoding string, acceptLanguage string) (*Response, error) {
	if err := json.Unmarshal([]byte(fingerprint), &r); err != nil {
		return nil, err
	}
	r.Fingerprint.Accept = accept
	r.Fingerprint.AcceptCharset = acceptCharset
	r.Fingerprint.AcceptEncoding = acceptEncoding
	r.Fingerprint.AcceptLang = acceptLanguage
	r.FingerprintID = fingerprintId
	r.UserID = userID
	r.HostAddress = hostAddress
	validateResponse, err := r.Post(c, valEndpoint)
	if err != nil {
		return nil, err
	}
	return validateResponse, nil
}

// Summary:
//	Helper function for posting to the dfp validate endpoint.
// Parameters:
//	[Required] c: passing in the client containing authorization and host information.
//	[Required] userID: the username of the user you wish to confirm a dfp for.
//	[Required] fingerprintId: the fingerprint id of the fingerprint you wish to confirm.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.

func (r *Request) ConfirmDfp(c *sa.Client, userID string, fingerprintId string) (*Response, error) {
	r.UserID = userID
	r.FingerprintID = fingerprintId
	confirmResponse, err := r.Post(c, confirmEndpoint)
	if err != nil {
		return nil, err
	}
	return confirmResponse, nil
}
<commit_msg>Space fix for golint<commit_after>package dfp

import (
	"encoding/json"
	"net/http"

	sa "github.com/secureauthcorp/saidp-sdk-go"
)

/*
**********************************************************************
*   @author jhickman@secureauth.com
*
*  Copyright (c) 2017, SecureAuth
*  All rights reserved.
*
*    Redistribution and use in source and binary forms, with or without modification,
*    are permitted provided that the following conditions are met:
*
*    1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
*
*    2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
*    in the documentation and/or other materials provided with the distribution.
*
*    3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived
*    from this software without specific prior written permission.
*
*    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
*    THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
*    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
*    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
*    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
*    EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**********************************************************************
 */

const (
	jsEndpoint      = "/api/v1/dfp/js"
	valEndpoint     = "/api/v1/dfp/validate"
	confirmEndpoint = "/api/v1/dfp/confirm"
)

// Response :
//	Response struct that will be populated after the post request.
type Response struct {
	FingerprintID   string         `json:"fingerprint_id,omitempty"`
	FingerprintName string         `json:"fingerprint_name,omitempty"`
	Score           string         `json:"score,omitempty"`
	MatchScore      string         `json:"match_score,omitempty"`
	UpdateScore     string         `json:"update_score,omitempty"`
	Status          string         `json:"status,omitempty"`
	Message         string         `json:"message,omitempty"`
	UserID          string         `json:"user_id,omitempty"`
	Source          string         `json:"src,omitempty"`
	HTTPResponse    *http.Response `json:"-,omitempty"`
}

// Request :
//	Request struct to build the required post parameters.
// Fields:
//	[Required] UserID: the username that you wish to validate a fingerprint for.
//	HostAddress: the IP Address of the user.
//	FingerprintID: used to validate known fingerprint or confirm a fingerprint.
//	Fingerprint: fingerprint value struct required to validate a fingerprint.
type Request struct {
	UserID        string      `json:"user_id,omitempty"`
	HostAddress   string      `json:"host_address,omitempty"`
	FingerprintID string      `json:"fingerprint_id,omitempty"`
	Fingerprint   Fingerprint `json:"fingerprint,omitempty"`
}

// Fingerprint :
//	Details for the fingerprint makeup.
// Fields:
//	All fields should be populated from the Json string returned by the imported javascript.
type Fingerprint struct {
	Fonts          string `json:"fonts,omitempty"`
	Plugins        string `json:"plugins,omitempty"`
	Timezone       string `json:"timezone,omitempty"`
	Video          string `json:"video,omitempty"`
	LocalStorage   string `json:"local_storage,omitempty"`
	SessionStorage string `json:"session_storage,omitempty"`
	IeUserData     string `json:"ie_user_data,omitempty"`
	CookieEnabled  string `json:"cookie_enabled,omitempty"`
	UserAgent      string `json:"user_agent,omitempty"`
	Accept         string `json:"accept,omitempty"`
	AcceptCharset  string `json:"accept_charset,omitempty"`
	AcceptEncoding string `json:"accept_encoding,omitempty"`
	AcceptLang     string `json:"accept_language,omitempty"`
}

// Get :
//	Executes a get to the dfp/js endpoint.
// Parameters:
//	[Required] r: struct used to perform get request.
//	[Required] c: passing in the client containing authorization and host information.
//	endpoint: the endpoint for the get request.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) Get(c *sa.Client, endpoint string) (*Response, error) {
	httpRequest, err := c.BuildGetRequest(endpoint)
	if err != nil {
		return nil, err
	}
	httpResponse, err := c.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	dfpResponse := new(Response)
	if err := json.NewDecoder(httpResponse.Body).Decode(dfpResponse); err != nil {
		return nil, err
	}
	dfpResponse.HTTPResponse = httpResponse
	httpResponse.Body.Close()
	return dfpResponse, nil
}

// Post :
//	Executes a post to the dfp endpoint.
// Parameters:
// 	[Required] r: should have all required fields of the struct populated before using.
// 	[Required] c: passing in the client containing authorization and host information.
//	[Required] endpoint: the endpoint for the post request.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) Post(c *sa.Client, endpoint string) (*Response, error) {
	jsonRequest, err := json.Marshal(r)
	if err != nil {
		return nil, err
	}
	httpRequest, err := c.BuildPostRequest(endpoint, string(jsonRequest))
	if err != nil {
		return nil, err
	}
	httpResponse, err := c.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	dfpResponse := new(Response)
	if err := json.NewDecoder(httpResponse.Body).Decode(dfpResponse); err != nil {
		return nil, err
	}
	dfpResponse.HTTPResponse = httpResponse
	httpResponse.Body.Close()
	return dfpResponse, nil
}

// GetDfpJs :
//	Helper function for Get request to retrieve the fingerprint javascript source.
// Parameters:
//	[Required] c: passing in the client containing authorization and host information.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) GetDfpJs(c *sa.Client) (*Response, error) {
	dpfResponse, err := r.Get(c, jsEndpoint)
	if err != nil {
		return nil, err
	}
	return dpfResponse, nil
}

// ValidateDfp :
//	Helper function for posting to the dfp validate endpoint.
// Parameters:
//	[Required] c: passing in the client containing authorization and host information.
//	[Required] userID: the username of the user you wish to validate a dfp for.
//	[Required] hostAddress: the ip address of the user's device.
//	fingerprintId: if it is a known fingerprint, provide the fingerprint id to validate against.
//	[Required] fingerprint: the json string returned by the javascript dfp script.
//	[Required] accept: accept header of the users request.
//	[Required] acceptCharset: the accept_charset header of the users request.
//	[Required] acceptEncoding: the accept_encoding header of the users request.
//	[Required] acceptLanguage: the accept_language header of the users request.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) ValidateDfp(c *sa.Client, userID string, hostAddress string, fingerprintId string, fingerprint string, accept string, acceptCharset string, acceptEncoding string, acceptLanguage string) (*Response, error) {
	if err := json.Unmarshal([]byte(fingerprint), &r); err != nil {
		return nil, err
	}
	r.Fingerprint.Accept = accept
	r.Fingerprint.AcceptCharset = acceptCharset
	r.Fingerprint.AcceptEncoding = acceptEncoding
	r.Fingerprint.AcceptLang = acceptLanguage
	r.FingerprintID = fingerprintId
	r.UserID = userID
	r.HostAddress = hostAddress
	validateResponse, err := r.Post(c, valEndpoint)
	if err != nil {
		return nil, err
	}
	return validateResponse, nil
}

// Summary:
//	Helper function for posting to the dfp validate endpoint.
// Parameters:
//	[Required] c: passing in the client containing authorization and host information.
//	[Required] userID: the username of the user you wish to confirm a dfp for.
//	[Required] fingerprintId: the fingerprint id of the fingerprint you wish to confirm.
// Returns:
//	Response: Struct marshaled from the Json response from the API endpoints.
//	Error: If an error is encountered, response will be nil and the error must be handled.
func (r *Request) ConfirmDfp(c *sa.Client, userID string, fingerprintId string) (*Response, error) {
	r.UserID = userID
	r.FingerprintID = fingerprintId
	confirmResponse, err := r.Post(c, confirmEndpoint)
	if err != nil {
		return nil, err
	}
	return confirmResponse, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011-2013 Frederic Langlet
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package entropy

import (
	"container/heap"
	"fmt"
	"kanzi"
)

const (
	FULL_ALPHABET            = 0
	PARTIAL_ALPHABET         = 1
	ALPHABET_256             = 0
	ALPHABET_NOT_256         = 1
	DELTA_ENCODED_ALPHABET   = 0
	BIT_ENCODED_ALPHABET_256 = 1
	PRESENT_SYMBOLS_MASK     = 0
	ABSENT_SYMBOLS_MASK      = 1
)

type ErrorComparator struct {
	symbols []int
	errors  []int
}

func ByIncreasingError(symbols []int, errors []int) ErrorComparator {
	return ErrorComparator{symbols: symbols, errors: errors}
}

func (this ErrorComparator) Less(i, j int) bool {
	// Check errors (natural order) as first key
	ri := this.symbols[i]
	rj := this.symbols[j]

	if this.errors[ri] != this.errors[rj] {
		return this.errors[ri] < this.errors[rj]
	}

	// Check index (natural order) as second key
	return ri < rj
}

func (this ErrorComparator) Len() int {
	return len(this.symbols)
}

func (this ErrorComparator) Swap(i, j int) {
	this.symbols[i], this.symbols[j] = this.symbols[j], this.symbols[i]
}

type FreqSortData struct {
	frequencies []int
	errors      []int
	symbol      int
}

type FreqSortPriorityQueue []*FreqSortData

func (this FreqSortPriorityQueue) Len() int {
	return len(this)
}

func (this FreqSortPriorityQueue) Less(i, j int) bool {
	di := this[i]
	dj := this[j]

	// Decreasing error
	if di.errors[di.symbol] != dj.errors[dj.symbol] {
		return di.errors[di.symbol] > dj.errors[dj.symbol]
	}

	// Decreasing frequency
	if di.frequencies[di.symbol] != dj.frequencies[dj.symbol] {
		return di.frequencies[di.symbol] > dj.frequencies[dj.symbol]
	}

	// Decreasing symbol
	return dj.symbol < di.symbol
}

func (this FreqSortPriorityQueue) Swap(i, j int) {
	this[i], this[j] = this[j], this[i]
}

func (this *FreqSortPriorityQueue) Push(data interface{}) {
	*this = append(*this, data.(*FreqSortData))
}

func (this *FreqSortPriorityQueue) Pop() interface{} {
	old := *this
	n := len(old)
	data := old[n-1]
	*this = old[0 : n-1]
	return data
}

type EntropyUtils struct {
	ranks  []int
	errors []int
}

func NewEntropyUtils() (*EntropyUtils, error) {
	this := new(EntropyUtils)
	this.ranks = make([]int, 0)
	this.errors = make([]int, 0)
	return this, nil
}

// alphabet must be sorted in increasing order
// alphabet length must be a power of 2
func EncodeAlphabet(obs kanzi.OutputBitStream, alphabet []int) int {
	alphabetSize := cap(alphabet)
        count := len(alphabet)

	// Alphabet length must be a power of 2
	if alphabetSize&(alphabetSize-1) != 0 {
		return -1
	}

	// First, push alphabet encoding mode
	if alphabetSize > 0 && count == alphabetSize {
		// Full alphabet
		obs.WriteBit(FULL_ALPHABET)

		if count == 256 {
			obs.WriteBit(ALPHABET_256) // shortcut
		} else {
			log := uint(1)

			for 1<<log <= count {
				log++
			}

			// Write alphabet size
			obs.WriteBit(ALPHABET_NOT_256)
			obs.WriteBits(uint64(log-1), 5)
			obs.WriteBits(uint64(count), log)
		}

		return count
	}

	obs.WriteBit(PARTIAL_ALPHABET)

	if alphabetSize == 256 && count >= 32 && count <= 224 {
		// Regular alphabet of symbols less than 256
		obs.WriteBit(BIT_ENCODED_ALPHABET_256)
		masks := make([]uint64, 4)

		for i := 0; i < count; i++ {
			masks[alphabet[i]>>6] |= (1 << uint(alphabet[i]&63))
		}

		for i := range masks {
			obs.WriteBits(masks[i], 64)
		}

		return count
	}

	obs.WriteBit(DELTA_ENCODED_ALPHABET)
	diffs := make([]int, 32)

	if alphabetSize-count < count {
		// Encode all missing symbols
		count = alphabetSize - count

		log := uint(1)

		for 1<<log <= count {
			log++
		}

		// Write length
		obs.WriteBits(uint64(log-1), 4)
		obs.WriteBits(uint64(count), log)

		if count == 0 {
			return 0
		}

		obs.WriteBit(ABSENT_SYMBOLS_MASK)

		log = 1

		for 1<<log <= alphabetSize {
			log++
		}

		// Write log(alphabet size)
		obs.WriteBits(uint64(log-1), 5)
		symbol := 0
		previous := 0

		// Create deltas of missing symbols
		for i, n := 0, 0; n < alphabetSize; {
			if symbol == alphabet[i] {
				if i < len(alphabet)-1-count {
					i++
				}

				symbol++
				continue
			}

			diffs[n] = int(symbol) - int(previous)
			symbol++
			previous = symbol
			n++
		}
	} else {
		// Encode all present symbols
		log := uint(1)

		for 1<<log <= count {
			log++
		}

		// Write length
		obs.WriteBits(uint64(log-1), 4)
		obs.WriteBits(uint64(count), log)

		if count == 0 {
			return 0
		}

		obs.WriteBit(PRESENT_SYMBOLS_MASK)
		previous := 0

		// Create deltas of present symbols
		for i := 0; i < count; i++ {
			diffs[i] = int(alphabet[i]) - int(previous)
			previous = alphabet[i] + 1
		}
	}

	ckSize := 16

	if count <= 64 {
		ckSize = 8
	}

	// Encode all deltas by chunks
	for i := 0; i < count; i += ckSize {
		max := 0

		// Find log(max(deltas)) for this chunk
		for j := i; j < count && j < i+ckSize; j++ {
			if max < diffs[j] {
				max = diffs[j]
			}
		}

		log := uint(1)

		for 1<<log <= max {
			log++
		}

		obs.WriteBits(uint64(log-1), 4)

		// Write deltas for this chunk
		for j := i; j < count && j < i+ckSize; j++ {
			encodeSize(obs, log, uint64(diffs[j]))
		}
	}

	return alphabetSize
}

func encodeSize(obs kanzi.OutputBitStream, log uint, val uint64) {
	obs.WriteBits(val, log)
}

func decodeSize(ibs kanzi.InputBitStream, log uint) uint64 {
	return ibs.ReadBits(log)
}

func DecodeAlphabet(ibs kanzi.InputBitStream, alphabet []int) (int, error) {
	// Read encoding mode from bitstream
	aphabetType := ibs.ReadBit()

	if aphabetType == FULL_ALPHABET {
		var alphabetSize int

		if ibs.ReadBit() == ALPHABET_256 {
			alphabetSize = 256
		} else {
			log := uint(1 + ibs.ReadBits(5))
			alphabetSize = int(ibs.ReadBits(log))
		}

		// Full alphabet
		for i := 0; i < alphabetSize; i++ {
			alphabet[i] = int(i)
		}

		return alphabetSize, nil
	}

	count := 0
	mode := ibs.ReadBit()

	if mode == BIT_ENCODED_ALPHABET_256 {
		// Decode presence flags
		for i := 0; i < 256; i += 64 {
			val := ibs.ReadBits(64)

			for j := 0; j < 64; j++ {
				if val&(uint64(1)<<uint(j)) != 0 {
					alphabet[count] = int(i + j)
					count++
				}
			}
		}

		return count, nil
	}
	// DELTA_ENCODED_ALPHABET
	log := uint(1 + ibs.ReadBits(4))
	count = int(ibs.ReadBits(log))

	if count == 0 {
		return 0, nil
	}

	ckSize := 16

	if count <= 64 {
		ckSize = 8
	}

	n := 0
	symbol := 0

	if ibs.ReadBit() == ABSENT_SYMBOLS_MASK {
		alphabetSize := 1 << uint(ibs.ReadBits(5))

		// Read missing symbols
		for i := 0; i < count; i += ckSize {
			log = uint(1 + ibs.ReadBits(4))

			// Read deltas for this chunk
			for j := i; j < count && j < i+ckSize; j++ {
				next := symbol + int(decodeSize(ibs, log))

				for symbol < next && n < alphabetSize {
					alphabet[n] = symbol
					symbol++
					n++
				}

				symbol++
			}
		}

		count = alphabetSize - count

		for n < count {
			alphabet[n] = symbol
			n++
			symbol++
		}

	} else {
		// Read present symbols
		for i := 0; i < count; i += ckSize {
			log = uint(1 + ibs.ReadBits(4))

			// Read deltas for this chunk
			for j := i; j < count && j < i+ckSize; j++ {
				symbol += int(decodeSize(ibs, log))
				alphabet[j] = symbol
				symbol++
			}
		}
	}

	return count, nil
}

// Returns the size of the alphabet
// The alphabet and freqs parameters are updated
func (this *EntropyUtils) NormalizeFrequencies(freqs []int, alphabet []int, count int, scale int) (int, error) {
	if count == 0 {
		return 0, nil
	}

	if scale < 1<<8 || scale > 1<<16 {
		return 0, fmt.Errorf("Invalid range parameter: %v (must be in [256..65536])", scale)
	}

	alphabetSize := 0

	// range == count intcut
	if count == scale {
		for i := range freqs {
			if freqs[i] != 0 {
				alphabet[alphabetSize] = int(i)
				alphabetSize++
			}
		}

		return alphabetSize, nil
	}

	if len(this.ranks) < len(alphabet) {
		this.ranks = make([]int, len(alphabet))
	}

	if len(this.errors) < len(alphabet) {
		this.errors = make([]int, len(alphabet))
	}

	sum := -scale

	// Scale frequencies by stretching distribution over complete range
	for i := range alphabet {
		alphabet[i] = 0
		this.errors[i] = -1
		this.ranks[i] = int(i)

		if freqs[i] == 0 {
			continue
		}

		sf := int64(freqs[i]) * int64(scale)
		scaledFreq := int(sf / int64(count))

		if scaledFreq == 0 {
			// Quantum of frequency
			scaledFreq = 1
		} else {
			// Find best frequency rounding value
			errCeiling := int64(scaledFreq+1)*int64(count) - sf
			errFloor := sf - int64(scaledFreq)*int64(count)

			if errCeiling < errFloor {
				scaledFreq++
				this.errors[i] = int(errCeiling)
			} else {
				this.errors[i] = int(errFloor)
			}
		}

		alphabet[alphabetSize] = int(i)
		alphabetSize++
		sum += scaledFreq
		freqs[i] = scaledFreq
	}

	if alphabetSize == 0 {
		return 0, nil
	}

	if alphabetSize == 1 {
		freqs[alphabet[0]] = scale
		return 1, nil
	}

	if sum != 0 {
		// Need to normalize frequency sum to range
		var inc int

		if sum > 0 {
			inc = -1
		} else {
			inc = 1
		}

		queue := make(FreqSortPriorityQueue, 0)

		// Create sorted queue of present symbols (except those with 'quantum frequency')
		for i := 0; i < alphabetSize; i++ {
			if this.errors[alphabet[i]] >= 0 {
				heap.Push(&queue, &FreqSortData{errors: this.errors, frequencies: freqs, symbol: alphabet[i]})
			}
		}

		for sum != 0 && len(queue) > 0 {
			// Remove symbol with highest error
			fsd := heap.Pop(&queue).(*FreqSortData)

			// Do not zero out any frequency
			if freqs[fsd.symbol] == -inc {
				continue
			}

			// Distort frequency and error
			freqs[fsd.symbol] += inc
			this.errors[fsd.symbol] -= scale
			sum += inc
			heap.Push(&queue, fsd)
		}
	}

	return alphabetSize, nil
}
<commit_msg>Fix incorrect loop end condition and array size.<commit_after>/*
Copyright 2011-2013 Frederic Langlet
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package entropy

import (
	"container/heap"
	"fmt"
	"kanzi"
)

const (
	FULL_ALPHABET            = 0
	PARTIAL_ALPHABET         = 1
	ALPHABET_256             = 0
	ALPHABET_NOT_256         = 1
	DELTA_ENCODED_ALPHABET   = 0
	BIT_ENCODED_ALPHABET_256 = 1
	PRESENT_SYMBOLS_MASK     = 0
	ABSENT_SYMBOLS_MASK      = 1
)

type ErrorComparator struct {
	symbols []int
	errors  []int
}

func ByIncreasingError(symbols []int, errors []int) ErrorComparator {
	return ErrorComparator{symbols: symbols, errors: errors}
}

func (this ErrorComparator) Less(i, j int) bool {
	// Check errors (natural order) as first key
	ri := this.symbols[i]
	rj := this.symbols[j]

	if this.errors[ri] != this.errors[rj] {
		return this.errors[ri] < this.errors[rj]
	}

	// Check index (natural order) as second key
	return ri < rj
}

func (this ErrorComparator) Len() int {
	return len(this.symbols)
}

func (this ErrorComparator) Swap(i, j int) {
	this.symbols[i], this.symbols[j] = this.symbols[j], this.symbols[i]
}

type FreqSortData struct {
	frequencies []int
	errors      []int
	symbol      int
}

type FreqSortPriorityQueue []*FreqSortData

func (this FreqSortPriorityQueue) Len() int {
	return len(this)
}

func (this FreqSortPriorityQueue) Less(i, j int) bool {
	di := this[i]
	dj := this[j]

	// Decreasing error
	if di.errors[di.symbol] != dj.errors[dj.symbol] {
		return di.errors[di.symbol] > dj.errors[dj.symbol]
	}

	// Decreasing frequency
	if di.frequencies[di.symbol] != dj.frequencies[dj.symbol] {
		return di.frequencies[di.symbol] > dj.frequencies[dj.symbol]
	}

	// Decreasing symbol
	return dj.symbol < di.symbol
}

func (this FreqSortPriorityQueue) Swap(i, j int) {
	this[i], this[j] = this[j], this[i]
}

func (this *FreqSortPriorityQueue) Push(data interface{}) {
	*this = append(*this, data.(*FreqSortData))
}

func (this *FreqSortPriorityQueue) Pop() interface{} {
	old := *this
	n := len(old)
	data := old[n-1]
	*this = old[0 : n-1]
	return data
}

type EntropyUtils struct {
	ranks  []int
	errors []int
}

func NewEntropyUtils() (*EntropyUtils, error) {
	this := new(EntropyUtils)
	this.ranks = make([]int, 0)
	this.errors = make([]int, 0)
	return this, nil
}

// alphabet must be sorted in increasing order
// alphabet length must be a power of 2
func EncodeAlphabet(obs kanzi.OutputBitStream, alphabet []int) int {
	alphabetSize := cap(alphabet)
        count := len(alphabet)

	// Alphabet length must be a power of 2
	if alphabetSize&(alphabetSize-1) != 0 {
		return -1
	}

	// First, push alphabet encoding mode
	if alphabetSize > 0 && count == alphabetSize {
		// Full alphabet
		obs.WriteBit(FULL_ALPHABET)

		if count == 256 {
			obs.WriteBit(ALPHABET_256) // shortcut
		} else {
			log := uint(1)

			for 1<<log <= count {
				log++
			}

			// Write alphabet size
			obs.WriteBit(ALPHABET_NOT_256)
			obs.WriteBits(uint64(log-1), 5)
			obs.WriteBits(uint64(count), log)
		}

		return count
	}

	obs.WriteBit(PARTIAL_ALPHABET)

	if alphabetSize == 256 && count >= 32 && count <= 224 {
		// Regular alphabet of symbols less than 256
		obs.WriteBit(BIT_ENCODED_ALPHABET_256)
		masks := make([]uint64, 4)

		for i := 0; i < count; i++ {
			masks[alphabet[i]>>6] |= (1 << uint(alphabet[i]&63))
		}

		for i := range masks {
			obs.WriteBits(masks[i], 64)
		}

		return count
	}

	obs.WriteBit(DELTA_ENCODED_ALPHABET)
	diffs := make([]int, count)

	if alphabetSize-count < count {
		// Encode all missing symbols
		count = alphabetSize - count
		log := uint(1)

		for 1<<log <= count {
			log++
		}

		// Write length
		obs.WriteBits(uint64(log-1), 4)
		obs.WriteBits(uint64(count), log)

		if count == 0 {
			return 0
		}

		obs.WriteBit(ABSENT_SYMBOLS_MASK)
		log = 1

		for 1<<log <= alphabetSize {
			log++
		}

		// Write log(alphabet size)
		obs.WriteBits(uint64(log-1), 5)
		symbol := 0
		previous := 0

		// Create deltas of missing symbols
		for i, n := 0, 0; n < count; {
			if symbol == alphabet[i] {
				if i < len(alphabet)-1-count {
					i++
				}

				symbol++
				continue
			}

			diffs[n] = symbol - previous
			symbol++
			previous = symbol
			n++
		}
	} else {
		// Encode all present symbols
		log := uint(1)

		for 1<<log <= count {
			log++
		}

		// Write length
		obs.WriteBits(uint64(log-1), 4)
		obs.WriteBits(uint64(count), log)

		if count == 0 {
			return 0
		}

		obs.WriteBit(PRESENT_SYMBOLS_MASK)
		previous := 0

		// Create deltas of present symbols
		for i := 0; i < count; i++ {
			diffs[i] = int(alphabet[i]) - int(previous)
			previous = alphabet[i] + 1
		}
	}

	ckSize := 16

	if count <= 64 {
		ckSize = 8
	}

	// Encode all deltas by chunks
	for i := 0; i < count; i += ckSize {
		max := 0

		// Find log(max(deltas)) for this chunk
		for j := i; j < count && j < i+ckSize; j++ {
			if max < diffs[j] {
				max = diffs[j]
			}
		}

		log := uint(1)

		for 1<<log <= max {
			log++
		}

		obs.WriteBits(uint64(log-1), 4)

		// Write deltas for this chunk
		for j := i; j < count && j < i+ckSize; j++ {
			encodeSize(obs, log, uint64(diffs[j]))
		}
	}

	return alphabetSize
}

func encodeSize(obs kanzi.OutputBitStream, log uint, val uint64) {
	obs.WriteBits(val, log)
}

func decodeSize(ibs kanzi.InputBitStream, log uint) uint64 {
	return ibs.ReadBits(log)
}

func DecodeAlphabet(ibs kanzi.InputBitStream, alphabet []int) (int, error) {
	// Read encoding mode from bitstream
	aphabetType := ibs.ReadBit()

	if aphabetType == FULL_ALPHABET {
		var alphabetSize int

		if ibs.ReadBit() == ALPHABET_256 {
			alphabetSize = 256
		} else {
			log := uint(1 + ibs.ReadBits(5))
			alphabetSize = int(ibs.ReadBits(log))
		}

		// Full alphabet
		for i := 0; i < alphabetSize; i++ {
			alphabet[i] = int(i)
		}

		return alphabetSize, nil
	}

	count := 0
	mode := ibs.ReadBit()

	if mode == BIT_ENCODED_ALPHABET_256 {
		// Decode presence flags
		for i := 0; i < 256; i += 64 {
			val := ibs.ReadBits(64)

			for j := 0; j < 64; j++ {
				if val&(uint64(1)<<uint(j)) != 0 {
					alphabet[count] = int(i + j)
					count++
				}
			}
		}

		return count, nil
	}
	// DELTA_ENCODED_ALPHABET
	log := uint(1 + ibs.ReadBits(4))
	count = int(ibs.ReadBits(log))

	if count == 0 {
		return 0, nil
	}

	ckSize := 16

	if count <= 64 {
		ckSize = 8
	}

	n := 0
	symbol := 0

	if ibs.ReadBit() == ABSENT_SYMBOLS_MASK {
		alphabetSize := 1 << uint(ibs.ReadBits(5))

		// Read missing symbols
		for i := 0; i < count; i += ckSize {
			log = uint(1 + ibs.ReadBits(4))

			// Read deltas for this chunk
			for j := i; j < count && j < i+ckSize; j++ {
				next := symbol + int(decodeSize(ibs, log))

				for symbol < next && n < alphabetSize {
					alphabet[n] = symbol
					symbol++
					n++
				}

				symbol++
			}
		}

		count = alphabetSize - count

		for n < count {
			alphabet[n] = symbol
			n++
			symbol++
		}

	} else {
		// Read present symbols
		for i := 0; i < count; i += ckSize {
			log = uint(1 + ibs.ReadBits(4))

			// Read deltas for this chunk
			for j := i; j < count && j < i+ckSize; j++ {
				symbol += int(decodeSize(ibs, log))
				alphabet[j] = symbol
				symbol++
			}
		}
	}

	return count, nil
}

// Returns the size of the alphabet
// The alphabet and freqs parameters are updated
func (this *EntropyUtils) NormalizeFrequencies(freqs []int, alphabet []int, count int, scale int) (int, error) {
	if count == 0 {
		return 0, nil
	}

	if scale < 1<<8 || scale > 1<<16 {
		return 0, fmt.Errorf("Invalid range parameter: %v (must be in [256..65536])", scale)
	}

	alphabetSize := 0

	// range == count intcut
	if count == scale {
		for i := range freqs {
			if freqs[i] != 0 {
				alphabet[alphabetSize] = int(i)
				alphabetSize++
			}
		}

		return alphabetSize, nil
	}

	if len(this.ranks) < len(alphabet) {
		this.ranks = make([]int, len(alphabet))
	}

	if len(this.errors) < len(alphabet) {
		this.errors = make([]int, len(alphabet))
	}

	sum := -scale

	// Scale frequencies by stretching distribution over complete range
	for i := range alphabet {
		alphabet[i] = 0
		this.errors[i] = -1
		this.ranks[i] = int(i)

		if freqs[i] == 0 {
			continue
		}

		sf := int64(freqs[i]) * int64(scale)
		scaledFreq := int(sf / int64(count))

		if scaledFreq == 0 {
			// Quantum of frequency
			scaledFreq = 1
		} else {
			// Find best frequency rounding value
			errCeiling := int64(scaledFreq+1)*int64(count) - sf
			errFloor := sf - int64(scaledFreq)*int64(count)

			if errCeiling < errFloor {
				scaledFreq++
				this.errors[i] = int(errCeiling)
			} else {
				this.errors[i] = int(errFloor)
			}
		}

		alphabet[alphabetSize] = int(i)
		alphabetSize++
		sum += scaledFreq
		freqs[i] = scaledFreq
	}

	if alphabetSize == 0 {
		return 0, nil
	}

	if alphabetSize == 1 {
		freqs[alphabet[0]] = scale
		return 1, nil
	}

	if sum != 0 {
		// Need to normalize frequency sum to range
		var inc int

		if sum > 0 {
			inc = -1
		} else {
			inc = 1
		}

		queue := make(FreqSortPriorityQueue, 0)

		// Create sorted queue of present symbols (except those with 'quantum frequency')
		for i := 0; i < alphabetSize; i++ {
			if this.errors[alphabet[i]] >= 0 {
				heap.Push(&queue, &FreqSortData{errors: this.errors, frequencies: freqs, symbol: alphabet[i]})
			}
		}

		for sum != 0 && len(queue) > 0 {
			// Remove symbol with highest error
			fsd := heap.Pop(&queue).(*FreqSortData)

			// Do not zero out any frequency
			if freqs[fsd.symbol] == -inc {
				continue
			}

			// Distort frequency and error
			freqs[fsd.symbol] += inc
			this.errors[fsd.symbol] -= scale
			sum += inc
			heap.Push(&queue, fsd)
		}
	}

	return alphabetSize, nil
}
<|endoftext|>


<commit_before>package config

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	jwt "github.com/dgrijalva/jwt-go"
	konfig "github.com/koding/kite/config"
	"github.com/koding/kite/kitekey"
)

// KonfigCache is a default konfig.bolt configuration.
//
// The konfig.bolt stores user configuration for
// KD / Klient apps.
var KonfigCache = &CacheOptions{
	File: filepath.Join(KodingHome(), "konfig.bolt"),
	BoltDB: &bolt.Options{
		Timeout: 5 * time.Second,
	},
	Bucket: []byte("konfig"),
}

// Endpoints represents a configuration of Koding
// endpoints, which are used by KD / Klient.
type Endpoints struct {
	// Koding base endpoint.
	Koding *Endpoint `json:"koding,omitempty"`

	// Tunnel / proxy environment endpoints.
	Tunnel  *Endpoint `json:"tunnel,omitempty"`
	IP      *Endpoint `json:"ip,omitempty"`
	IPCheck *Endpoint `json:"ipCheck,omitempty"`

	// Klient / KD endpoints.
	KlientLatest *Endpoint `json:"klientLatest,omitempty"`
	KDLatest     *Endpoint `json:"kdLatest,omitempty"`
	Klient       *Endpoint `json:"klient,omitempty"`
}

// Kloud gives an endpoint for Kloud kite.
func (e *Endpoints) Kloud() *Endpoint {
	return e.Koding.WithPath("/kloud/kite")
}

// Kontrol gives an endpoint for Kontrol kite.
func (e *Endpoints) Kontrol() *Endpoint {
	return e.Koding.WithPath("/kontrol/kite")
}

// Remote gives an endpoint for remote.api.
func (e *Endpoints) Remote() *Endpoint {
	return e.Koding.WithPath("/remote.api")
}

// Social gives an endpoint for social.api.
func (e *Endpoints) Social() *Endpoint {
	return e.Koding.WithPath("/api/social")
}

// Local describes configuration of local paths.
type Local struct {
	// Mount is a default home path of mounted directories.
	//
	// If empty, defaults to ~/koding/mnt/.
	MountHome string `json:"mountHome,omitempty"`

	// Mounts maps named mounts to local paths.
	//
	// The Mounts["default"] export is used as
	// a default one when caller does not specify
	// a mount path.
	//
	// The Mounts["default"] defaults to $HOME.
	Mounts map[string]string `json:"mounts,omitempty"`
}

// MountPath gives a path for the named mount.
//
// If the named mount does not exist, it returns false.
//
// If the path contains '~' - it is expended to
// a home directory of a current user.
func (l *Local) MountPath(name string) (string, bool) {
	if l == nil {
		return "", false
	}

	if dir, ok := l.Mounts[name]; ok {
		return expandHome(dir), true
	}

	return "", false
}

// Template represents a KD template configuration.
type Template struct {
	File string `json:"file,omitempty"`
}

// Konfig represents a single configuration stored
// in a konfig.bolt database.
type Konfig struct {
	Endpoints *Endpoints `json:"endpoints,omitempty"`

	KontrolURL string `json:"kontrolURL,omitempty"` // deprecated / read-only
	TunnelURL  string `json:"tunnelURL,omitempty"`  // deprecated / read-only

	// Kite configuration.
	Environment string `json:"environment,omitempty"`
	KiteKeyFile string `json:"kiteKeyFile,omitempty"`
	KiteKey     string `json:"kiteKey,omitempty"`

	// Local describes configuration of local paths.
	Local *Local `json:"local,omitempty"`

	// Template describes configuration of KD template.
	Template *Template `json:"template,omitempty"`

	// Koding networking configuration.
	//
	// TODO(rjeczalik): store command line flags in konfig.bolt
	// per Koding executable (KD / Klient).
	TunnelID string `json:"tunnelID,omitempty"`

	// Public S3 bucket for writing logs.
	PublicBucketName   string `json:"publicBucketName,omitempty"`
	PublicBucketRegion string `json:"publicBucketRegion,omitempty"`

	Debug bool `json:"debug,string,omitempty"`
}

// KiteHome gives directory of the kite.key file.
//
// Deprecated: Use KiteKeyFile instead.
func (k *Konfig) KiteHome() string {
	return filepath.Dir(k.KiteKeyFile)
}

// KiteConfig build *kite.Config value which is used
// to initialize new kite.Kite values.
func (k *Konfig) KiteConfig() *konfig.Config {
	return k.buildKiteConfig()
}

// KlientGzURL gives an URL for Klient binary.
//
// TODO(rjeczalik): Rework into lookup and move to kloud/metadata package.
func (k *Konfig) KlientGzURL() string {
	u := *k.Endpoints.KlientLatest.Public.URL
	u.Path = path.Join(path.Dir(u.Path), "latest", "klient.gz")
	return u.String()
}

// Valid implements the stack.Validator interface.
func (k *Konfig) Valid() error {
	// TODO(rjeczalik): remove when KontrolURL is gone
	if _, err := url.Parse(k.KontrolURL); err == nil && k.KontrolURL != "" {
		return nil
	}

	if k.Endpoints == nil {
		return errors.New("endpoints are nil")
	}
	if k.Endpoints.Koding == nil {
		return errors.New("koding base endpoint is nil")
	}
	if k.Endpoints.Koding.Public == nil {
		return errors.New("public URL for koding base endpoint is nil")
	}
	return nil
}

// ID gives an identifier of the Konfig value.
//
// The konfig.bolt stores multiple configurations, one for each
// baseurl.
//
// The ID is unique per baseurl.
func (k *Konfig) ID() string {
	return ID(k.KodingPublic().String())
}

// ID creates an identifier for the given Koding base URL.
func ID(kodingURL string) string {
	if kodingURL == "" {
		return ""
	}
	if u, err := url.Parse(kodingURL); err == nil {
		// Since id is input sensitive we clean the path so "example.com/koding"
		// "example.com/koding/" are effectively the same urls.
		u.Path = strings.TrimRight(path.Clean(u.Path), "/")
		switch u.Scheme {
		case "http":
			u.Host = trimRight(u.Host, ":80")
		case "https":
			u.Host = trimRight(u.Host, ":443")
		}
		kodingURL = u.String()
	}
	hash := sha1.Sum([]byte(kodingURL))
	return hex.EncodeToString(hash[:4])
}

// KodingPublic is here for backward-compatibility purposes.
//
// Old klient and kd deployments may not have .Endpoints configuration
// on a first run, this is why we fallback to old KontrolURL field.
//
// Which we should eventually get rid of.
//
// Deprecated: Use k.Endpoints.Koding.Public instead.
func (k *Konfig) KodingPublic() *url.URL {
	if e := k.Endpoints; e != nil && e.Koding != nil && e.Koding.Public != nil {
		return e.Koding.Public.URL
	}

	if u, err := url.Parse(k.KontrolURL); err == nil {
		u.Path = ""
		return u
	}

	return nil
}

func (k *Konfig) buildKiteConfig() *konfig.Config {
	kiteKey := k.KiteKey

	if kiteKey == "" && k.KiteKeyFile != "" {
		if p, err := ioutil.ReadFile(k.KiteKeyFile); err == nil && len(p) != 0 {
			kiteKey = string(bytes.TrimSpace(p))
		}
	}

	if kiteKey != "" {
		tok, err := jwt.ParseWithClaims(kiteKey, &kitekey.KiteClaims{}, kitekey.GetKontrolKey)
		if err == nil {
			cfg := NewKiteConfig(k.Debug)

			if err = cfg.ReadToken(tok); err == nil {
				return cfg
			}
		}
	}

	if cfg, err := ReadKiteConfig(k.Debug); err == nil {
		return cfg
	}

	return NewKiteConfig(k.Debug)
}

// NewKonfigURL gives new configuration for the given base URL.
func NewKonfigURL(koding *url.URL) *Konfig {
	return &Konfig{
		Endpoints: &Endpoints{
			Koding: NewEndpointURL(koding),
		},
	}
}

type Konfigs map[string]*Konfig

func (kfg Konfigs) Slice() []*Konfig {
	keys := make([]string, 0, len(kfg))
	for k := range kfg {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	slice := make([]*Konfig, 0, len(kfg))
	for _, key := range keys {
		slice = append(slice, kfg[key])
	}
	return slice
}

// Environment is a hacky workaround for kd <-> klient environments.
// The managed klient expects to have kd from production channel,
// and devmanaged klient - from development. Depending from which
// app we load the the default Koding configuration, we need
// to cross-map the environments.
//
// TODO(rjeczalik): This should be fixed by removing managed / devmanaged
// channels - that is to improve "connect your vm" modal to
// not depend on special klient environment.
type Environments struct {
	Env       string
	KlientEnv string // Env is used if empty
	KDEnv     string // Env is used if empty
}

func (e *Environments) klientEnv() string {
	if e.KlientEnv != "" {
		return e.KlientEnv
	}
	return e.Env
}

func (e *Environments) kdEnv() string {
	if e.KDEnv != "" {
		return e.KDEnv
	}
	return e.Env
}

// NewKonfig creates new configuration by reading
// embedded kites/config/config.json file.
func NewKonfig(e *Environments) *Konfig {
	return &Konfig{
		Environment: e.Env,
		Endpoints: &Endpoints{
			Koding:       Builtin.Endpoints.KodingBase.Copy(),
			Tunnel:       Builtin.Endpoints.TunnelServer.Copy(),
			IP:           Builtin.Endpoints.IP.Copy(),
			IPCheck:      Builtin.Endpoints.IPCheck.Copy(),
			KlientLatest: ReplaceEnv(Builtin.Endpoints.KlientLatest, e.klientEnv()),
			KDLatest:     ReplaceEnv(Builtin.Endpoints.KDLatest, RmManaged(e.kdEnv())),
			Klient: &Endpoint{
				Private: &URL{&url.URL{
					Scheme: "http",
					Host:   "127.0.0.1:56789",
					Path:   "/kite",
				}},
			},
		},
		Local: &Local{
			MountHome: filepath.Join(CurrentUser.HomeDir, "koding", "mnt"),
			Mounts: map[string]string{
				"default": CurrentUser.HomeDir,
			},
		},
		Template: &Template{
			File: "kd.yaml",
		},
		PublicBucketName:   Builtin.Buckets.PublicLogs.Name,
		PublicBucketRegion: Builtin.Buckets.PublicLogs.Region,
		Debug:              false,
	}
}

func expandHome(path string) string {
	const home = "~" + string(os.PathSeparator)

	switch {
	case path == "~":
		return CurrentUser.HomeDir
	case strings.HasPrefix(path, home):
		return filepath.Join(CurrentUser.HomeDir, path[len(home):])
	default:
		return path
	}
}

func trimRight(str, substr string) string {
	if strings.HasSuffix(str, substr) {
		return str[:len(str)-len(substr)]
	}
	return str
}
<commit_msg>kites/config: use strings.TrimSuffix<commit_after>package config

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	jwt "github.com/dgrijalva/jwt-go"
	konfig "github.com/koding/kite/config"
	"github.com/koding/kite/kitekey"
)

// KonfigCache is a default konfig.bolt configuration.
//
// The konfig.bolt stores user configuration for
// KD / Klient apps.
var KonfigCache = &CacheOptions{
	File: filepath.Join(KodingHome(), "konfig.bolt"),
	BoltDB: &bolt.Options{
		Timeout: 5 * time.Second,
	},
	Bucket: []byte("konfig"),
}

// Endpoints represents a configuration of Koding
// endpoints, which are used by KD / Klient.
type Endpoints struct {
	// Koding base endpoint.
	Koding *Endpoint `json:"koding,omitempty"`

	// Tunnel / proxy environment endpoints.
	Tunnel  *Endpoint `json:"tunnel,omitempty"`
	IP      *Endpoint `json:"ip,omitempty"`
	IPCheck *Endpoint `json:"ipCheck,omitempty"`

	// Klient / KD endpoints.
	KlientLatest *Endpoint `json:"klientLatest,omitempty"`
	KDLatest     *Endpoint `json:"kdLatest,omitempty"`
	Klient       *Endpoint `json:"klient,omitempty"`
}

// Kloud gives an endpoint for Kloud kite.
func (e *Endpoints) Kloud() *Endpoint {
	return e.Koding.WithPath("/kloud/kite")
}

// Kontrol gives an endpoint for Kontrol kite.
func (e *Endpoints) Kontrol() *Endpoint {
	return e.Koding.WithPath("/kontrol/kite")
}

// Remote gives an endpoint for remote.api.
func (e *Endpoints) Remote() *Endpoint {
	return e.Koding.WithPath("/remote.api")
}

// Social gives an endpoint for social.api.
func (e *Endpoints) Social() *Endpoint {
	return e.Koding.WithPath("/api/social")
}

// Local describes configuration of local paths.
type Local struct {
	// Mount is a default home path of mounted directories.
	//
	// If empty, defaults to ~/koding/mnt/.
	MountHome string `json:"mountHome,omitempty"`

	// Mounts maps named mounts to local paths.
	//
	// The Mounts["default"] export is used as
	// a default one when caller does not specify
	// a mount path.
	//
	// The Mounts["default"] defaults to $HOME.
	Mounts map[string]string `json:"mounts,omitempty"`
}

// MountPath gives a path for the named mount.
//
// If the named mount does not exist, it returns false.
//
// If the path contains '~' - it is expended to
// a home directory of a current user.
func (l *Local) MountPath(name string) (string, bool) {
	if l == nil {
		return "", false
	}

	if dir, ok := l.Mounts[name]; ok {
		return expandHome(dir), true
	}

	return "", false
}

// Template represents a KD template configuration.
type Template struct {
	File string `json:"file,omitempty"`
}

// Konfig represents a single configuration stored
// in a konfig.bolt database.
type Konfig struct {
	Endpoints *Endpoints `json:"endpoints,omitempty"`

	KontrolURL string `json:"kontrolURL,omitempty"` // deprecated / read-only
	TunnelURL  string `json:"tunnelURL,omitempty"`  // deprecated / read-only

	// Kite configuration.
	Environment string `json:"environment,omitempty"`
	KiteKeyFile string `json:"kiteKeyFile,omitempty"`
	KiteKey     string `json:"kiteKey,omitempty"`

	// Local describes configuration of local paths.
	Local *Local `json:"local,omitempty"`

	// Template describes configuration of KD template.
	Template *Template `json:"template,omitempty"`

	// Koding networking configuration.
	//
	// TODO(rjeczalik): store command line flags in konfig.bolt
	// per Koding executable (KD / Klient).
	TunnelID string `json:"tunnelID,omitempty"`

	// Public S3 bucket for writing logs.
	PublicBucketName   string `json:"publicBucketName,omitempty"`
	PublicBucketRegion string `json:"publicBucketRegion,omitempty"`

	Debug bool `json:"debug,string,omitempty"`
}

// KiteHome gives directory of the kite.key file.
//
// Deprecated: Use KiteKeyFile instead.
func (k *Konfig) KiteHome() string {
	return filepath.Dir(k.KiteKeyFile)
}

// KiteConfig build *kite.Config value which is used
// to initialize new kite.Kite values.
func (k *Konfig) KiteConfig() *konfig.Config {
	return k.buildKiteConfig()
}

// KlientGzURL gives an URL for Klient binary.
//
// TODO(rjeczalik): Rework into lookup and move to kloud/metadata package.
func (k *Konfig) KlientGzURL() string {
	u := *k.Endpoints.KlientLatest.Public.URL
	u.Path = path.Join(path.Dir(u.Path), "latest", "klient.gz")
	return u.String()
}

// Valid implements the stack.Validator interface.
func (k *Konfig) Valid() error {
	// TODO(rjeczalik): remove when KontrolURL is gone
	if _, err := url.Parse(k.KontrolURL); err == nil && k.KontrolURL != "" {
		return nil
	}

	if k.Endpoints == nil {
		return errors.New("endpoints are nil")
	}
	if k.Endpoints.Koding == nil {
		return errors.New("koding base endpoint is nil")
	}
	if k.Endpoints.Koding.Public == nil {
		return errors.New("public URL for koding base endpoint is nil")
	}
	return nil
}

// ID gives an identifier of the Konfig value.
//
// The konfig.bolt stores multiple configurations, one for each
// baseurl.
//
// The ID is unique per baseurl.
func (k *Konfig) ID() string {
	return ID(k.KodingPublic().String())
}

// ID creates an identifier for the given Koding base URL.
func ID(kodingURL string) string {
	if kodingURL == "" {
		return ""
	}
	if u, err := url.Parse(kodingURL); err == nil {
		// Since id is input sensitive we clean the path so "example.com/koding"
		// "example.com/koding/" are effectively the same urls.
		u.Path = strings.TrimRight(path.Clean(u.Path), "/")
		switch u.Scheme {
		case "http":
			u.Host = strings.TrimSuffix(u.Host, ":80")
		case "https":
			u.Host = strings.TrimSuffix(u.Host, ":443")
		}
		kodingURL = u.String()
	}
	hash := sha1.Sum([]byte(kodingURL))
	return hex.EncodeToString(hash[:4])
}

// KodingPublic is here for backward-compatibility purposes.
//
// Old klient and kd deployments may not have .Endpoints configuration
// on a first run, this is why we fallback to old KontrolURL field.
//
// Which we should eventually get rid of.
//
// Deprecated: Use k.Endpoints.Koding.Public instead.
func (k *Konfig) KodingPublic() *url.URL {
	if e := k.Endpoints; e != nil && e.Koding != nil && e.Koding.Public != nil {
		return e.Koding.Public.URL
	}

	if u, err := url.Parse(k.KontrolURL); err == nil {
		u.Path = ""
		return u
	}

	return nil
}

func (k *Konfig) buildKiteConfig() *konfig.Config {
	kiteKey := k.KiteKey

	if kiteKey == "" && k.KiteKeyFile != "" {
		if p, err := ioutil.ReadFile(k.KiteKeyFile); err == nil && len(p) != 0 {
			kiteKey = string(bytes.TrimSpace(p))
		}
	}

	if kiteKey != "" {
		tok, err := jwt.ParseWithClaims(kiteKey, &kitekey.KiteClaims{}, kitekey.GetKontrolKey)
		if err == nil {
			cfg := NewKiteConfig(k.Debug)

			if err = cfg.ReadToken(tok); err == nil {
				return cfg
			}
		}
	}

	if cfg, err := ReadKiteConfig(k.Debug); err == nil {
		return cfg
	}

	return NewKiteConfig(k.Debug)
}

// NewKonfigURL gives new configuration for the given base URL.
func NewKonfigURL(koding *url.URL) *Konfig {
	return &Konfig{
		Endpoints: &Endpoints{
			Koding: NewEndpointURL(koding),
		},
	}
}

type Konfigs map[string]*Konfig

func (kfg Konfigs) Slice() []*Konfig {
	keys := make([]string, 0, len(kfg))
	for k := range kfg {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	slice := make([]*Konfig, 0, len(kfg))
	for _, key := range keys {
		slice = append(slice, kfg[key])
	}
	return slice
}

// Environment is a hacky workaround for kd <-> klient environments.
// The managed klient expects to have kd from production channel,
// and devmanaged klient - from development. Depending from which
// app we load the the default Koding configuration, we need
// to cross-map the environments.
//
// TODO(rjeczalik): This should be fixed by removing managed / devmanaged
// channels - that is to improve "connect your vm" modal to
// not depend on special klient environment.
type Environments struct {
	Env       string
	KlientEnv string // Env is used if empty
	KDEnv     string // Env is used if empty
}

func (e *Environments) klientEnv() string {
	if e.KlientEnv != "" {
		return e.KlientEnv
	}
	return e.Env
}

func (e *Environments) kdEnv() string {
	if e.KDEnv != "" {
		return e.KDEnv
	}
	return e.Env
}

// NewKonfig creates new configuration by reading
// embedded kites/config/config.json file.
func NewKonfig(e *Environments) *Konfig {
	return &Konfig{
		Environment: e.Env,
		Endpoints: &Endpoints{
			Koding:       Builtin.Endpoints.KodingBase.Copy(),
			Tunnel:       Builtin.Endpoints.TunnelServer.Copy(),
			IP:           Builtin.Endpoints.IP.Copy(),
			IPCheck:      Builtin.Endpoints.IPCheck.Copy(),
			KlientLatest: ReplaceEnv(Builtin.Endpoints.KlientLatest, e.klientEnv()),
			KDLatest:     ReplaceEnv(Builtin.Endpoints.KDLatest, RmManaged(e.kdEnv())),
			Klient: &Endpoint{
				Private: &URL{&url.URL{
					Scheme: "http",
					Host:   "127.0.0.1:56789",
					Path:   "/kite",
				}},
			},
		},
		Local: &Local{
			MountHome: filepath.Join(CurrentUser.HomeDir, "koding", "mnt"),
			Mounts: map[string]string{
				"default": CurrentUser.HomeDir,
			},
		},
		Template: &Template{
			File: "kd.yaml",
		},
		PublicBucketName:   Builtin.Buckets.PublicLogs.Name,
		PublicBucketRegion: Builtin.Buckets.PublicLogs.Region,
		Debug:              false,
	}
}

func expandHome(path string) string {
	const home = "~" + string(os.PathSeparator)

	switch {
	case path == "~":
		return CurrentUser.HomeDir
	case strings.HasPrefix(path, home):
		return filepath.Join(CurrentUser.HomeDir, path[len(home):])
	default:
		return path
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/security/access"
	"v.io/x/lib/gosh"
	"v.io/x/lib/set"
	"v.io/x/ref"
	"v.io/x/ref/lib/signals"
	_ "v.io/x/ref/runtime/factories/generic"
	"v.io/x/ref/services/identity/identitylib"
	"v.io/x/ref/services/mounttable/mounttablelib"
	"v.io/x/ref/services/xproxy/xproxy"
	"v.io/x/ref/test/expect"
	"v.io/x/ref/test/v23test"
)

// NOTE(sadovsky): It appears that a lot of this code has been copied from
// v.io/x/ref/cmd/servicerunner/main.go.

var (
	runTestsWatch bool
)

func init() {
	flag.BoolVar(&runTestsWatch, "runTestsWatch", false, "if true runs the tests in watch mode")
}

// TODO(sadovsky): Switch to using v23test.Shell.StartRootMountTable.
var runMT = gosh.Register("runMT", func(mp string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()

	mt, err := mounttablelib.NewMountTableDispatcher(ctx, "", "", "mounttable")
	if err != nil {
		return fmt.Errorf("mounttablelib.NewMountTableDispatcher failed: %s", err)
	}

	_, server, err := v23.WithNewDispatchingServer(ctx, mp, mt, options.ServesMountTable(true))
	if err != nil {
		return fmt.Errorf("root failed: %v", err)
	}
	fmt.Printf("PID=%d\n", os.Getpid())
	for _, ep := range server.Status().Endpoints {
		fmt.Printf("MT_NAME=%s\n", ep.Name())
	}
	<-signals.ShutdownOnSignals(ctx)
	return nil
})

// Helper function to simply print an error and then exit.
func exitOnError(err error, desc string) {
	if err != nil {
		fmt.Fprintln(os.Stderr, desc, err)
		os.Exit(1)
	}
}

// updateVars captures the vars from the given session and adds them to the
// given vars map, overwriting existing entries.
// TODO(sadovsky): Switch to gosh.SendVars/AwaitVars.
func updateVars(s *expect.Session, vars map[string]string, varNames ...string) error {
	varsToAdd := set.StringBool.FromSlice(varNames)
	numLeft := len(varsToAdd)

	for {
		l := s.ReadLine()
		if err := s.OriginalError(); err != nil {
			return err // EOF or otherwise
		}
		parts := strings.Split(l, "=")
		if len(parts) != 2 {
			return fmt.Errorf("Unexpected line: %s", l)
		}
		if varsToAdd[parts[0]] {
			numLeft--
			vars[parts[0]] = parts[1]
			if numLeft == 0 {
				break
			}
		}
	}
	return nil
}

func main() {
	gosh.InitMain()

	// If we ever get a SIGHUP (terminal closes), then end the program.
	signalChannel := make(chan os.Signal)
	signal.Notify(signalChannel, syscall.SIGHUP)
	go func() {
		sig := <-signalChannel
		switch sig {
		case syscall.SIGHUP:
			os.Exit(1)
		}
	}()

	// Try running the program; on failure, exit with error status code.
	if !run() {
		os.Exit(1)
	}
}

// Runs the services and cleans up afterwards.
// Returns true if the run was successful.
func run() bool {
	// In order to prevent conflicts, tests and webapp use different mounttable
	// ports.
	port := 8884
	cottagePort := 8885
	housePort := 8886
	host := "localhost"

	err := os.MkdirAll("tmp/runner", 0750)
	exitOnError(err, "Could not make temp directory")

	vars := map[string]string{}
	sh := v23test.NewShell(nil, v23test.Opts{ChildOutputDir: "tmp/runner"})
	defer sh.Cleanup()
	ctx := sh.Ctx

	// Run a mounttable for tests
	cRoot := sh.Fn(runMT, "root")
	cRoot.Args = append(cRoot.Args, "--v23.tcp.protocol=wsh", fmt.Sprintf("--v23.tcp.address=%s:%d", host, port))
	cRoot.Start()
	exitOnError(err, "Failed to start root mount table")
	exitOnError(updateVars(cRoot.S, vars, "MT_NAME"), "Failed to get MT_NAME")

	// Set ref.EnvNamespacePrefix env var, consumed downstream.
	sh.Vars[ref.EnvNamespacePrefix] = vars["MT_NAME"]
	v23.GetNamespace(ctx).SetRoots(vars["MT_NAME"])

	// Run the cottage mounttable at host/cottage.
	cCottage := sh.Fn(runMT, "cottage")
	cCottage.Args = append(cCottage.Args, "--v23.tcp.protocol=wsh", fmt.Sprintf("--v23.tcp.address=%s:%d", host, cottagePort))
	cCottage.Start()
	exitOnError(err, "Failed to start cottage mount table")

	// run the house mounttable at host/house.
	cHouse := sh.Fn(runMT, "house")
	cHouse.Args = append(cHouse.Args, "--v23.tcp.protocol=wsh", fmt.Sprintf("--v23.tcp.address=%s:%d", host, housePort))
	cHouse.Start()
	exitOnError(err, "Failed to start house mount table")

	// Just print out the collected variables. This is for debugging purposes.
	bytes, err := json.Marshal(vars)
	exitOnError(err, "Failed to marshal the collected variables")
	fmt.Println(string(bytes))

	// Also set HOUSE_MOUNTTABLE (used in the tests)
	os.Setenv("HOUSE_MOUNTTABLE", fmt.Sprintf("/%s:%d", host, housePort))

	lspec := v23.GetListenSpec(ctx)
	lspec.Addrs = rpc.ListenAddrs{{"wsh", ":0"}}
	ctx = v23.WithListenSpec(ctx, lspec)
	ctx, cancel := context.WithCancel(ctx)
	proxyACL := access.AccessList{In: security.DefaultBlessingPatterns(v23.GetPrincipal(ctx))}
	proxy, err := xproxy.New(ctx, "test/proxy", proxyACL)
	exitOnError(err, "Failed to start proxy")
	vars["PROXY_NAME"] = proxy.ListeningEndpoints()[0].Name()
	defer func() {
		cancel()
		<-proxy.Closed()
	}()

	cIdentityd := sh.Fn(identitylib.TestIdentityd)
	cIdentityd.Args = append(cIdentityd.Args, "--v23.tcp.protocol=wsh", "--v23.tcp.address=:0", "--http-addr=localhost:0")
	cIdentityd.Start()
	exitOnError(err, "Failed to start identityd")
	exitOnError(updateVars(cIdentityd.S, vars, "TEST_IDENTITYD_NAME", "TEST_IDENTITYD_HTTP_ADDR"), "Failed to obtain identityd address")

	// Setup a lot of environment variables; these are used for the tests and building the test extension.
	os.Setenv(ref.EnvNamespacePrefix, vars["MT_NAME"])
	os.Setenv("PROXY_ADDR", vars["PROXY_NAME"])
	os.Setenv("IDENTITYD", fmt.Sprintf("%s/google", vars["TEST_IDENTITYD_NAME"]))
	os.Setenv("IDENTITYD_BLESSING_URL", fmt.Sprintf("%s/auth/blessing-root", vars["TEST_IDENTITYD_HTTP_ADDR"]))
	os.Setenv("DEBUG", "false")

	testsOk := runProva()

	fmt.Println("Cleaning up launched services...")
	return testsOk
}

// Run the prova tests and convert its tap output to xunit.
func runProva() bool {
	// This is also useful information for routing the test output.
	JIRI_ROOT := os.Getenv("JIRI_ROOT")
	VANADIUM_JS := fmt.Sprintf("%s/release/javascript/core", JIRI_ROOT)
	VANADIUM_BROWSER := fmt.Sprintf("%s/release/projects/browser", JIRI_ROOT)

	TAP_XUNIT := fmt.Sprintf("%s/node_modules/.bin/tap-xunit", VANADIUM_BROWSER)
	XUNIT_OUTPUT_FILE := os.Getenv("XUNIT_OUTPUT_FILE")
	if XUNIT_OUTPUT_FILE == "" {
		XUNIT_OUTPUT_FILE = fmt.Sprintf("%s/test_output.xml", os.Getenv("TMPDIR"))
	}
	TAP_XUNIT_OPTIONS := " --package=namespace-browser"

	// Make sure we're in the right folder when we run make test-extension.
	vbroot, err := os.Open(VANADIUM_BROWSER)
	exitOnError(err, "Failed to open vanadium browser dir")
	err = vbroot.Chdir()
	exitOnError(err, "Failed to change to vanadium browser dir")

	// Make the test-extension, this should also remove the old one.
	fmt.Println("Rebuilding test extension...")
	cmdExtensionClean := exec.Command("rm", "-fr", fmt.Sprintf("%s/extension/build-test", VANADIUM_JS))
	err = cmdExtensionClean.Run()
	exitOnError(err, "Failed to clean test extension")
	cmdExtensionBuild := exec.Command("make", "-C", fmt.Sprintf("%s/extension", VANADIUM_JS), "build-test")
	out, err := cmdExtensionBuild.CombinedOutput()
	exitOnError(err, fmt.Sprintf("Failed to build test extension: make -C %s/extension build-test [[\n%s\n]]", VANADIUM_JS, out))

	// These are the basic prova options.
	options := []string{
		"test/**/*.js",
		"--browser",
		"--includeFilenameAsPackage",
		"--launch",
		"chrome",
		"--plugin",
		"proxyquireify/plugin",
		"--transform",
		"envify,./main-transform",
		"--log",
		"tmp/chrome.log",
		fmt.Sprintf("--options=--load-extension=%s/extension/build-test/,--ignore-certificate-errors,--enable-logging=stderr", VANADIUM_JS),
	}

	// Normal tests have a few more options and a different port from the watch tests.
	var PROVA_PORT int
	if !runTestsWatch {
		PROVA_PORT = 8893
		options = append(options, "--headless", "--quit", "--progress", "--tap")
		fmt.Printf("\033[34m-Executing tests. See %s for test xunit output.\033[0m\n", XUNIT_OUTPUT_FILE)
	} else {
		PROVA_PORT = 8894
		fmt.Println("\033[34m-Running tests in watch mode.\033[0m")
	}
	options = append(options, "--port", fmt.Sprintf("%d", PROVA_PORT))

	// This is the prova command.
	cmdProva := exec.Command(
		fmt.Sprintf("%s/node_modules/.bin/prova", VANADIUM_BROWSER),
		options...,
	)
	fmt.Printf("\033[34m-Go to \033[32mhttp://0.0.0.0:%d\033[34m to see tests running.\033[0m\n", PROVA_PORT)
	fmt.Println(cmdProva)

	// Collect the prova stdout. This information needs to be sent to xunit.
	provaOut, err := cmdProva.StdoutPipe()
	exitOnError(err, "Failed to get prova stdout pipe")

	// Setup the tap to xunit command. It uses Prova's stdout as input.
	// The output will got the xunit output file.
	cmdTap := exec.Command(TAP_XUNIT, TAP_XUNIT_OPTIONS)
	cmdTap.Stdin = io.TeeReader(provaOut, os.Stdout) // Tee the prova output to see it on the console too.
	outfile, err := os.Create(XUNIT_OUTPUT_FILE)
	exitOnError(err, "Failed to create xunit output file")
	defer outfile.Close()
	bufferedWriter := bufio.NewWriter(outfile)
	cmdTap.Stdout = bufferedWriter
	defer bufferedWriter.Flush() // Ensure that the full xunit output is written.

	// We start the tap command...
	err = cmdTap.Start()
	exitOnError(err, "Failed to start tap to xunit command")

	// Meanwhile, run Prova to completion. If there was an error, print ERROR, otherwise PASS.
	err = cmdProva.Run()
	testsOk := true
	if err != nil {
		fmt.Println(err)
		fmt.Println("\033[31m\033[1mERROR\033[0m")
		testsOk = false
	} else {
		fmt.Println("\033[32m\033[1mPASS\033[0m")
	}

	// Wait for tap to xunit to finish itself off. This file will be ready for reading by Jenkins.
	fmt.Println("Converting Tap output to XUnit")
	err = cmdTap.Wait()
	exitOnError(err, "Failed tap to xunit conversion")

	return testsOk
}
<commit_msg>browser: gosh: address TODOs<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"syscall"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/options"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/security/access"
	"v.io/x/lib/gosh"
	"v.io/x/lib/set"
	"v.io/x/ref"
	"v.io/x/ref/lib/signals"
	_ "v.io/x/ref/runtime/factories/generic"
	"v.io/x/ref/services/identity/identitylib"
	"v.io/x/ref/services/mounttable/mounttablelib"
	"v.io/x/ref/services/xproxy/xproxy"
	"v.io/x/ref/test/expect"
	"v.io/x/ref/test/v23test"
)

// NOTE(sadovsky): It appears that a lot of this code has been copied from
// v.io/x/ref/cmd/servicerunner/main.go.

var (
	runTestsWatch bool
)

func init() {
	flag.BoolVar(&runTestsWatch, "runTestsWatch", false, "if true runs the tests in watch mode")
}

// TODO(sadovsky): Switch to using v23test.Shell.StartRootMountTable.
var runMT = gosh.RegisterFunc("runMT", func(mp string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()

	mt, err := mounttablelib.NewMountTableDispatcher(ctx, "", "", "mounttable")
	if err != nil {
		return fmt.Errorf("mounttablelib.NewMountTableDispatcher failed: %s", err)
	}

	_, server, err := v23.WithNewDispatchingServer(ctx, mp, mt, options.ServesMountTable(true))
	if err != nil {
		return fmt.Errorf("root failed: %v", err)
	}
	fmt.Printf("PID=%d\n", os.Getpid())
	for _, ep := range server.Status().Endpoints {
		fmt.Printf("MT_NAME=%s\n", ep.Name())
	}
	<-signals.ShutdownOnSignals(ctx)
	return nil
})

// Helper function to simply print an error and then exit.
func exitOnError(err error, desc string) {
	if err != nil {
		fmt.Fprintln(os.Stderr, desc, err)
		os.Exit(1)
	}
}

// updateVars captures the vars from the given session and adds them to the
// given vars map, overwriting existing entries.
// TODO(sadovsky): Switch to gosh.SendVars/AwaitVars.
func updateVars(s *expect.Session, vars map[string]string, varNames ...string) error {
	varsToAdd := set.StringBool.FromSlice(varNames)
	numLeft := len(varsToAdd)

	for {
		l := s.ReadLine()
		if err := s.OriginalError(); err != nil {
			return err // EOF or otherwise
		}
		parts := strings.Split(l, "=")
		if len(parts) != 2 {
			return fmt.Errorf("Unexpected line: %s", l)
		}
		if varsToAdd[parts[0]] {
			numLeft--
			vars[parts[0]] = parts[1]
			if numLeft == 0 {
				break
			}
		}
	}
	return nil
}

func main() {
	gosh.InitMain()

	// If we ever get a SIGHUP (terminal closes), then end the program.
	signalChannel := make(chan os.Signal)
	signal.Notify(signalChannel, syscall.SIGHUP)
	go func() {
		sig := <-signalChannel
		switch sig {
		case syscall.SIGHUP:
			os.Exit(1)
		}
	}()

	// Try running the program; on failure, exit with error status code.
	if !run() {
		os.Exit(1)
	}
}

// Runs the services and cleans up afterwards.
// Returns true if the run was successful.
func run() bool {
	// In order to prevent conflicts, tests and webapp use different mounttable
	// ports.
	port := 8884
	cottagePort := 8885
	housePort := 8886
	host := "localhost"

	err := os.MkdirAll("tmp/runner", 0750)
	exitOnError(err, "Could not make temp directory")

	vars := map[string]string{}
	sh := v23test.NewShell(nil, v23test.Opts{ChildOutputDir: "tmp/runner"})
	defer sh.Cleanup()
	ctx := sh.Ctx

	// Run a mounttable for tests
	cRoot := sh.FuncCmd(runMT, "root")
	cRoot.Args = append(cRoot.Args, "--v23.tcp.protocol=wsh", fmt.Sprintf("--v23.tcp.address=%s:%d", host, port))
	cRoot.Start()
	exitOnError(err, "Failed to start root mount table")
	exitOnError(updateVars(cRoot.S, vars, "MT_NAME"), "Failed to get MT_NAME")

	// Set ref.EnvNamespacePrefix env var, consumed downstream.
	sh.Vars[ref.EnvNamespacePrefix] = vars["MT_NAME"]
	v23.GetNamespace(ctx).SetRoots(vars["MT_NAME"])

	// Run the cottage mounttable at host/cottage.
	cCottage := sh.FuncCmd(runMT, "cottage")
	cCottage.Args = append(cCottage.Args, "--v23.tcp.protocol=wsh", fmt.Sprintf("--v23.tcp.address=%s:%d", host, cottagePort))
	cCottage.Start()
	exitOnError(err, "Failed to start cottage mount table")

	// run the house mounttable at host/house.
	cHouse := sh.FuncCmd(runMT, "house")
	cHouse.Args = append(cHouse.Args, "--v23.tcp.protocol=wsh", fmt.Sprintf("--v23.tcp.address=%s:%d", host, housePort))
	cHouse.Start()
	exitOnError(err, "Failed to start house mount table")

	// Just print out the collected variables. This is for debugging purposes.
	bytes, err := json.Marshal(vars)
	exitOnError(err, "Failed to marshal the collected variables")
	fmt.Println(string(bytes))

	// Also set HOUSE_MOUNTTABLE (used in the tests)
	os.Setenv("HOUSE_MOUNTTABLE", fmt.Sprintf("/%s:%d", host, housePort))

	lspec := v23.GetListenSpec(ctx)
	lspec.Addrs = rpc.ListenAddrs{{"wsh", ":0"}}
	ctx = v23.WithListenSpec(ctx, lspec)
	ctx, cancel := context.WithCancel(ctx)
	proxyACL := access.AccessList{In: security.DefaultBlessingPatterns(v23.GetPrincipal(ctx))}
	proxy, err := xproxy.New(ctx, "test/proxy", proxyACL)
	exitOnError(err, "Failed to start proxy")
	vars["PROXY_NAME"] = proxy.ListeningEndpoints()[0].Name()
	defer func() {
		cancel()
		<-proxy.Closed()
	}()

	cIdentityd := sh.FuncCmd(identitylib.TestIdentityd)
	cIdentityd.Args = append(cIdentityd.Args, "--v23.tcp.protocol=wsh", "--v23.tcp.address=:0", "--http-addr=localhost:0")
	cIdentityd.Start()
	exitOnError(err, "Failed to start identityd")
	exitOnError(updateVars(cIdentityd.S, vars, "TEST_IDENTITYD_NAME", "TEST_IDENTITYD_HTTP_ADDR"), "Failed to obtain identityd address")

	// Setup a lot of environment variables; these are used for the tests and building the test extension.
	os.Setenv(ref.EnvNamespacePrefix, vars["MT_NAME"])
	os.Setenv("PROXY_ADDR", vars["PROXY_NAME"])
	os.Setenv("IDENTITYD", fmt.Sprintf("%s/google", vars["TEST_IDENTITYD_NAME"]))
	os.Setenv("IDENTITYD_BLESSING_URL", fmt.Sprintf("%s/auth/blessing-root", vars["TEST_IDENTITYD_HTTP_ADDR"]))
	os.Setenv("DEBUG", "false")

	testsOk := runProva()

	fmt.Println("Cleaning up launched services...")
	return testsOk
}

// Run the prova tests and convert its tap output to xunit.
func runProva() bool {
	// This is also useful information for routing the test output.
	JIRI_ROOT := os.Getenv("JIRI_ROOT")
	VANADIUM_JS := fmt.Sprintf("%s/release/javascript/core", JIRI_ROOT)
	VANADIUM_BROWSER := fmt.Sprintf("%s/release/projects/browser", JIRI_ROOT)

	TAP_XUNIT := fmt.Sprintf("%s/node_modules/.bin/tap-xunit", VANADIUM_BROWSER)
	XUNIT_OUTPUT_FILE := os.Getenv("XUNIT_OUTPUT_FILE")
	if XUNIT_OUTPUT_FILE == "" {
		XUNIT_OUTPUT_FILE = fmt.Sprintf("%s/test_output.xml", os.Getenv("TMPDIR"))
	}
	TAP_XUNIT_OPTIONS := " --package=namespace-browser"

	// Make sure we're in the right folder when we run make test-extension.
	vbroot, err := os.Open(VANADIUM_BROWSER)
	exitOnError(err, "Failed to open vanadium browser dir")
	err = vbroot.Chdir()
	exitOnError(err, "Failed to change to vanadium browser dir")

	// Make the test-extension, this should also remove the old one.
	fmt.Println("Rebuilding test extension...")
	cmdExtensionClean := exec.Command("rm", "-fr", fmt.Sprintf("%s/extension/build-test", VANADIUM_JS))
	err = cmdExtensionClean.Run()
	exitOnError(err, "Failed to clean test extension")
	cmdExtensionBuild := exec.Command("make", "-C", fmt.Sprintf("%s/extension", VANADIUM_JS), "build-test")
	out, err := cmdExtensionBuild.CombinedOutput()
	exitOnError(err, fmt.Sprintf("Failed to build test extension: make -C %s/extension build-test [[\n%s\n]]", VANADIUM_JS, out))

	// These are the basic prova options.
	options := []string{
		"test/**/*.js",
		"--browser",
		"--includeFilenameAsPackage",
		"--launch",
		"chrome",
		"--plugin",
		"proxyquireify/plugin",
		"--transform",
		"envify,./main-transform",
		"--log",
		"tmp/chrome.log",
		fmt.Sprintf("--options=--load-extension=%s/extension/build-test/,--ignore-certificate-errors,--enable-logging=stderr", VANADIUM_JS),
	}

	// Normal tests have a few more options and a different port from the watch tests.
	var PROVA_PORT int
	if !runTestsWatch {
		PROVA_PORT = 8893
		options = append(options, "--headless", "--quit", "--progress", "--tap")
		fmt.Printf("\033[34m-Executing tests. See %s for test xunit output.\033[0m\n", XUNIT_OUTPUT_FILE)
	} else {
		PROVA_PORT = 8894
		fmt.Println("\033[34m-Running tests in watch mode.\033[0m")
	}
	options = append(options, "--port", fmt.Sprintf("%d", PROVA_PORT))

	// This is the prova command.
	cmdProva := exec.Command(
		fmt.Sprintf("%s/node_modules/.bin/prova", VANADIUM_BROWSER),
		options...,
	)
	fmt.Printf("\033[34m-Go to \033[32mhttp://0.0.0.0:%d\033[34m to see tests running.\033[0m\n", PROVA_PORT)
	fmt.Println(cmdProva)

	// Collect the prova stdout. This information needs to be sent to xunit.
	provaOut, err := cmdProva.StdoutPipe()
	exitOnError(err, "Failed to get prova stdout pipe")

	// Setup the tap to xunit command. It uses Prova's stdout as input.
	// The output will got the xunit output file.
	cmdTap := exec.Command(TAP_XUNIT, TAP_XUNIT_OPTIONS)
	cmdTap.Stdin = io.TeeReader(provaOut, os.Stdout) // Tee the prova output to see it on the console too.
	outfile, err := os.Create(XUNIT_OUTPUT_FILE)
	exitOnError(err, "Failed to create xunit output file")
	defer outfile.Close()
	bufferedWriter := bufio.NewWriter(outfile)
	cmdTap.Stdout = bufferedWriter
	defer bufferedWriter.Flush() // Ensure that the full xunit output is written.

	// We start the tap command...
	err = cmdTap.Start()
	exitOnError(err, "Failed to start tap to xunit command")

	// Meanwhile, run Prova to completion. If there was an error, print ERROR, otherwise PASS.
	err = cmdProva.Run()
	testsOk := true
	if err != nil {
		fmt.Println(err)
		fmt.Println("\033[31m\033[1mERROR\033[0m")
		testsOk = false
	} else {
		fmt.Println("\033[32m\033[1mPASS\033[0m")
	}

	// Wait for tap to xunit to finish itself off. This file will be ready for reading by Jenkins.
	fmt.Println("Converting Tap output to XUnit")
	err = cmdTap.Wait()
	exitOnError(err, "Failed tap to xunit conversion")

	return testsOk
}
<|endoftext|>


<commit_before>//TODO:
package main

import (
	"fmt"
	"github.com/dgrijalva/jwt-go"
	"github.com/dgrijalva/jwt-go/request"
	"net/http"
	"time"

	"github.com/justinas/alice"
	"net/url"
)

var mySigningKey = []byte("secret")

var middleware = alice.New(SetHeader, ValidateToken)

func LoginHandler(w http.ResponseWriter, r *http.Request) {

	//TODO: 1: verify the login

	//TODO: 2: if the login failed, exit

	//3: create and return token
	token := jwt.New(jwt.SigningMethodHS256)
	claims := token.Claims.(jwt.MapClaims)
	claims["admin"] = true
	//claims["name"] = "??"
	claims["exp"] = time.Now().Add(time.Minute * 10).Unix()

	/* Sign the token with our secret */
	tokenString, _ := token.SignedString(mySigningKey)

	//TODO return token in body or header? or in other mode so that the subsequent requests from the same browser will save the token
	//w.Write([]byte(tokenString))
	//w.Header().Add("Authorization","Bearer "+tokenString)

	values := url.Values{}
	values.Add("token", tokenString)

	redirectRequestUrl := fmt.Sprintf("localhost:8080/hello?%s",
		values.Encode())

	http.Redirect(w, r, redirectRequestUrl, http.StatusOK)
	//w.Write([]byte("authorized"))
}

func SetHeader(next http.Handler) http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Println("### SetHeader begin", r.URL)
		//TODO: is this the only way to read the query parameters?
		vars, err := url.ParseQuery(r.URL.RawQuery)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}

		if val, exists := vars["token"]; exists {
			r.Header.Add("Authorization","Bearer "+val[0])
		}
		next.ServeHTTP(w, r)
		fmt.Println("### SetHeader end")
	})
}

//func ValidateTokenMiddleware(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
func ValidateToken(next http.Handler) http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Println("### ValidateToken begin", r.URL)
		token, err := request.ParseFromRequest(r, request.AuthorizationHeaderExtractor,
			func(token *jwt.Token) (interface{}, error) {
				return mySigningKey, nil
			})
		if err != nil {
			fmt.Println("### ValidateToken error: ", err)
			//w.WriteHeader(http.StatusUnauthorized)
			//fmt.Fprint(w, "\nUnauthorized access to this resource\n"+err.Error())
			http.Redirect(w, r, "localhost:8080/login", http.StatusUnauthorized)
			return
		}
		if !token.Valid {
			w.WriteHeader(http.StatusUnauthorized)
			fmt.Println("### ValidateToken error: Token is not valid")
			http.Redirect(w, r, "localhost:8080/login", http.StatusUnauthorized)
			return
		}
		next.ServeHTTP(w, r)

		fmt.Println("### ValidateToken end")
	})
}

func welcomeHandler(w http.ResponseWriter, r *http.Request) {

	fmt.Println("### welcomeHandler begin")
	w.Write([]byte("hello world"))
	fmt.Println("### welcomeHandler end")
}

func main() {
	http.Handle("/hello", middleware.ThenFunc(welcomeHandler))
	http.HandleFunc("/login", LoginHandler)

	http.ListenAndServe(":8080", nil)
}
<commit_msg>JWT example : fixed the statuscode for redirect<commit_after>//TODO:
package main

import (
	"fmt"
	"github.com/dgrijalva/jwt-go"
	"github.com/dgrijalva/jwt-go/request"
	"net/http"
	"time"

	"github.com/justinas/alice"
	"net/url"
)

var mySigningKey = []byte("secret")

var middleware = alice.New(SetHeader, ValidateToken)

func LoginHandler(w http.ResponseWriter, r *http.Request) {

	//TODO: 1: verify the login

	//TODO: 2: if the login failed, exit

	//3: create and return token
	token := jwt.New(jwt.SigningMethodHS256)
	claims := token.Claims.(jwt.MapClaims)
	claims["admin"] = true
	//claims["name"] = "??"
	claims["exp"] = time.Now().Add(time.Minute * 10).Unix()

	/* Sign the token with our secret */
	tokenString, _ := token.SignedString(mySigningKey)

	//TODO return token in body or header? or in other mode so that the subsequent requests from the same browser will save the token
	//w.Write([]byte(tokenString))
	//w.Header().Add("Authorization","Bearer "+tokenString)

	values := url.Values{}
	values.Add("token", tokenString)

	redirectRequestUrl := fmt.Sprintf("localhost:8080/hello?%s",
		values.Encode())

	http.Redirect(w, r, redirectRequestUrl, 302)
	//w.Write([]byte("authorized"))
}

func SetHeader(next http.Handler) http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Println("### SetHeader begin", r.URL)
		//TODO: is this the only way to read the query parameters? or r.FormValue("token")
		vars, err := url.ParseQuery(r.URL.RawQuery)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		if val, exists := vars["token"]; exists {
			r.Header.Add("Authorization","Bearer "+val[0])
		}

		next.ServeHTTP(w, r)
		fmt.Println("### SetHeader end")
	})
}

//func ValidateTokenMiddleware(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
func ValidateToken(next http.Handler) http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Println("### ValidateToken begin", r.URL)
		token, err := request.ParseFromRequest(r, request.AuthorizationHeaderExtractor,
			func(token *jwt.Token) (interface{}, error) {
				return mySigningKey, nil
			})
		if err != nil {
			fmt.Println("### ValidateToken error: ", err)
			//w.WriteHeader(http.StatusUnauthorized)
			//fmt.Fprint(w, "\nUnauthorized access to this resource\n"+err.Error())
			http.Redirect(w, r, "localhost:8080/login", 302)
			return
		}
		if !token.Valid {
			w.WriteHeader(http.StatusUnauthorized)
			fmt.Println("### ValidateToken error: Token is not valid")
			http.Redirect(w, r, "localhost:8080/login", 302)
			return
		}
		next.ServeHTTP(w, r)

		fmt.Println("### ValidateToken end")
	})
}

func welcomeHandler(w http.ResponseWriter, r *http.Request) {

	fmt.Println("### welcomeHandler begin")
	w.Write([]byte("hello world"))
	fmt.Println("### welcomeHandler end")
}

func main() {
	http.Handle("/hello", middleware.ThenFunc(welcomeHandler))
	http.HandleFunc("/login", LoginHandler)

	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package cmd

import (
	"bufio"
	"fmt"
	"github.com/qiniu/api.v7/cdn"
	"github.com/qiniu/qshell/iqshell"
	"github.com/spf13/cobra"
	"io"
	"os"
	"strings"
)

const (
	// CDN刷新一次性最大的刷新文件列表
	BATCH_CDN_REFRESH_URLS_ALLOW_MAX = 100

	// CDN目录刷新一次性最大的刷新目录数
	BATCH_CDN_REFRESH_DIRS_ALLOW_MAX = 10

	// 预取一次最大的预取数目
	BATCH_CDN_PREFETCH_ALLOW_MAX = 100
)

var (
	prefetchFile string
	isDir        bool
)

var cdnPreCmd = &cobra.Command{
	Use:   "cdnprefetch [-i <UrlListFile>]",
	Short: "Batch prefetch the urls in the url list file",
	Long:  "Batch prefetch the urls in the url list file or from stdin if UrlListFile not specified",
	Args:  cobra.ExactArgs(0),
	Run:   CdnPrefetch,
}

var cdnRefreshCmd = &cobra.Command{
	Use:   "cdnrefresh [-i <UrlListFile>]",
	Short: "Batch refresh the cdn cache by the url list file",
	Long:  "Batch refresh the cdn cache by the url list file or from stdin if UrlListFile not specified",
	Args:  cobra.ExactArgs(0),
	Run:   CdnRefresh,
}

func init() {
	cdnRefreshCmd.Flags().BoolVarP(&isDir, "dirs", "r", false, "refresh directory")
	cdnRefreshCmd.Flags().StringVarP(&prefetchFile, "input-file", "i", "", "input file")
	cdnPreCmd.Flags().StringVarP(&prefetchFile, "input-file", "i", "", "input file")

	RootCmd.AddCommand(cdnPreCmd, cdnRefreshCmd)
}

// 【cdnrefresh】刷新所有CDN节点
func CdnRefresh(cmd *cobra.Command, params []string) {
	var urlListFile string

	if prefetchFile != "" {
		urlListFile = prefetchFile
	} else {
		urlListFile = "stdin"
	}

	var fp io.ReadCloser
	var err error

	if urlListFile == "stdin" {
		fp = os.Stdin
	} else {
		fp, err = os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open refresh item list file error,", err)
			os.Exit(iqshell.STATUS_HALT)
		}
		defer fp.Close()
	}
	cm := iqshell.GetCdnManager()
	scanner := bufio.NewScanner(fp)

	itemsToRefresh := make([]string, 0, 100)

	if isDir {
		for scanner.Scan() {
			item := strings.TrimSpace(scanner.Text())
			if item == "" {
				continue
			}
			itemsToRefresh = append(itemsToRefresh, item)

			if len(itemsToRefresh) == BATCH_CDN_REFRESH_DIRS_ALLOW_MAX {
				cdnRefresh(cm, nil, itemsToRefresh)
				itemsToRefresh = make([]string, 0, 10)
			}
		}
	} else {
		for scanner.Scan() {
			item := strings.TrimSpace(scanner.Text())
			if item == "" {
				continue
			}
			itemsToRefresh = append(itemsToRefresh, item)

			if len(itemsToRefresh) == BATCH_CDN_REFRESH_URLS_ALLOW_MAX {
				cdnRefresh(cm, itemsToRefresh, nil)
				itemsToRefresh = make([]string, 0, 100)
			}
		}
	}

	//check final items
	if len(itemsToRefresh) > 0 {
		if isDir {
			cdnRefresh(cm, nil, itemsToRefresh)
		} else {
			cdnRefresh(cm, itemsToRefresh, nil)
		}
	}
}

func cdnRefresh(cm *cdn.CdnManager, urls []string, dirs []string) {
	resp, err := cm.RefreshUrlsAndDirs(urls, dirs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "CDN refresh error: %v\n", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}

//  【cdnprefetch】CDN 文件预取
func CdnPrefetch(cmd *cobra.Command, params []string) {
	var urlListFile string

	if prefetchFile != "" {
		urlListFile = prefetchFile
	} else {
		urlListFile = "stdin"
	}

	var fp io.ReadCloser
	var err error

	if urlListFile == "stdin" {
		fp = os.Stdin
	} else {
		fp, err = os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open url list file error,", err)
			os.Exit(iqshell.STATUS_HALT)
		}
		defer fp.Close()
	}
	cm := iqshell.GetCdnManager()
	scanner := bufio.NewScanner(fp)

	urlsToPrefetch := make([]string, 0, 10)
	for scanner.Scan() {
		url := strings.TrimSpace(scanner.Text())
		if url == "" {
			continue
		}
		urlsToPrefetch = append(urlsToPrefetch, url)

		if len(urlsToPrefetch) == BATCH_CDN_PREFETCH_ALLOW_MAX {
			cdnPrefetch(cm, urlsToPrefetch)
			urlsToPrefetch = make([]string, 0, 10)
		}
	}

	if len(urlsToPrefetch) > 0 {
		cdnPrefetch(cm, urlsToPrefetch)
	}
}

func cdnPrefetch(cm *cdn.CdnManager, urls []string) {
	resp, err := cm.PrefetchUrls(urls)
	if err != nil {
		fmt.Fprintf(os.Stderr, "CDN prefetch error: %v\n", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}
<commit_msg>Update cdn.go<commit_after>package cmd

import (
	"bufio"
	"fmt"
	"github.com/qiniu/api.v7/cdn"
	"github.com/qiniu/qshell/iqshell"
	"github.com/spf13/cobra"
	"io"
	"os"
	"strings"
)

const (
	// CDN刷新一次性最大的刷新文件列表
	BATCH_CDN_REFRESH_URLS_ALLOW_MAX = 20

	// CDN目录刷新一次性最大的刷新目录数
	BATCH_CDN_REFRESH_DIRS_ALLOW_MAX = 10

	// 预取一次最大的预取数目
	BATCH_CDN_PREFETCH_ALLOW_MAX = 20
)

var (
	prefetchFile string
	isDir        bool
)

var cdnPreCmd = &cobra.Command{
	Use:   "cdnprefetch [-i <UrlListFile>]",
	Short: "Batch prefetch the urls in the url list file",
	Long:  "Batch prefetch the urls in the url list file or from stdin if UrlListFile not specified",
	Args:  cobra.ExactArgs(0),
	Run:   CdnPrefetch,
}

var cdnRefreshCmd = &cobra.Command{
	Use:   "cdnrefresh [-i <UrlListFile>]",
	Short: "Batch refresh the cdn cache by the url list file",
	Long:  "Batch refresh the cdn cache by the url list file or from stdin if UrlListFile not specified",
	Args:  cobra.ExactArgs(0),
	Run:   CdnRefresh,
}

func init() {
	cdnRefreshCmd.Flags().BoolVarP(&isDir, "dirs", "r", false, "refresh directory")
	cdnRefreshCmd.Flags().StringVarP(&prefetchFile, "input-file", "i", "", "input file")
	cdnPreCmd.Flags().StringVarP(&prefetchFile, "input-file", "i", "", "input file")

	RootCmd.AddCommand(cdnPreCmd, cdnRefreshCmd)
}

// 【cdnrefresh】刷新所有CDN节点
func CdnRefresh(cmd *cobra.Command, params []string) {
	var urlListFile string

	if prefetchFile != "" {
		urlListFile = prefetchFile
	} else {
		urlListFile = "stdin"
	}

	var fp io.ReadCloser
	var err error

	if urlListFile == "stdin" {
		fp = os.Stdin
	} else {
		fp, err = os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open refresh item list file error,", err)
			os.Exit(iqshell.STATUS_HALT)
		}
		defer fp.Close()
	}
	cm := iqshell.GetCdnManager()
	scanner := bufio.NewScanner(fp)

	itemsToRefresh := make([]string, 0, 100)

	if isDir {
		for scanner.Scan() {
			item := strings.TrimSpace(scanner.Text())
			if item == "" {
				continue
			}
			itemsToRefresh = append(itemsToRefresh, item)

			if len(itemsToRefresh) == BATCH_CDN_REFRESH_DIRS_ALLOW_MAX {
				cdnRefresh(cm, nil, itemsToRefresh)
				itemsToRefresh = make([]string, 0, 10)
			}
		}
	} else {
		for scanner.Scan() {
			item := strings.TrimSpace(scanner.Text())
			if item == "" {
				continue
			}
			itemsToRefresh = append(itemsToRefresh, item)

			if len(itemsToRefresh) == BATCH_CDN_REFRESH_URLS_ALLOW_MAX {
				cdnRefresh(cm, itemsToRefresh, nil)
				itemsToRefresh = make([]string, 0, 100)
			}
		}
	}

	//check final items
	if len(itemsToRefresh) > 0 {
		if isDir {
			cdnRefresh(cm, nil, itemsToRefresh)
		} else {
			cdnRefresh(cm, itemsToRefresh, nil)
		}
	}
}

func cdnRefresh(cm *cdn.CdnManager, urls []string, dirs []string) {
	resp, err := cm.RefreshUrlsAndDirs(urls, dirs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "CDN refresh error: %v\n", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}

//  【cdnprefetch】CDN 文件预取
func CdnPrefetch(cmd *cobra.Command, params []string) {
	var urlListFile string

	if prefetchFile != "" {
		urlListFile = prefetchFile
	} else {
		urlListFile = "stdin"
	}

	var fp io.ReadCloser
	var err error

	if urlListFile == "stdin" {
		fp = os.Stdin
	} else {
		fp, err = os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open url list file error,", err)
			os.Exit(iqshell.STATUS_HALT)
		}
		defer fp.Close()
	}
	cm := iqshell.GetCdnManager()
	scanner := bufio.NewScanner(fp)

	urlsToPrefetch := make([]string, 0, 10)
	for scanner.Scan() {
		url := strings.TrimSpace(scanner.Text())
		if url == "" {
			continue
		}
		urlsToPrefetch = append(urlsToPrefetch, url)

		if len(urlsToPrefetch) == BATCH_CDN_PREFETCH_ALLOW_MAX {
			cdnPrefetch(cm, urlsToPrefetch)
			urlsToPrefetch = make([]string, 0, 10)
		}
	}

	if len(urlsToPrefetch) > 0 {
		cdnPrefetch(cm, urlsToPrefetch)
	}
}

func cdnPrefetch(cm *cdn.CdnManager, urls []string) {
	resp, err := cm.PrefetchUrls(urls)
	if err != nil {
		fmt.Fprintf(os.Stderr, "CDN prefetch error: %v\n", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 the lostromos Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"go.uber.org/zap"
)

var (
	cfgFile string
	logger  *zap.SugaredLogger
)

// LostromosCmd represents the base command when called without any subcommands
var LostromosCmd = &cobra.Command{
	Use:   "lostromos",
	Short: "Create K8s resources from Custom Resources",
	Long: `Lostromos will monitor all the resources created in your K8s CRD
and create, update, and delete resources based on the templates provided.`,
}

// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the LostromosCmd.
func Execute() {
	// Before we exit ensure we sync all logs out
	defer func() {
		if logger == nil {
			return
		}
		if err := logger.Sync(); err != nil {
			logger.Errorw("failed to sync logger", "error", err)
		}
	}()
	if err := LostromosCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	LostromosCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is /etc/lostromos.yaml)")
	LostromosCmd.PersistentFlags().BoolP("debug", "", false, "enable debug logging")
	LostromosCmd.PersistentFlags().BoolP("pretty", "", false, "print logs in human readable form instead of json")

	viperBindFlag("logging.debug", LostromosCmd.PersistentFlags().Lookup("debug"))
	viperBindFlag("logging.pretty", LostromosCmd.PersistentFlags().Lookup("pretty"))
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Use default config file /etc/lostromos.yaml
		viper.AddConfigPath("/etc")
		viper.SetConfigName("lostromos.yaml")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	err := viper.ReadInConfig()

	// Setup logging after all config options are read in
	setupLogging()
	if err != nil {
		logger.Debugw("failed to read config file", "error", err)
	}
	logger.Infow("loading config...", "configFile", viper.ConfigFileUsed())
}

func viperBindFlag(name string, flag *pflag.Flag) {
	err := viper.BindPFlag(name, flag)
	if err != nil {
		panic(err)
	}
}

func setupLogging() {
	cfg := zap.NewProductionConfig()
	if viper.GetBool("logging.pretty") {
		cfg = zap.NewDevelopmentConfig()
	}
	if viper.GetBool("logging.debug") {
		cfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
	} else {
		cfg.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
	}
	l, err := cfg.Build()
	if err != nil {
		panic(err)
	}
	logger = l.Sugar()
}
<commit_msg>remove extension in SetConfigName()<commit_after>// Copyright 2017 the lostromos Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"go.uber.org/zap"
)

var (
	cfgFile string
	logger  *zap.SugaredLogger
)

// LostromosCmd represents the base command when called without any subcommands
var LostromosCmd = &cobra.Command{
	Use:   "lostromos",
	Short: "Create K8s resources from Custom Resources",
	Long: `Lostromos will monitor all the resources created in your K8s CRD
and create, update, and delete resources based on the templates provided.`,
}

// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the LostromosCmd.
func Execute() {
	// Before we exit ensure we sync all logs out
	defer func() {
		if logger == nil {
			return
		}
		if err := logger.Sync(); err != nil {
			logger.Errorw("failed to sync logger", "error", err)
		}
	}()
	if err := LostromosCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	LostromosCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is /etc/lostromos.yaml)")
	LostromosCmd.PersistentFlags().BoolP("debug", "", false, "enable debug logging")
	LostromosCmd.PersistentFlags().BoolP("pretty", "", false, "print logs in human readable form instead of json")

	viperBindFlag("logging.debug", LostromosCmd.PersistentFlags().Lookup("debug"))
	viperBindFlag("logging.pretty", LostromosCmd.PersistentFlags().Lookup("pretty"))
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Use default config file /etc/lostromos.yaml
		viper.AddConfigPath("/etc")
		viper.SetConfigName("lostromos")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	err := viper.ReadInConfig()

	// Setup logging after all config options are read in
	setupLogging()
	if err != nil {
		logger.Debugw("failed to read config file", "error", err)
	}
	logger.Infow("loading config...", "configFile", viper.ConfigFileUsed())
}

func viperBindFlag(name string, flag *pflag.Flag) {
	err := viper.BindPFlag(name, flag)
	if err != nil {
		panic(err)
	}
}

func setupLogging() {
	cfg := zap.NewProductionConfig()
	if viper.GetBool("logging.pretty") {
		cfg = zap.NewDevelopmentConfig()
	}
	if viper.GetBool("logging.debug") {
		cfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
	} else {
		cfg.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
	}
	l, err := cfg.Build()
	if err != nil {
		panic(err)
	}
	logger = l.Sugar()
}
<|endoftext|>