<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"fmt"
	"log"
	"net/http"
)

// The HTTP request methods defined by RFC.
const (
	MethodConnect = "CONNECT" // RFC 7231, 4.3.6
	MethodDelete  = "DELETE"  // RFC 7231, 4.3.5
	MethodGet     = "GET"     // RFC 7231, 4.3.1
	MethodHead    = "HEAD"    // RFC 7231, 4.3.2
	MethodOptions = "OPTIONS" // RFC 7231, 4.3.7
	MethodPatch   = "PATCH"   // RFC 5789
	MethodPost    = "POST"    // RFC 7231, 4.3.3
	MethodPut     = "PUT"     // RFC 7231, 4.3.4
	MethodTrace   = "TRACE"   // RFC 7231, 4.3.8
)

// ServeMux is an HTTP request multiplexer. It matches the URL of each incoming
// request against a list of registered patterns and calls the handler for
// the pattern that most closely matches the URL.
//
// Patterns names are fixed, rooted paths, like "/favicon.ico", or rooted
// subtrees like "/images/" (note the trailing slash). Longer patterns take
// precedence over shorter ones, so that if there are handlers registered for
// both "/images/" and "/images/thumbnails/", the latter handler will be called
// for paths beginning "/images/thumbnails/" and the former will receive
// requests for any other paths in the "/images/" subtree.
//
// Note that since a pattern ending in a slash names a rooted subtree, the
// pattern "/" matches all paths not matched by other registered patterns,
// not just the URL with Path == "/".
//
// If a subtree has been registered and a request is received naming the subtree
// root without its trailing slash, ServeMux redirects that request to
// the subtree root (adding the trailing slash). This behavior can be overridden
// with a separate registration for the path without the trailing slash. For
// example, registering "/images/" causes ServeMux to redirect a request for
// "/images" to "/images/", unless "/images" has been registered separately.
//
// Patterns may optionally begin with a host name, restricting matches to URLs
// on that host only.  Host-specific patterns take precedence over general
// patterns, so that a handler might register for the two patterns "/codesearch"
// and "codesearch.google.com/" without also taking over requests for
// "http://www.google.com/".
//
// ServeMux also takes care of sanitizing the URL request path and the Host
// header, stripping the port number and redirecting any request containing . or
// .. elements or repeated slashes to an equivalent, cleaner URL.
//
// Multiple handlers can be registered for a single pattern, as long as they
// handle different HTTP methods.
type ServeMux struct {
	mux      *http.ServeMux
	handlers map[string]*registeredHandler

	dispatcher              Dispatcher
	interceptors            []Interceptor
	methodNotAllowedHandler handlerRegistration
}

// ServeHTTP dispatches the request to the handler whose method matches the
// incoming request and whose pattern most closely matches the request URL.
//
//  For each incoming request:
//  - [Before Phase] Interceptor.Before methods are called for every installed
//    interceptor, until an interceptor writes to a ResponseWriter (including
//    errors) or panics,
//  - the handler is called after a [Before Phase] if no writes or panics occured,
//  - the handler triggers the [Commit Phase] by writing to the ResponseWriter,
//  - [Commit Phase] Interceptor.Commit methods run for every interceptor whose
//    Before method was called,
//  - [Dispatcher Phase] after the [Commit Phase], the Dispatcher's appropriate
//    write method is called; the Dispatcher is responsible for determining whether
//    the response is indeed safe and writing it,
//  - if the handler attempts to write more than once, it is treated as an
//    unrecoverable error; the request processing ends abrubptly with a panic and
//    nothing else happens (note: this will change as soon as [After Phase] is
//    introduced)
//
// Interceptors should NOT rely on the order they're run.
func (m *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.mux.ServeHTTP(w, r)
}

func (m *ServeMux) Handle(pattern string, method string, h Handler, cfgs ...InterceptorConfig) {
	methodNotAllowed := handlerConfig{
		Dispatcher:   m.dispatcher,
		Handler:      m.methodNotAllowedHandler.handler,
		Interceptors: configureInterceptors(m.interceptors, m.methodNotAllowedHandler.cfgs),
	}

	if m.handlers[pattern] == nil {
		m.handlers[pattern] = &registeredHandler{
			pattern:          pattern,
			methodNotAllowed: methodNotAllowed,
			methods:          make(map[string]handlerConfig),
		}
		m.mux.Handle(pattern, m.handlers[pattern])
	}
	m.handlers[pattern].handleMethod(method,
		handlerConfig{
			Dispatcher:   m.dispatcher,
			Handler:      h,
			Interceptors: configureInterceptors(m.interceptors, cfgs),
		})
}

// ServeMuxConfig is a builder for ServeMux.
type ServeMuxConfig struct {
	dispatcher Dispatcher
	handlers   []struct {
		pattern, method string
		h               Handler
		cfgs            []InterceptorConfig
	}
	interceptors            []Interceptor
	methodNotAllowedHandler handlerRegistration
}

// NewServeMuxConfig crates a ServeMuxConfig with the provided Dispatcher. If
// the provided Dispatcher is nil, the DefaultDispatcher is used.
func NewServeMuxConfig(disp Dispatcher) *ServeMuxConfig {
	if disp == nil {
		disp = &DefaultDispatcher{}
	}
	return &ServeMuxConfig{
		dispatcher: disp,
		methodNotAllowedHandler: handlerRegistration{
			handler: HandlerFunc(defaultMethotNotAllowed),
		},
	}
}

type handlerRegistration struct {
	pattern string
	method  string
	handler Handler
	cfgs    []InterceptorConfig
}

// Handle registers a handler for the given pattern and method. If a handler is
// registered twice for the same pattern and method, Build will panic.
//
// InterceptorConfigs can be passed in order to modify the behavior of the
// interceptors on a registered handler. Passing an InterceptorConfig whose
// corresponding Interceptor was not installed will produce no effect. If
// multiple configurations are passed for the same Interceptor, Mux will panic.
func (s *ServeMuxConfig) Handle(pattern string, method string, h Handler, cfgs ...InterceptorConfig) {
	s.handlers = append(s.handlers, struct {
		pattern string
		method  string
		h       Handler
		cfgs    []InterceptorConfig
	}{
		pattern: pattern,
		method:  method,
		h:       h,
		cfgs:    cfgs,
	})
}

// HandleMethodNotAllowed registers a handler that runs when a given method is
// not allowed for a registered path.
func (s *ServeMuxConfig) HandleMethodNotAllowed(h Handler, cfgs ...InterceptorConfig) {
	s.methodNotAllowedHandler = handlerRegistration{
		handler: h,
		cfgs:    cfgs,
	}
}

func defaultMethotNotAllowed(w ResponseWriter, req *IncomingRequest) Result {
	return w.WriteError(StatusMethodNotAllowed)
}

// Intercept installs the given interceptors.
//
// Interceptors order is respected and interceptors are always run in the
// order they've been installed.
//
// Calling Intercept multiple times is valid. Interceptors that are added last
// will run last.
func (s *ServeMuxConfig) Intercept(is ...Interceptor) {
	s.interceptors = append(s.interceptors, is...)
}

// Mux returns the ServeMux with a copy of the current configuration.
func (s *ServeMuxConfig) Mux() *ServeMux {
	freezeLocalDev = true
	if IsLocalDev() {
		log.Println("Warning: creating safehttp.Mux in dev mode. This configuration is not valid for production use")
	}

	if s.dispatcher == nil {
		panic("Use NewServeMuxConfig instead of creating ServeMuxConfig using a composite literal.")
	}

	m := &ServeMux{
		mux:                     http.NewServeMux(),
		handlers:                make(map[string]*registeredHandler),
		dispatcher:              s.dispatcher,
		interceptors:            s.interceptors,
		methodNotAllowedHandler: s.methodNotAllowedHandler,
	}

	for _, handler := range s.handlers {
		m.Handle(handler.pattern, handler.method, handler.h, handler.cfgs...)
	}
	return m
}

// Clone creates a copy of the current config.
// This can be used to create several instances of Mux that share the same set of
// plugins and some common handlers.
func (s *ServeMuxConfig) Clone() *ServeMuxConfig {
	c := &ServeMuxConfig{
		dispatcher:              s.dispatcher,
		interceptors:            make([]Interceptor, len(s.interceptors)),
		methodNotAllowedHandler: s.methodNotAllowedHandler,
	}
	copy(c.handlers, s.handlers)
	copy(c.interceptors, s.interceptors)
	return c
}

type registeredHandler struct {
	pattern          string
	methods          map[string]handlerConfig
	methodNotAllowed handlerConfig
}

func (rh *registeredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	cfg, ok := rh.methods[r.Method]
	if !ok {
		cfg = rh.methodNotAllowed
	}
	processRequest(cfg, w, r)
}

func (rh *registeredHandler) handleMethod(method string, cfg handlerConfig) {
	if _, exists := rh.methods[method]; exists {
		panic(fmt.Sprintf("double registration of (pattern = %q, method = %q)", rh.pattern, method))
	}
	rh.methods[method] = cfg
}

func configureInterceptors(interceptors []Interceptor, cfgs []InterceptorConfig) []configuredInterceptor {
	var its []configuredInterceptor
	for _, it := range interceptors {
		var matches []InterceptorConfig
		for _, c := range cfgs {
			if it.Match(c) {
				matches = append(matches, c)
			}
		}

		if len(matches) > 1 {
			msg := fmt.Sprintf("multiple configurations specified for interceptor %T: ", it)
			for _, match := range matches {
				msg += fmt.Sprintf("%#v", match)
			}
			panic(msg)
		}

		var cfg InterceptorConfig
		if len(matches) == 1 {
			cfg = matches[0]
		}
		its = append(its, configuredInterceptor{interceptor: it, config: cfg})
	}
	return its
}
<commit_msg>removed unused fields from handlerRegistration (the whole struct will likely go away soon)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"fmt"
	"log"
	"net/http"
)

// The HTTP request methods defined by RFC.
const (
	MethodConnect = "CONNECT" // RFC 7231, 4.3.6
	MethodDelete  = "DELETE"  // RFC 7231, 4.3.5
	MethodGet     = "GET"     // RFC 7231, 4.3.1
	MethodHead    = "HEAD"    // RFC 7231, 4.3.2
	MethodOptions = "OPTIONS" // RFC 7231, 4.3.7
	MethodPatch   = "PATCH"   // RFC 5789
	MethodPost    = "POST"    // RFC 7231, 4.3.3
	MethodPut     = "PUT"     // RFC 7231, 4.3.4
	MethodTrace   = "TRACE"   // RFC 7231, 4.3.8
)

// ServeMux is an HTTP request multiplexer. It matches the URL of each incoming
// request against a list of registered patterns and calls the handler for
// the pattern that most closely matches the URL.
//
// Patterns names are fixed, rooted paths, like "/favicon.ico", or rooted
// subtrees like "/images/" (note the trailing slash). Longer patterns take
// precedence over shorter ones, so that if there are handlers registered for
// both "/images/" and "/images/thumbnails/", the latter handler will be called
// for paths beginning "/images/thumbnails/" and the former will receive
// requests for any other paths in the "/images/" subtree.
//
// Note that since a pattern ending in a slash names a rooted subtree, the
// pattern "/" matches all paths not matched by other registered patterns,
// not just the URL with Path == "/".
//
// If a subtree has been registered and a request is received naming the subtree
// root without its trailing slash, ServeMux redirects that request to
// the subtree root (adding the trailing slash). This behavior can be overridden
// with a separate registration for the path without the trailing slash. For
// example, registering "/images/" causes ServeMux to redirect a request for
// "/images" to "/images/", unless "/images" has been registered separately.
//
// Patterns may optionally begin with a host name, restricting matches to URLs
// on that host only.  Host-specific patterns take precedence over general
// patterns, so that a handler might register for the two patterns "/codesearch"
// and "codesearch.google.com/" without also taking over requests for
// "http://www.google.com/".
//
// ServeMux also takes care of sanitizing the URL request path and the Host
// header, stripping the port number and redirecting any request containing . or
// .. elements or repeated slashes to an equivalent, cleaner URL.
//
// Multiple handlers can be registered for a single pattern, as long as they
// handle different HTTP methods.
type ServeMux struct {
	mux      *http.ServeMux
	handlers map[string]*registeredHandler

	dispatcher              Dispatcher
	interceptors            []Interceptor
	methodNotAllowedHandler handlerRegistration
}

// ServeHTTP dispatches the request to the handler whose method matches the
// incoming request and whose pattern most closely matches the request URL.
//
//  For each incoming request:
//  - [Before Phase] Interceptor.Before methods are called for every installed
//    interceptor, until an interceptor writes to a ResponseWriter (including
//    errors) or panics,
//  - the handler is called after a [Before Phase] if no writes or panics occured,
//  - the handler triggers the [Commit Phase] by writing to the ResponseWriter,
//  - [Commit Phase] Interceptor.Commit methods run for every interceptor whose
//    Before method was called,
//  - [Dispatcher Phase] after the [Commit Phase], the Dispatcher's appropriate
//    write method is called; the Dispatcher is responsible for determining whether
//    the response is indeed safe and writing it,
//  - if the handler attempts to write more than once, it is treated as an
//    unrecoverable error; the request processing ends abrubptly with a panic and
//    nothing else happens (note: this will change as soon as [After Phase] is
//    introduced)
//
// Interceptors should NOT rely on the order they're run.
func (m *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.mux.ServeHTTP(w, r)
}

// Handle registers a handler for the given pattern and method. If a handler is
// registered twice for the same pattern and method, Build will panic.
//
// InterceptorConfigs can be passed in order to modify the behavior of the
// interceptors on a registered handler. Passing an InterceptorConfig whose
// corresponding Interceptor was not installed will produce no effect. If
// multiple configurations are passed for the same Interceptor, Mux will panic.
func (m *ServeMux) Handle(pattern string, method string, h Handler, cfgs ...InterceptorConfig) {
	methodNotAllowed := handlerConfig{
		Dispatcher:   m.dispatcher,
		Handler:      m.methodNotAllowedHandler.handler,
		Interceptors: configureInterceptors(m.interceptors, m.methodNotAllowedHandler.cfgs),
	}

	if m.handlers[pattern] == nil {
		m.handlers[pattern] = &registeredHandler{
			pattern:          pattern,
			methodNotAllowed: methodNotAllowed,
			methods:          make(map[string]handlerConfig),
		}
		m.mux.Handle(pattern, m.handlers[pattern])
	}
	m.handlers[pattern].handleMethod(method,
		handlerConfig{
			Dispatcher:   m.dispatcher,
			Handler:      h,
			Interceptors: configureInterceptors(m.interceptors, cfgs),
		})
}

// ServeMuxConfig is a builder for ServeMux.
type ServeMuxConfig struct {
	dispatcher Dispatcher
	handlers   []struct {
		pattern, method string
		h               Handler
		cfgs            []InterceptorConfig
	}
	interceptors            []Interceptor
	methodNotAllowedHandler handlerRegistration
}

// NewServeMuxConfig crates a ServeMuxConfig with the provided Dispatcher. If
// the provided Dispatcher is nil, the DefaultDispatcher is used.
func NewServeMuxConfig(disp Dispatcher) *ServeMuxConfig {
	if disp == nil {
		disp = &DefaultDispatcher{}
	}
	return &ServeMuxConfig{
		dispatcher: disp,
		methodNotAllowedHandler: handlerRegistration{
			handler: HandlerFunc(defaultMethotNotAllowed),
		},
	}
}

type handlerRegistration struct {
	handler Handler
	cfgs    []InterceptorConfig
}

func (s *ServeMuxConfig) Handle(pattern string, method string, h Handler, cfgs ...InterceptorConfig) {
	s.handlers = append(s.handlers, struct {
		pattern string
		method  string
		h       Handler
		cfgs    []InterceptorConfig
	}{
		pattern: pattern,
		method:  method,
		h:       h,
		cfgs:    cfgs,
	})
}

// HandleMethodNotAllowed registers a handler that runs when a given method is
// not allowed for a registered path.
func (s *ServeMuxConfig) HandleMethodNotAllowed(h Handler, cfgs ...InterceptorConfig) {
	s.methodNotAllowedHandler = handlerRegistration{
		handler: h,
		cfgs:    cfgs,
	}
}

func defaultMethotNotAllowed(w ResponseWriter, req *IncomingRequest) Result {
	return w.WriteError(StatusMethodNotAllowed)
}

// Intercept installs the given interceptors.
//
// Interceptors order is respected and interceptors are always run in the
// order they've been installed.
//
// Calling Intercept multiple times is valid. Interceptors that are added last
// will run last.
func (s *ServeMuxConfig) Intercept(is ...Interceptor) {
	s.interceptors = append(s.interceptors, is...)
}

// Mux returns the ServeMux with a copy of the current configuration.
func (s *ServeMuxConfig) Mux() *ServeMux {
	freezeLocalDev = true
	if IsLocalDev() {
		log.Println("Warning: creating safehttp.Mux in dev mode. This configuration is not valid for production use")
	}

	if s.dispatcher == nil {
		panic("Use NewServeMuxConfig instead of creating ServeMuxConfig using a composite literal.")
	}

	m := &ServeMux{
		mux:                     http.NewServeMux(),
		handlers:                make(map[string]*registeredHandler),
		dispatcher:              s.dispatcher,
		interceptors:            s.interceptors,
		methodNotAllowedHandler: s.methodNotAllowedHandler,
	}

	for _, handler := range s.handlers {
		m.Handle(handler.pattern, handler.method, handler.h, handler.cfgs...)
	}
	return m
}

// Clone creates a copy of the current config.
// This can be used to create several instances of Mux that share the same set of
// plugins and some common handlers.
func (s *ServeMuxConfig) Clone() *ServeMuxConfig {
	c := &ServeMuxConfig{
		dispatcher:              s.dispatcher,
		interceptors:            make([]Interceptor, len(s.interceptors)),
		methodNotAllowedHandler: s.methodNotAllowedHandler,
	}
	copy(c.handlers, s.handlers)
	copy(c.interceptors, s.interceptors)
	return c
}

type registeredHandler struct {
	pattern          string
	methods          map[string]handlerConfig
	methodNotAllowed handlerConfig
}

func (rh *registeredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	cfg, ok := rh.methods[r.Method]
	if !ok {
		cfg = rh.methodNotAllowed
	}
	processRequest(cfg, w, r)
}

func (rh *registeredHandler) handleMethod(method string, cfg handlerConfig) {
	if _, exists := rh.methods[method]; exists {
		panic(fmt.Sprintf("double registration of (pattern = %q, method = %q)", rh.pattern, method))
	}
	rh.methods[method] = cfg
}

func configureInterceptors(interceptors []Interceptor, cfgs []InterceptorConfig) []configuredInterceptor {
	var its []configuredInterceptor
	for _, it := range interceptors {
		var matches []InterceptorConfig
		for _, c := range cfgs {
			if it.Match(c) {
				matches = append(matches, c)
			}
		}

		if len(matches) > 1 {
			msg := fmt.Sprintf("multiple configurations specified for interceptor %T: ", it)
			for _, match := range matches {
				msg += fmt.Sprintf("%#v", match)
			}
			panic(msg)
		}

		var cfg InterceptorConfig
		if len(matches) == 1 {
			cfg = matches[0]
		}
		its = append(its, configuredInterceptor{interceptor: it, config: cfg})
	}
	return its
}
<|endoftext|>


<commit_before>// Package instagram provides a minimialist instagram API wrapper.
package instagram

import (
	"encoding/json"
	"fmt"
	"io"
	// "io/ioutil"
	"net/http"
	"net/url"
)

var (
	baseUrl = "https://api.instagram.com/v1"
)

type Api struct {
	ClientId    string
	AccessToken string
}

// Create an API with either a ClientId OR an accessToken. Only one is required. Access tokens are preferred because they keep rate limiting down.
func New(clientId string, accessToken string) *Api {
	if clientId == "" && accessToken == "" {
		panic("ClientId or AccessToken must be given to create an Api")
	}

	return &Api{
		ClientId:    clientId,
		AccessToken: accessToken,
	}
}

// -- Implementation of request --

func buildGetRequest(urlStr string, params url.Values) (*http.Request, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	// If we are getting, then we can't merge query params
	if params != nil {
		if u.RawQuery != "" {
			return nil, fmt.Errorf("Cannot merge query params in urlStr and params")
		}
		u.RawQuery = params.Encode()
	}

	return http.NewRequest("GET", u.String(), nil)
}

func (api *Api) extendParams(p url.Values) url.Values {
	if p == nil {
		p = url.Values{}
	}
	if api.AccessToken != "" {
		p.Set("access_token", api.AccessToken)
	} else {
		p.Set("client_id", api.ClientId)
	}
	return p
}

func (api *Api) get(path string, params url.Values, r interface{}) error {
	params = api.extendParams(params)
	req, err := buildGetRequest(urlify(path), params)
	if err != nil {
		return err
	}
	return api.do(req, r)
}

func (api *Api) do(req *http.Request, r interface{}) error {
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return apiError(resp)
	}

	return decodeResponse(resp.Body, r)
}

func decodeResponse(body io.Reader, to interface{}) error {
	//b, _ := ioutil.ReadAll(body)
	//fmt.Println("Body:",string(b))
	// err := json.Unmarshal(b, to)
	err := json.NewDecoder(body).Decode(to)
	//fmt.Println("Body: " + string(body))

	if err != nil {
		return fmt.Errorf("instagram: error decoding body; %s", err.Error())
	}
	return nil
}

func apiError(resp *http.Response) error {
	m := new(MetaResponse)
	if err := decodeResponse(resp.Body, m); err != nil {
		return err
	}

	var err MetaError
	if m.Meta != nil {
		err = MetaError(*m.Meta)
	} else {
		err = MetaError(Meta{Code: resp.StatusCode, ErrorMessage: resp.Status})
	}
	return &err
}

func urlify(path string) string {
	return baseUrl + path
}

type MetaError Meta

func (m *MetaError) Error() string {
	return fmt.Sprintf("Error making api call: Code %d %s %s", m.Code, m.ErrorType, m.ErrorMessage)
}

func ensureParams(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}
<commit_msg>eduted godeps debug<commit_after>// Package instagram provides a minimialist instagram API wrapper.
package instagram

import (
	"encoding/json"
	"fmt"
	"io"
	// "io/ioutil"
	"net/http"
	"net/url"
)

var (
	baseUrl = "https://api.instagram.com/v1"
)

type Api struct {
	ClientId    string
	AccessToken string
}

// Create an API with either a ClientId OR an accessToken. Only one is required. Access tokens are preferred because they keep rate limiting down.
func New(clientId string, accessToken string) *Api {
	if clientId == "" && accessToken == "" {
		panic("ClientId or AccessToken must be given to create an Api")
	}

	return &Api{
		ClientId:    clientId,
		AccessToken: accessToken,
	}
}

// -- Implementation of request --

func buildGetRequest(urlStr string, params url.Values) (*http.Request, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	fmt.Println("params:",params)
	fmt.Println("uRawQuery:",u.RawQuery)
	fmt.Println("urlStr:",urlStr)
	// If we are getting, then we can't merge query params
	if params != nil {
		if u.RawQuery != "" {
			return nil, fmt.Errorf("Cannot merge query params in urlStr and params")
		}
		u.RawQuery = params.Encode()
	}

	return http.NewRequest("GET", u.String(), nil)
}

func (api *Api) extendParams(p url.Values) url.Values {
	if p == nil {
		p = url.Values{}
	}
	if api.AccessToken != "" {
		p.Set("access_token", api.AccessToken)
	} else {
		p.Set("client_id", api.ClientId)
	}
	return p
}

func (api *Api) get(path string, params url.Values, r interface{}) error {
	params = api.extendParams(params)
	req, err := buildGetRequest(urlify(path), params)
	if err != nil {
		return err
	}
	return api.do(req, r)
}

func (api *Api) do(req *http.Request, r interface{}) error {
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return apiError(resp)
	}

	return decodeResponse(resp.Body, r)
}

func decodeResponse(body io.Reader, to interface{}) error {
	//b, _ := ioutil.ReadAll(body)
	//fmt.Println("Body:",string(b))
	// err := json.Unmarshal(b, to)
	err := json.NewDecoder(body).Decode(to)
	//fmt.Println("Body: " + string(body))

	if err != nil {
		return fmt.Errorf("instagram: error decoding body; %s", err.Error())
	}
	return nil
}

func apiError(resp *http.Response) error {
	m := new(MetaResponse)
	if err := decodeResponse(resp.Body, m); err != nil {
		return err
	}

	var err MetaError
	if m.Meta != nil {
		err = MetaError(*m.Meta)
	} else {
		err = MetaError(Meta{Code: resp.StatusCode, ErrorMessage: resp.Status})
	}
	return &err
}

func urlify(path string) string {
	return baseUrl + path
}

type MetaError Meta

func (m *MetaError) Error() string {
	return fmt.Sprintf("Error making api call: Code %d %s %s", m.Code, m.ErrorType, m.ErrorMessage)
}

func ensureParams(v url.Values) url.Values {
	if v == nil {
		return url.Values{}
	}
	return v
}
<|endoftext|>


<commit_before>package bsdiff

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"

	humanize "github.com/dustin/go-humanize"
)

// ErrCorrupt indicates that a patch is corrupted, most often that it would produce a longer file
// than specified
var ErrCorrupt = errors.New("corrupt patch")

// ReadMessageFunc should read the passed protobuf and relay any errors.
// See the `wire` package for an example implementation.
type ReadMessageFunc func(msg interface{}) error

// BSPatch applies patch to old, according to the bspatch algorithm,
// and writes the result to new.
func BSPatch(old io.Reader, new io.Writer, newSize int64, readMessage ReadMessageFunc) error {
	// TODO: still debating whether we should take an io.ReadSeeker instead... probably?
	// the consumer can still do `ReadAll` themselves and pass a bytes.NewBuffer()
	obuf, err := ioutil.ReadAll(old)
	if err != nil {
		return err
	}

	// TODO: write directly to new instead of using a buffer
	nbuf := make([]byte, newSize)

	var oldpos, newpos int64

	ctrl := &Control{}

	for {
		ctrl.Reset()

		err = readMessage(ctrl)
		if err != nil {
			return err
		}

		if ctrl.Eof {
			break
		}

		// Sanity-check
		if newpos+int64(len(ctrl.Add)) > newSize {
			return ErrCorrupt
		}

		// Add old data to diff string
		for i := int64(0); i < int64(len(ctrl.Add)); i++ {
			nbuf[newpos+i] = ctrl.Add[i] + obuf[oldpos+i]
		}

		// Adjust pointers
		newpos += int64(len(ctrl.Add))
		oldpos += int64(len(ctrl.Add))

		// Sanity-check
		if newpos+int64(len(ctrl.Copy)) > newSize {
			return ErrCorrupt
		}

		// Read extra string
		copy(nbuf[newpos:], ctrl.Copy)

		// Adjust pointers
		newpos += int64(len(ctrl.Copy))
		oldpos += ctrl.Seek
	}

	if newpos != newSize {
		return fmt.Errorf("bsdiff: expected new file to be %d, was %d (%s difference)", newSize, newpos, humanize.IBytes(uint64(newSize-newpos)))
	}

	// Write the new file
	_, err = new.Write(nbuf)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>BSPatch => Patch<commit_after>package bsdiff

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"

	humanize "github.com/dustin/go-humanize"
)

// ErrCorrupt indicates that a patch is corrupted, most often that it would produce a longer file
// than specified
var ErrCorrupt = errors.New("corrupt patch")

// ReadMessageFunc should read the passed protobuf and relay any errors.
// See the `wire` package for an example implementation.
type ReadMessageFunc func(msg interface{}) error

// Patch applies patch to old, according to the bspatch algorithm,
// and writes the result to new.
func Patch(old io.Reader, new io.Writer, newSize int64, readMessage ReadMessageFunc) error {
	// TODO: still debating whether we should take an io.ReadSeeker instead... probably?
	// the consumer can still do `ReadAll` themselves and pass a bytes.NewBuffer()
	obuf, err := ioutil.ReadAll(old)
	if err != nil {
		return err
	}

	// TODO: write directly to new instead of using a buffer
	nbuf := make([]byte, newSize)

	var oldpos, newpos int64

	ctrl := &Control{}

	for {
		ctrl.Reset()

		err = readMessage(ctrl)
		if err != nil {
			return err
		}

		if ctrl.Eof {
			break
		}

		// Sanity-check
		if newpos+int64(len(ctrl.Add)) > newSize {
			return ErrCorrupt
		}

		// Add old data to diff string
		for i := int64(0); i < int64(len(ctrl.Add)); i++ {
			nbuf[newpos+i] = ctrl.Add[i] + obuf[oldpos+i]
		}

		// Adjust pointers
		newpos += int64(len(ctrl.Add))
		oldpos += int64(len(ctrl.Add))

		// Sanity-check
		if newpos+int64(len(ctrl.Copy)) > newSize {
			return ErrCorrupt
		}

		// Read extra string
		copy(nbuf[newpos:], ctrl.Copy)

		// Adjust pointers
		newpos += int64(len(ctrl.Copy))
		oldpos += ctrl.Seek
	}

	if newpos != newSize {
		return fmt.Errorf("bsdiff: expected new file to be %d, was %d (%s difference)", newSize, newpos, humanize.IBytes(uint64(newSize-newpos)))
	}

	// Write the new file
	_, err = new.Write(nbuf)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	// Major is the current major version of master branch.
	Major = 0
	// Minor is the current minor version of master branch.
	Minor = 4
	// Patch is the curernt patched version of the master branch.
	Patch = 1
	// Release is the current release level of the master branch. Valid values
	// are dev (developement unreleased), rcX (release candidate with current
	// iteration), stable (indicates a final released version).
	Release = "dev"
)
<commit_msg>Release 0.4.1 stable preparation<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	// Major is the current major version of master branch.
	Major = 0
	// Minor is the current minor version of master branch.
	Minor = 4
	// Patch is the curernt patched version of the master branch.
	Patch = 1
	// Release is the current release level of the master branch. Valid values
	// are dev (developement unreleased), rcX (release candidate with current
	// iteration), stable (indicates a final released version).
	Release = "stable"
)
<|endoftext|>


<commit_before>package integration_test

import (
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("JRuby App", func() {
	var app *cutlass.App

	AfterEach(func() { app = DestroyApp(app) })

	Context("without start command", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "sinatra_jruby"))
			app.Memory = "512M"
		})

		It("", func() {
			PushAppAndConfirm(app)

			By("the buildpack logged it installed a specific version of JRuby", func() {
				Expect(app.Stdout.String()).To(ContainSubstring("Installing openjdk"))
				Expect(app.Stdout.String()).To(MatchRegexp("ruby-2.3.\\d+-jruby-9.\\d+.\\d+.0"))
				Expect(app.GetBody("/ruby")).To(MatchRegexp("jruby 2.3.\\d+"))
			})

			By("the OpenJDK runs properly", func() {
				Expect(app.Stdout.String()).ToNot(ContainSubstring("OpenJDK 64-Bit Server VM warning"))
			})
		})

		Context("a cached buildpack", func() {
			BeforeEach(SkipUnlessCached)

			AssertNoInternetTraffic("sinatra_jruby")
		})
	})
	Context("with a jruby start command", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "jruby_start_command"))
			app.Memory = "512M"
		})

		It("stages and runs successfully", func() {
			PushAppAndConfirm(app)
		})
	})
})
<commit_msg>Build assertions from fixture Gemfile for sinatra_jruby integration test [#157851009]<commit_after>package integration_test

import (
	"fmt"
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"regexp"
	"io/ioutil"
)

var _ = Describe("JRuby App", func() {
	var app *cutlass.App

	AfterEach(func() { app = DestroyApp(app) })

	Context("without start command", func() {
		var dir, rubyVersion, jrubyVersion string

		BeforeEach(func() {
			dir = filepath.Join(bpDir, "fixtures", "sinatra_jruby")
			data, err:= ioutil.ReadFile(filepath.Join(dir, "Gemfile"))
			Expect(err).To(BeNil())
			re := regexp.MustCompile(`ruby '(\d+.\d+.\d+)', :engine => 'jruby', :engine_version => '(\d+.\d+.\d+.\d+)'`)
			matches := re.FindStringSubmatch(string(data))
			rubyVersion = matches[1]
			jrubyVersion = matches[2]

			app = cutlass.New(dir)
			app.Memory = "512M"
		})

		It("installs the correct version of JRuby", func() {
			PushAppAndConfirm(app)

			By("the buildpack logged it installed a specific version of JRuby", func() {
				Expect(app.Stdout.String()).To(ContainSubstring("Installing openjdk"))
				Expect(app.Stdout.String()).To(ContainSubstring(fmt.Sprintf("Installing jruby ruby-%s-jruby-%s", rubyVersion, jrubyVersion)))
				Expect(app.GetBody("/ruby")).To(ContainSubstring(fmt.Sprintf("jruby %s", rubyVersion)))
			})

			By("the OpenJDK runs properly", func() {
				Expect(app.Stdout.String()).ToNot(ContainSubstring("OpenJDK 64-Bit Server VM warning"))
			})
		})

		Context("a cached buildpack", func() {
			BeforeEach(SkipUnlessCached)

			AssertNoInternetTraffic("sinatra_jruby")
		})
	})
	Context("with a jruby start command", func() {
		BeforeEach(func() {
			app = cutlass.New(filepath.Join(bpDir, "fixtures", "jruby_start_command"))
			app.Memory = "512M"
		})

		It("stages and runs successfully", func() {
			PushAppAndConfirm(app)
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package downloader

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http/httptest"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"

	"go.chromium.org/luci/common/data/caching/cache"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/isolatedclient/isolatedfake"
	"go.chromium.org/luci/common/testing/testfs"

	. "github.com/smartystreets/goconvey/convey"
)

func TestNormalizePathSeparator(t *testing.T) {
	t.Parallel()

	Convey("Check path normalization", t, func() {
		Convey("posix path", func() {
			So(normalizePathSeparator("a/b"), ShouldEqual, filepath.Join("a", "b"))
		})

		Convey("windows path", func() {
			So(normalizePathSeparator(`a\b`), ShouldEqual, filepath.Join("a", "b"))
		})
	})
}

func TestDownloaderFetchIsolated(t *testing.T) {
	t.Parallel()
	ctx := context.Background()

	data1 := []byte("hello world!")
	data2 := []byte("wat")
	tardata := genTar(t)

	namespace := isolatedclient.DefaultNamespace
	h := isolated.GetHash(namespace)
	server := isolatedfake.New()
	data1hash := server.Inject(namespace, data1)
	data2hash := server.Inject(namespace, data2)
	tardatahash := server.Inject(namespace, tardata)
	tardataname := fmt.Sprintf("%s.tar", tardatahash)

	onePath := filepath.Join("foo", "one.txt")
	twoPath := filepath.Join("foo", "two.txt")
	posixPath := "posix/path"
	winPath := `win\path`
	isolated1 := isolated.New(h)
	isolated1.Files = map[string]isolated.File{
		onePath:     isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		twoPath:     isolated.BasicFile(data2hash, 0764, int64(len(data2))),
		posixPath:   isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		winPath:     isolated.BasicFile(data2hash, 0664, int64(len(data2))),
		tardataname: isolated.TarFile(tardatahash, int64(len(tardata))),
	}
	isolated1bytes, _ := json.Marshal(&isolated1)
	isolated1hash := server.Inject(namespace, isolated1bytes)

	lolPath := filepath.Join("bar", "lol.txt")
	oloPath := filepath.Join("foo", "boz", "olo.txt")
	isolated2 := isolated.New(h)
	isolated2.Files = map[string]isolated.File{
		lolPath: isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		oloPath: isolated.BasicFile(data2hash, 0664, int64(len(data2))),
	}
	isolatedFiles := stringset.NewFromSlice([]string{
		tardataname,
		onePath,
		twoPath,
		normalizePathSeparator(posixPath),
		normalizePathSeparator(winPath),
		lolPath,
		oloPath,
		// In tardata
		"file1",
		"file2",
		filepath.Join("tar", "posix", "path"),
		filepath.Join("tar", "win", "path"),
	}...)
	blahPath := "blah.txt"

	// Symlinks not supported on Windows.
	if runtime.GOOS != "windows" {
		isolated2.Files[blahPath] = isolated.SymLink(oloPath)
		isolatedFiles.Add(blahPath)
	}
	isolated2.Includes = isolated.HexDigests{isolated1hash}
	isolated2bytes, _ := json.Marshal(&isolated2)
	isolated2hash := server.Inject(namespace, isolated2bytes)

	ts := httptest.NewServer(server)
	defer ts.Close()
	client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

	Convey(`A downloader should be able to download the isolated.`, t, func() {
		tmpDir, err := ioutil.TempDir("", "isolated")
		So(err, ShouldBeNil)
		defer func() {
			So(os.RemoveAll(tmpDir), ShouldBeNil)
		}()

		mu := sync.Mutex{}
		var files []string
		d := New(ctx, client, isolated2hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		b, err := ioutil.ReadFile(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		fi, err := os.Stat(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0111, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		fi, err = os.Stat(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0011, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, lolPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, oloPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		// Check files in tar archive
		_, err = os.Stat(filepath.Join(tmpDir, "file1"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "file2"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "posix", "path"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "win", "path"))
		So(err, ShouldBeNil)

		_, err = ioutil.ReadFile(filepath.Join(tmpDir, tardataname))
		So(os.IsNotExist(err), ShouldBeTrue)

		if runtime.GOOS != "windows" {
			l, err := os.Readlink(filepath.Join(tmpDir, blahPath))
			So(err, ShouldBeNil)
			So(l, ShouldResemble, oloPath)
		}
	})
}

// genTar returns a valid tar file.
func genTar(t *testing.T) []byte {
	b := bytes.Buffer{}
	tw := tar.NewWriter(&b)
	d := []byte("hello file1")
	if err := tw.WriteHeader(&tar.Header{Name: "file1", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte(strings.Repeat("hello file2", 100))
	if err := tw.WriteHeader(&tar.Header{Name: "file2", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("posixpath")
	if err := tw.WriteHeader(&tar.Header{Name: "tar/posix/path", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("winpath")
	if err := tw.WriteHeader(&tar.Header{Name: `tar\win\path`, Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}

	if err := tw.Close(); err != nil {
		t.Fatal(err)
	}
	return b.Bytes()
}

func TestDownloaderWithCache(t *testing.T) {
	t.Parallel()
	Convey(`TestDownloaderWithCache`, t, testfs.MustWithTempDir(t, "", func(tmpDir string) {
		ctx := context.Background()

		miss := []byte("cache miss")
		hit := []byte("cache hit")
		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		misshash := server.Inject(namespace, miss)
		hithash := isolated.HashBytes(isolated.GetHash(namespace), hit)

		missPath := filepath.Join("foo", "miss.txt")
		hitPath := filepath.Join("foo", "hit.txt")
		isolated1 := isolated.New(h)

		isolated1.Files = map[string]isolated.File{
			missPath: isolated.BasicFile(misshash, 0664, int64(len(miss))),
			hitPath:  isolated.BasicFile(hithash, 0664, int64(len(hit))),
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		isolatedFiles := stringset.NewFromSlice([]string{
			missPath,
			hitPath,
		}...)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		mu := sync.Mutex{}
		var files []string
		policy := cache.Policies{
			MaxSize:  1024,
			MaxItems: 1024,
		}
		var cacheObj cache.Cache
		Convey("memcache", func() {
			cacheObj = cache.NewMemory(policy, namespace)
		})

		Convey("diskcache", func() {
			cacheDir := filepath.Join(tmpDir, "cache")
			So(os.MkdirAll(cacheDir, os.ModePerm), ShouldBeNil)
			var err error
			cacheObj, err = cache.NewDisk(policy, cacheDir, namespace)
			So(err, ShouldBeNil)
		})

		So(cacheObj.Add(hithash, bytes.NewReader(hit)), ShouldBeNil)

		d := New(ctx, client, isolated1hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
			Cache: cacheObj,
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		So(cacheObj.Touch(misshash), ShouldBeTrue)
	}))
}

func TestFetchAndMap(t *testing.T) {
	t.Parallel()
	Convey(`TestFetchAndMap`, t, func() {
		ctx := context.Background()
		tmpDir, err := ioutil.TempDir("", "isolated")
		So(err, ShouldBeNil)
		defer func() {
			So(os.RemoveAll(tmpDir), ShouldBeNil)
		}()

		data1 := []byte("hello world!")

		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		data1hash := server.Inject(namespace, data1)

		onePath := filepath.Join("foo", "one.txt")
		onePathFile := isolated.BasicFile(data1hash, 0664, int64(len(data1)))
		isolated1 := isolated.New(h)

		tardata := genTar(t)
		tarhash := server.Inject(namespace, tardata)

		isolated1.Files = map[string]isolated.File{
			onePath:   onePathFile,
			"tar.tar": isolated.TarFile(tarhash, int64(len(tardata))),
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		policy := cache.Policies{
			MaxSize:  1024 * 1024,
			MaxItems: 1024,
		}
		memcache := cache.NewMemory(policy, namespace)

		isomap, stats, err := FetchAndMap(ctx, isolated1hash, client, memcache, tmpDir)
		So(err, ShouldBeNil)
		So(isomap, ShouldResemble, &isolated.Isolated{
			Algo:    "sha-1",
			Version: "1.4",
		})

		So(stats.Duration, ShouldBeGreaterThan, 0)
		So(stats.ItemsCold, ShouldResemble, []byte{120, 156, 226, 249, 162, 15, 8, 0, 0, 255, 255, 2, 62, 1, 48})
		So(stats.ItemsHot, ShouldResemble, []byte(nil))

		buf, err := ioutil.ReadFile(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		So(buf, ShouldResemble, data1)

		So(memcache.Touch(tarhash), ShouldBeTrue)
	})
}
<commit_msg>downloader: remove cache.NewMemory usage<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package downloader

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http/httptest"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"

	"go.chromium.org/luci/common/data/caching/cache"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/isolatedclient/isolatedfake"
	"go.chromium.org/luci/common/testing/testfs"

	. "github.com/smartystreets/goconvey/convey"
)

func TestNormalizePathSeparator(t *testing.T) {
	t.Parallel()

	Convey("Check path normalization", t, func() {
		Convey("posix path", func() {
			So(normalizePathSeparator("a/b"), ShouldEqual, filepath.Join("a", "b"))
		})

		Convey("windows path", func() {
			So(normalizePathSeparator(`a\b`), ShouldEqual, filepath.Join("a", "b"))
		})
	})
}

func TestDownloaderFetchIsolated(t *testing.T) {
	t.Parallel()
	ctx := context.Background()

	data1 := []byte("hello world!")
	data2 := []byte("wat")
	tardata := genTar(t)

	namespace := isolatedclient.DefaultNamespace
	h := isolated.GetHash(namespace)
	server := isolatedfake.New()
	data1hash := server.Inject(namespace, data1)
	data2hash := server.Inject(namespace, data2)
	tardatahash := server.Inject(namespace, tardata)
	tardataname := fmt.Sprintf("%s.tar", tardatahash)

	onePath := filepath.Join("foo", "one.txt")
	twoPath := filepath.Join("foo", "two.txt")
	posixPath := "posix/path"
	winPath := `win\path`
	isolated1 := isolated.New(h)
	isolated1.Files = map[string]isolated.File{
		onePath:     isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		twoPath:     isolated.BasicFile(data2hash, 0764, int64(len(data2))),
		posixPath:   isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		winPath:     isolated.BasicFile(data2hash, 0664, int64(len(data2))),
		tardataname: isolated.TarFile(tardatahash, int64(len(tardata))),
	}
	isolated1bytes, _ := json.Marshal(&isolated1)
	isolated1hash := server.Inject(namespace, isolated1bytes)

	lolPath := filepath.Join("bar", "lol.txt")
	oloPath := filepath.Join("foo", "boz", "olo.txt")
	isolated2 := isolated.New(h)
	isolated2.Files = map[string]isolated.File{
		lolPath: isolated.BasicFile(data1hash, 0664, int64(len(data1))),
		oloPath: isolated.BasicFile(data2hash, 0664, int64(len(data2))),
	}
	isolatedFiles := stringset.NewFromSlice([]string{
		tardataname,
		onePath,
		twoPath,
		normalizePathSeparator(posixPath),
		normalizePathSeparator(winPath),
		lolPath,
		oloPath,
		// In tardata
		"file1",
		"file2",
		filepath.Join("tar", "posix", "path"),
		filepath.Join("tar", "win", "path"),
	}...)
	blahPath := "blah.txt"

	// Symlinks not supported on Windows.
	if runtime.GOOS != "windows" {
		isolated2.Files[blahPath] = isolated.SymLink(oloPath)
		isolatedFiles.Add(blahPath)
	}
	isolated2.Includes = isolated.HexDigests{isolated1hash}
	isolated2bytes, _ := json.Marshal(&isolated2)
	isolated2hash := server.Inject(namespace, isolated2bytes)

	ts := httptest.NewServer(server)
	defer ts.Close()
	client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

	Convey(`A downloader should be able to download the isolated.`, t, func() {
		tmpDir, err := ioutil.TempDir("", "isolated")
		So(err, ShouldBeNil)
		defer func() {
			So(os.RemoveAll(tmpDir), ShouldBeNil)
		}()

		mu := sync.Mutex{}
		var files []string
		d := New(ctx, client, isolated2hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		b, err := ioutil.ReadFile(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		fi, err := os.Stat(filepath.Join(tmpDir, onePath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0111, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		fi, err = os.Stat(filepath.Join(tmpDir, twoPath))
		So(err, ShouldBeNil)
		// to ignore effect of umask, only check executable bit.
		So(fi.Mode()&0011, ShouldEqual, 0)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, lolPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data1)

		b, err = ioutil.ReadFile(filepath.Join(tmpDir, oloPath))
		So(err, ShouldBeNil)
		So(b, ShouldResemble, data2)

		// Check files in tar archive
		_, err = os.Stat(filepath.Join(tmpDir, "file1"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "file2"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "posix", "path"))
		So(err, ShouldBeNil)

		_, err = os.Stat(filepath.Join(tmpDir, "tar", "win", "path"))
		So(err, ShouldBeNil)

		_, err = ioutil.ReadFile(filepath.Join(tmpDir, tardataname))
		So(os.IsNotExist(err), ShouldBeTrue)

		if runtime.GOOS != "windows" {
			l, err := os.Readlink(filepath.Join(tmpDir, blahPath))
			So(err, ShouldBeNil)
			So(l, ShouldResemble, oloPath)
		}
	})
}

// genTar returns a valid tar file.
func genTar(t *testing.T) []byte {
	b := bytes.Buffer{}
	tw := tar.NewWriter(&b)
	d := []byte("hello file1")
	if err := tw.WriteHeader(&tar.Header{Name: "file1", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte(strings.Repeat("hello file2", 100))
	if err := tw.WriteHeader(&tar.Header{Name: "file2", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("posixpath")
	if err := tw.WriteHeader(&tar.Header{Name: "tar/posix/path", Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}
	d = []byte("winpath")
	if err := tw.WriteHeader(&tar.Header{Name: `tar\win\path`, Mode: 0644, Typeflag: tar.TypeReg, Size: int64(len(d))}); err != nil {
		t.Fatal(err)
	}
	if _, err := tw.Write(d); err != nil {
		t.Fatal(err)
	}

	if err := tw.Close(); err != nil {
		t.Fatal(err)
	}
	return b.Bytes()
}

func TestDownloaderWithCache(t *testing.T) {
	t.Parallel()
	Convey(`TestDownloaderWithCache`, t, testfs.MustWithTempDir(t, "", func(tmpDir string) {
		ctx := context.Background()

		miss := []byte("cache miss")
		hit := []byte("cache hit")
		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		misshash := server.Inject(namespace, miss)
		hithash := isolated.HashBytes(isolated.GetHash(namespace), hit)

		missPath := filepath.Join("foo", "miss.txt")
		hitPath := filepath.Join("foo", "hit.txt")
		isolated1 := isolated.New(h)

		isolated1.Files = map[string]isolated.File{
			missPath: isolated.BasicFile(misshash, 0664, int64(len(miss))),
			hitPath:  isolated.BasicFile(hithash, 0664, int64(len(hit))),
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		isolatedFiles := stringset.NewFromSlice([]string{
			missPath,
			hitPath,
		}...)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		mu := sync.Mutex{}
		var files []string
		policy := cache.Policies{
			MaxSize:  1024,
			MaxItems: 1024,
		}
		var cacheObj cache.Cache
		Convey("diskcache", func() {
			cacheDir := filepath.Join(tmpDir, "cache")
			So(os.MkdirAll(cacheDir, os.ModePerm), ShouldBeNil)
			var err error
			cacheObj, err = cache.NewDisk(policy, cacheDir, namespace)
			So(err, ShouldBeNil)
		})

		So(cacheObj.Add(hithash, bytes.NewReader(hit)), ShouldBeNil)

		d := New(ctx, client, isolated1hash, tmpDir, &Options{
			FileCallback: func(name string, _ *isolated.File) {
				mu.Lock()
				files = append(files, name)
				mu.Unlock()
			},
			Cache: cacheObj,
		})
		So(d.Wait(), ShouldBeNil)
		So(stringset.NewFromSlice(files...), ShouldResemble, isolatedFiles)

		So(cacheObj.Touch(misshash), ShouldBeTrue)
	}))
}

func TestFetchAndMap(t *testing.T) {
	t.Parallel()

	Convey(`TestFetchAndMap`, t, testfs.MustWithTempDir(t, "", func(tmpdir string) {
		ctx := context.Background()
		isolatedDir := filepath.Join(tmpdir, "isolated")
		So(os.Mkdir(isolatedDir, 0o700), ShouldBeNil)

		data1 := []byte("hello world!")

		namespace := isolatedclient.DefaultNamespace
		h := isolated.GetHash(namespace)
		server := isolatedfake.New()
		data1hash := server.Inject(namespace, data1)

		onePath := filepath.Join("foo", "one.txt")
		onePathFile := isolated.BasicFile(data1hash, 0664, int64(len(data1)))
		isolated1 := isolated.New(h)

		tardata := genTar(t)
		tarhash := server.Inject(namespace, tardata)

		isolated1.Files = map[string]isolated.File{
			onePath:   onePathFile,
			"tar.tar": isolated.TarFile(tarhash, int64(len(tardata))),
		}
		isolated1bytes, _ := json.Marshal(&isolated1)
		isolated1hash := server.Inject(namespace, isolated1bytes)

		ts := httptest.NewServer(server)
		defer ts.Close()
		client := isolatedclient.NewClient(ts.URL, isolatedclient.WithNamespace(namespace))

		policy := cache.Policies{
			MaxSize:  1024 * 1024,
			MaxItems: 1024,
		}
		diskcache, err := cache.NewDisk(policy, filepath.Join(tmpdir, "cache"), namespace)
		So(err, ShouldBeNil)
		defer func() {
			So(diskcache.Close(), ShouldBeNil)
		}()

		isomap, stats, err := FetchAndMap(ctx, isolated1hash, client, diskcache, isolatedDir)
		So(err, ShouldBeNil)
		So(isomap, ShouldResemble, &isolated.Isolated{
			Algo:    "sha-1",
			Version: "1.4",
		})

		So(stats.Duration, ShouldBeGreaterThan, 0)
		So(stats.ItemsCold, ShouldResemble, []byte{120, 156, 226, 249, 162, 15, 8, 0, 0, 255, 255, 2, 62, 1, 48})
		So(stats.ItemsHot, ShouldResemble, []byte(nil))

		buf, err := ioutil.ReadFile(filepath.Join(isolatedDir, onePath))
		So(err, ShouldBeNil)
		So(buf, ShouldResemble, data1)

		So(diskcache.Touch(tarhash), ShouldBeTrue)
	}))
}
<|endoftext|>


<commit_before>package genetics

type Population struct {
	// Species in the Population. Note that the species should comprise all the genomes
	species []*Species
	// The highest species number
	lastSpecies int

}
<commit_msg>Added additional fields<commit_after>package genetics

type Population struct {
	// Species in the Population. Note that the species should comprise all the genomes
	Species     []*Species
	// The highest species number
	LastSpecies int
	// For holding the genetic innovations of the newest generation
	Innovations []*Innovation

	// The current innovation number for population
	currInnovNum int64
}

// Returns current innovation number and increment innovations number counter after that
func (p *Population) getInnovationNumberAndIncrement() int64 {
	inn_num := p.currInnovNum
	p.currInnovNum++
	return inn_num
}
<|endoftext|>


<commit_before>package workflow

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"

	"github.com/openshift/installer/installer/pkg/config"
	configgenerator "github.com/openshift/installer/installer/pkg/config-generator"
)

const (
	assetsStep       = "assets"
	binaryPrefix     = "installer"
	bootstrapOff     = "-var=tectonic_bootstrap=false"
	bootstrapOn      = "-var=tectonic_bootstrap=true"
	configFileName   = "config.yaml"
	etcdStep         = "etcd"
	internalFileName = "internal.yaml"
	joinWorkersStep  = "joining_workers"
	mastersStep      = "masters"
	newTLSStep       = "newtls"
	stepsBaseDir     = "steps"
	tlsStep          = "tls"
	tncDNSStep       = "tnc_dns"
	topologyStep     = "topology"
)

func copyFile(fromFilePath, toFilePath string) error {
	from, err := os.Open(fromFilePath)
	if err != nil {
		return err
	}
	defer from.Close()

	to, err := os.OpenFile(toFilePath, os.O_RDWR|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	defer to.Close()

	_, err = io.Copy(to, from)
	return err
}

// returns the directory containing templates for a given step. If platform is
// specified, it looks for a subdirectory with platform first, falling back if
// there are no platform-specific templates for that step
func findStepTemplates(stepName string, platform config.Platform) (string, error) {
	base, err := baseLocation()
	if err != nil {
		return "", fmt.Errorf("error looking up step %s templates: %v", stepName, err)
	}
	for _, path := range []string{
		filepath.Join(base, stepsBaseDir, stepName, platformPath(platform)),
		filepath.Join(base, stepsBaseDir, stepName)} {

		stat, err := os.Stat(path)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return "", fmt.Errorf("invalid path for '%s' templates: %s", base, err)
		}
		if !stat.IsDir() {
			return "", fmt.Errorf("invalid path for '%s' templates", base)
		}
		return path, nil
	}
	return "", os.ErrNotExist
}

func platformPath(platform config.Platform) string {
	switch platform {
	case config.PlatformLibvirt:
		return "libvirt"
	case config.PlatformAWS:
		return "aws"
	}
	panic("invalid platform")
}

func generateClusterConfigMaps(m *metadata) error {
	clusterGeneratedPath := filepath.Join(m.clusterDir, generatedPath)
	if err := os.MkdirAll(clusterGeneratedPath, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create cluster generated directory at %s", clusterGeneratedPath)
	}

	configGenerator := configgenerator.New(m.cluster)

	kcoConfig, err := configGenerator.CoreConfig()
	if err != nil {
		return err
	}

	kcoConfigFilePath := filepath.Join(clusterGeneratedPath, kcoConfigFileName)
	if err := writeFile(kcoConfigFilePath, kcoConfig); err != nil {
		return err
	}

	tncoConfig, err := configGenerator.TncoConfig()
	if err != nil {
		return err
	}

	tncoConfigFilePath := filepath.Join(clusterGeneratedPath, tncoConfigFileName)
	if err := writeFile(tncoConfigFilePath, tncoConfig); err != nil {
		return err
	}

	kubeSystem, err := configGenerator.KubeSystem()
	if err != nil {
		return err
	}

	kubePath := filepath.Join(m.clusterDir, kubeSystemPath)
	if err := os.MkdirAll(kubePath, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create manifests directory at %s", kubePath)
	}

	kubeSystemConfigFilePath := filepath.Join(kubePath, kubeSystemFileName)
	if err := writeFile(kubeSystemConfigFilePath, kubeSystem); err != nil {
		return err
	}

	tectonicSystem, err := configGenerator.TectonicSystem()
	if err != nil {
		return err
	}

	tectonicPath := filepath.Join(m.clusterDir, tectonicSystemPath)
	if err := os.MkdirAll(tectonicPath, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create tectonic directory at %s", tectonicPath)
	}

	tectonicSystemConfigFilePath := filepath.Join(tectonicPath, tectonicSystemFileName)
	return writeFile(tectonicSystemConfigFilePath, tectonicSystem)
}

func readClusterConfig(configFilePath string, internalFilePath string) (*config.Cluster, error) {
	cfg, err := config.ParseConfigFile(configFilePath)
	if err != nil {
		return nil, fmt.Errorf("%s is not a valid config file: %s", configFilePath, err)
	}

	if internalFilePath != "" {
		internal, err := config.ParseInternalFile(internalFilePath)
		if err != nil {
			return nil, fmt.Errorf("%s is not a valid internal file: %s", internalFilePath, err)
		}
		cfg.Internal = *internal
	}

	return cfg, nil
}

func readClusterConfigStep(m *metadata) error {
	if m.clusterDir == "" {
		return errors.New("no cluster dir given for reading config")
	}
	configFilePath := filepath.Join(m.clusterDir, configFileName)
	internalFilePath := filepath.Join(m.clusterDir, internalFileName)

	cluster, err := readClusterConfig(configFilePath, internalFilePath)
	if err != nil {
		return err
	}

	if err := cluster.ValidateAndLog(); err != nil {
		return err
	}

	m.cluster = *cluster

	return nil
}

func writeFile(path, content string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	if _, err := fmt.Fprintln(w, content); err != nil {
		return err
	}
	w.Flush()

	return nil
}

func baseLocation() (string, error) {
	ex, err := os.Executable()
	if err != nil {
		return "", fmt.Errorf("undetermined location of own executable: %s", err)
	}
	ex = path.Dir(ex)
	if path.Base(ex) != binaryPrefix {
		return "", fmt.Errorf("%s executable in unknown location: %s", path.Base(ex), err)
	}
	return path.Dir(ex), nil
}

func clusterIsBootstrapped(stateDir string) bool {
	return hasStateFile(stateDir, topologyStep) &&
		hasStateFile(stateDir, mastersStep) &&
		hasStateFile(stateDir, tncDNSStep)
}

func createTNCCNAME(m *metadata) error {
	return runInstallStep(m, tncDNSStep, []string{bootstrapOn}...)
}

func createTNCARecord(m *metadata) error {
	return runInstallStep(m, tncDNSStep, []string{bootstrapOff}...)
}

func destroyTNCDNS(m *metadata) error {
	return runDestroyStep(m, tncDNSStep, []string{bootstrapOff}...)
}
<commit_msg>installer/pkg/workflow/utils: Simplify findStepTemplates path logic<commit_after>package workflow

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"

	"github.com/openshift/installer/installer/pkg/config"
	configgenerator "github.com/openshift/installer/installer/pkg/config-generator"
)

const (
	assetsStep       = "assets"
	binaryPrefix     = "installer"
	bootstrapOff     = "-var=tectonic_bootstrap=false"
	bootstrapOn      = "-var=tectonic_bootstrap=true"
	configFileName   = "config.yaml"
	etcdStep         = "etcd"
	internalFileName = "internal.yaml"
	joinWorkersStep  = "joining_workers"
	mastersStep      = "masters"
	newTLSStep       = "newtls"
	stepsBaseDir     = "steps"
	tlsStep          = "tls"
	tncDNSStep       = "tnc_dns"
	topologyStep     = "topology"
)

func copyFile(fromFilePath, toFilePath string) error {
	from, err := os.Open(fromFilePath)
	if err != nil {
		return err
	}
	defer from.Close()

	to, err := os.OpenFile(toFilePath, os.O_RDWR|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	defer to.Close()

	_, err = io.Copy(to, from)
	return err
}

// returns the directory containing templates for a given step. If platform is
// specified, it looks for a subdirectory with platform first, falling back if
// there are no platform-specific templates for that step
func findStepTemplates(stepName string, platform config.Platform) (string, error) {
	base, err := baseLocation()
	if err != nil {
		return "", fmt.Errorf("error looking up step %s templates: %v", stepName, err)
	}
	stepDir := filepath.Join(base, stepsBaseDir, stepName)
	for _, path := range []string{
		filepath.Join(stepDir, string(platform)),
		stepDir} {

		stat, err := os.Stat(path)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return "", fmt.Errorf("invalid path for '%s' templates: %s", base, err)
		}
		if !stat.IsDir() {
			return "", fmt.Errorf("invalid path for '%s' templates", base)
		}
		return path, nil
	}
	return "", os.ErrNotExist
}

func generateClusterConfigMaps(m *metadata) error {
	clusterGeneratedPath := filepath.Join(m.clusterDir, generatedPath)
	if err := os.MkdirAll(clusterGeneratedPath, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create cluster generated directory at %s", clusterGeneratedPath)
	}

	configGenerator := configgenerator.New(m.cluster)

	kcoConfig, err := configGenerator.CoreConfig()
	if err != nil {
		return err
	}

	kcoConfigFilePath := filepath.Join(clusterGeneratedPath, kcoConfigFileName)
	if err := writeFile(kcoConfigFilePath, kcoConfig); err != nil {
		return err
	}

	tncoConfig, err := configGenerator.TncoConfig()
	if err != nil {
		return err
	}

	tncoConfigFilePath := filepath.Join(clusterGeneratedPath, tncoConfigFileName)
	if err := writeFile(tncoConfigFilePath, tncoConfig); err != nil {
		return err
	}

	kubeSystem, err := configGenerator.KubeSystem()
	if err != nil {
		return err
	}

	kubePath := filepath.Join(m.clusterDir, kubeSystemPath)
	if err := os.MkdirAll(kubePath, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create manifests directory at %s", kubePath)
	}

	kubeSystemConfigFilePath := filepath.Join(kubePath, kubeSystemFileName)
	if err := writeFile(kubeSystemConfigFilePath, kubeSystem); err != nil {
		return err
	}

	tectonicSystem, err := configGenerator.TectonicSystem()
	if err != nil {
		return err
	}

	tectonicPath := filepath.Join(m.clusterDir, tectonicSystemPath)
	if err := os.MkdirAll(tectonicPath, os.ModeDir|0755); err != nil {
		return fmt.Errorf("Failed to create tectonic directory at %s", tectonicPath)
	}

	tectonicSystemConfigFilePath := filepath.Join(tectonicPath, tectonicSystemFileName)
	return writeFile(tectonicSystemConfigFilePath, tectonicSystem)
}

func readClusterConfig(configFilePath string, internalFilePath string) (*config.Cluster, error) {
	cfg, err := config.ParseConfigFile(configFilePath)
	if err != nil {
		return nil, fmt.Errorf("%s is not a valid config file: %s", configFilePath, err)
	}

	if internalFilePath != "" {
		internal, err := config.ParseInternalFile(internalFilePath)
		if err != nil {
			return nil, fmt.Errorf("%s is not a valid internal file: %s", internalFilePath, err)
		}
		cfg.Internal = *internal
	}

	return cfg, nil
}

func readClusterConfigStep(m *metadata) error {
	if m.clusterDir == "" {
		return errors.New("no cluster dir given for reading config")
	}
	configFilePath := filepath.Join(m.clusterDir, configFileName)
	internalFilePath := filepath.Join(m.clusterDir, internalFileName)

	cluster, err := readClusterConfig(configFilePath, internalFilePath)
	if err != nil {
		return err
	}

	if err := cluster.ValidateAndLog(); err != nil {
		return err
	}

	m.cluster = *cluster

	return nil
}

func writeFile(path, content string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	if _, err := fmt.Fprintln(w, content); err != nil {
		return err
	}
	w.Flush()

	return nil
}

func baseLocation() (string, error) {
	ex, err := os.Executable()
	if err != nil {
		return "", fmt.Errorf("undetermined location of own executable: %s", err)
	}
	ex = path.Dir(ex)
	if path.Base(ex) != binaryPrefix {
		return "", fmt.Errorf("%s executable in unknown location: %s", path.Base(ex), err)
	}
	return path.Dir(ex), nil
}

func clusterIsBootstrapped(stateDir string) bool {
	return hasStateFile(stateDir, topologyStep) &&
		hasStateFile(stateDir, mastersStep) &&
		hasStateFile(stateDir, tncDNSStep)
}

func createTNCCNAME(m *metadata) error {
	return runInstallStep(m, tncDNSStep, []string{bootstrapOn}...)
}

func createTNCARecord(m *metadata) error {
	return runInstallStep(m, tncDNSStep, []string{bootstrapOff}...)
}

func destroyTNCDNS(m *metadata) error {
	return runDestroyStep(m, tncDNSStep, []string{bootstrapOff}...)
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package metrics

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"testing/quick"
	"time"

	"github.com/google/mtail/internal/metrics/datum"
	"github.com/google/mtail/internal/testutil"
)

func TestKindType(t *testing.T) {
	v := Kind(0)
	if s := v.String(); s != "Unknown" {
		t.Errorf("Kind.String() returned %q not Unknown", s)
	}
	v = Counter
	if s := v.String(); s != "Counter" {
		t.Errorf("Kind.String() returned %q not Counter", s)
	}
	v = Gauge
	if s := v.String(); s != "Gauge" {
		t.Errorf("Kind.String() returned %q not Gauge", s)
	}
	v = Timer
	if s := v.String(); s != "Timer" {
		t.Errorf("Kind.String() returned %q not Timer", s)
	}
}

func TestScalarMetric(t *testing.T) {
	v := NewMetric("test", "prog", Counter, Int)
	d, err := v.GetDatum()
	if err != nil {
		t.Errorf("no datum: %s", err)
	}
	datum.IncIntBy(d, 1, time.Now().UTC())
	lv := v.FindLabelValueOrNil([]string{})
	if lv == nil {
		t.Errorf("couldn't find labelvalue")
	}
	newD := lv.Value
	if newD == nil {
		t.Errorf("new_d is nil")
	}
	if newD.ValueString() != "1" {
		t.Errorf("value not 1")
	}
	d2, err := v.GetDatum("a", "b")
	if err == nil {
		t.Errorf("datum with keys sohuld have returned no value, got %v", d2)
	}
}

func TestDimensionedMetric(t *testing.T) {
	v := NewMetric("test", "prog", Counter, Int, "foo")
	d, _ := v.GetDatum("a")
	datum.IncIntBy(d, 1, time.Now().UTC())
	if v.FindLabelValueOrNil([]string{"a"}).Value.ValueString() != "1" {
		t.Errorf("fail")
	}

	v = NewMetric("test", "prog", Counter, Int, "foo", "bar")
	d, _ = v.GetDatum("a", "b")
	datum.IncIntBy(d, 1, time.Now().UTC())
	if v.FindLabelValueOrNil([]string{"a", "b"}).Value.ValueString() != "1" {
		t.Errorf("fail")
	}

	v = NewMetric("test", "prog", Counter, Int, "foo", "bar", "quux")
	d, _ = v.GetDatum("a", "b", "c")
	datum.IncIntBy(d, 1, time.Now().UTC())
	if v.FindLabelValueOrNil([]string{"a", "b", "c"}).Value.ValueString() != "1" {
		t.Errorf("fail")
	}
}

var labelSetTests = []struct {
	values         []string
	expectedLabels map[string]string
}{
	{
		[]string{"a", "b", "c"},
		map[string]string{"foo": "a", "bar": "b", "quux": "c"},
	},
	{
		[]string{"a", "b", "d"},
		map[string]string{"foo": "a", "bar": "b", "quux": "d"},
	},
}

func TestEmitLabelSet(t *testing.T) {
	ts := time.Now().UTC()
	for _, tc := range labelSetTests {
		tc := tc
		t.Run(fmt.Sprintf("%v", tc.values), func(t *testing.T) {
			t.Parallel()
			m := NewMetric("test", "prog", Gauge, Int, "foo", "bar", "quux")
			d, _ := m.GetDatum(tc.values...)
			datum.SetInt(d, 37, ts)

			c := make(chan *LabelSet)

			go m.EmitLabelSets(c)

			ls := <-c

			diff := testutil.Diff(tc.expectedLabels, ls.Labels)
			if diff != "" {
				t.Error(diff)
			}
		})
	}
}

func TestFindLabelValueOrNil(t *testing.T) {
	m0 := NewMetric("foo", "prog", Counter, Int)
	if r0 := m0.FindLabelValueOrNil([]string{}); r0 != nil {
		t.Errorf("m0 should be nil: %v", r0)
	}
	d, err := m0.GetDatum()
	if err != nil {
		t.Errorf("Bad datum %v: %v\n", d, err)
	}
	if r1 := m0.FindLabelValueOrNil([]string{}); r1 == nil {
		t.Errorf("m0 should not be nil: %v", r1)
	}
	m1 := NewMetric("bar", "prog", Counter, Int, "a")
	d1, err1 := m1.GetDatum("1")
	if err1 != nil {
		t.Errorf("err1 %v: %v\n", d1, err1)
	}
	if r2 := m1.FindLabelValueOrNil([]string{"0"}); r2 != nil {
		t.Errorf("r2 should be nil")
	}
	if r3 := m1.FindLabelValueOrNil([]string{"1"}); r3 == nil {
		t.Errorf("r3 should be non nil")
	}
}

func timeGenerator(rand *rand.Rand) time.Time {
	months := []time.Month{
		time.January, time.February, time.March,
		time.April, time.May, time.June,
		time.July, time.August, time.September,
		time.October, time.November, time.December,
	}

	return time.Date(
		rand.Intn(9999),
		months[rand.Intn(len(months))],
		rand.Intn(31),
		rand.Intn(24),
		rand.Intn(60),
		rand.Intn(60),
		int(rand.Int31()),
		time.UTC,
	)
}

func TestMetricJSONRoundTrip(t *testing.T) {
	rand := rand.New(rand.NewSource(0))
	f := func(name, prog string, kind Kind, keys []string, val, ti, tns int64) bool {
		m := NewMetric(name, prog, kind, Int, keys...)
		labels := make([]string, 0)
		for range keys {
			if l, ok := quick.Value(reflect.TypeOf(name), rand); ok {
				labels = append(labels, l.String())
			} else {
				t.Errorf("failed to create value for labels")
				break
			}
		}
		d, _ := m.GetDatum(labels...)
		datum.SetInt(d, val, timeGenerator(rand))

		j, e := json.Marshal(m)
		if e != nil {
			t.Errorf("json.Marshal failed: %s\n", e)
			return false
		}

		r := newMetric(0)
		e = json.Unmarshal(j, &r)
		if e != nil {
			t.Errorf("json.Unmarshal failed: %s\n", e)
			return false
		}

		if diff := testutil.Diff(m, r, testutil.IgnoreUnexported(sync.RWMutex{})); diff != "" {
			t.Errorf("Round trip wasn't stable:\n%s", diff)
			return false
		}
		return true
	}
	q := quick.Config{MaxCount: 100000}
	if testing.Short() {
		q.MaxCount = 1000
	}
	if err := quick.Check(f, nil); err != nil {
		t.Error(err)
	}
}

func TestTimer(t *testing.T) {
	m := NewMetric("test", "prog", Timer, Int)
	n := NewMetric("test", "prog", Timer, Int)
	diff := testutil.Diff(m, n, testutil.IgnoreUnexported(sync.RWMutex{}))
	if diff != "" {
		t.Errorf("Identical metrics not the same:\n%s", diff)
	}
	d, _ := m.GetDatum()
	datum.IncIntBy(d, 1, time.Now().UTC())
	lv := m.FindLabelValueOrNil([]string{})
	if lv == nil {
		t.Errorf("couldn't find labelvalue")
	}
	newD := lv.Value
	if newD == nil {
		t.Errorf("new_d is nil")
	}
	if newD.ValueString() != "1" {
		t.Errorf("value not 1")
	}
}

func TestRemoveMetricLabelValue(t *testing.T) {
	m := NewMetric("test", "prog", Counter, Int, "a", "b", "c")
	_, e := m.GetDatum("a", "a", "a")
	if e != nil {
		t.Errorf("Getdatum failed: %s", e)
	}
	lv := m.FindLabelValueOrNil([]string{"a", "a", "a"})
	if lv == nil {
		t.Errorf("coidln't find labelvalue")
	}
	e = m.RemoveDatum("a", "a", "a")
	if e != nil {
		t.Errorf("couldn't remove datum: %s", e)
	}
	lv = m.FindLabelValueOrNil([]string{"a", "a", "a"})
	if lv != nil {
		t.Errorf("label value still exists")
	}
}
<commit_msg>Fix quick.Check config in metric_test.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package metrics

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"testing/quick"
	"time"

	"github.com/google/mtail/internal/metrics/datum"
	"github.com/google/mtail/internal/testutil"
)

func TestKindType(t *testing.T) {
	v := Kind(0)
	if s := v.String(); s != "Unknown" {
		t.Errorf("Kind.String() returned %q not Unknown", s)
	}
	v = Counter
	if s := v.String(); s != "Counter" {
		t.Errorf("Kind.String() returned %q not Counter", s)
	}
	v = Gauge
	if s := v.String(); s != "Gauge" {
		t.Errorf("Kind.String() returned %q not Gauge", s)
	}
	v = Timer
	if s := v.String(); s != "Timer" {
		t.Errorf("Kind.String() returned %q not Timer", s)
	}
}

func TestScalarMetric(t *testing.T) {
	v := NewMetric("test", "prog", Counter, Int)
	d, err := v.GetDatum()
	if err != nil {
		t.Errorf("no datum: %s", err)
	}
	datum.IncIntBy(d, 1, time.Now().UTC())
	lv := v.FindLabelValueOrNil([]string{})
	if lv == nil {
		t.Errorf("couldn't find labelvalue")
	}
	newD := lv.Value
	if newD == nil {
		t.Errorf("new_d is nil")
	}
	if newD.ValueString() != "1" {
		t.Errorf("value not 1")
	}
	d2, err := v.GetDatum("a", "b")
	if err == nil {
		t.Errorf("datum with keys sohuld have returned no value, got %v", d2)
	}
}

func TestDimensionedMetric(t *testing.T) {
	v := NewMetric("test", "prog", Counter, Int, "foo")
	d, _ := v.GetDatum("a")
	datum.IncIntBy(d, 1, time.Now().UTC())
	if v.FindLabelValueOrNil([]string{"a"}).Value.ValueString() != "1" {
		t.Errorf("fail")
	}

	v = NewMetric("test", "prog", Counter, Int, "foo", "bar")
	d, _ = v.GetDatum("a", "b")
	datum.IncIntBy(d, 1, time.Now().UTC())
	if v.FindLabelValueOrNil([]string{"a", "b"}).Value.ValueString() != "1" {
		t.Errorf("fail")
	}

	v = NewMetric("test", "prog", Counter, Int, "foo", "bar", "quux")
	d, _ = v.GetDatum("a", "b", "c")
	datum.IncIntBy(d, 1, time.Now().UTC())
	if v.FindLabelValueOrNil([]string{"a", "b", "c"}).Value.ValueString() != "1" {
		t.Errorf("fail")
	}
}

var labelSetTests = []struct {
	values         []string
	expectedLabels map[string]string
}{
	{
		[]string{"a", "b", "c"},
		map[string]string{"foo": "a", "bar": "b", "quux": "c"},
	},
	{
		[]string{"a", "b", "d"},
		map[string]string{"foo": "a", "bar": "b", "quux": "d"},
	},
}

func TestEmitLabelSet(t *testing.T) {
	ts := time.Now().UTC()
	for _, tc := range labelSetTests {
		tc := tc
		t.Run(fmt.Sprintf("%v", tc.values), func(t *testing.T) {
			t.Parallel()
			m := NewMetric("test", "prog", Gauge, Int, "foo", "bar", "quux")
			d, _ := m.GetDatum(tc.values...)
			datum.SetInt(d, 37, ts)

			c := make(chan *LabelSet)

			go m.EmitLabelSets(c)

			ls := <-c

			diff := testutil.Diff(tc.expectedLabels, ls.Labels)
			if diff != "" {
				t.Error(diff)
			}
		})
	}
}

func TestFindLabelValueOrNil(t *testing.T) {
	m0 := NewMetric("foo", "prog", Counter, Int)
	if r0 := m0.FindLabelValueOrNil([]string{}); r0 != nil {
		t.Errorf("m0 should be nil: %v", r0)
	}
	d, err := m0.GetDatum()
	if err != nil {
		t.Errorf("Bad datum %v: %v\n", d, err)
	}
	if r1 := m0.FindLabelValueOrNil([]string{}); r1 == nil {
		t.Errorf("m0 should not be nil: %v", r1)
	}
	m1 := NewMetric("bar", "prog", Counter, Int, "a")
	d1, err1 := m1.GetDatum("1")
	if err1 != nil {
		t.Errorf("err1 %v: %v\n", d1, err1)
	}
	if r2 := m1.FindLabelValueOrNil([]string{"0"}); r2 != nil {
		t.Errorf("r2 should be nil")
	}
	if r3 := m1.FindLabelValueOrNil([]string{"1"}); r3 == nil {
		t.Errorf("r3 should be non nil")
	}
}

func timeGenerator(rand *rand.Rand) time.Time {
	months := []time.Month{
		time.January, time.February, time.March,
		time.April, time.May, time.June,
		time.July, time.August, time.September,
		time.October, time.November, time.December,
	}

	return time.Date(
		rand.Intn(9999),
		months[rand.Intn(len(months))],
		rand.Intn(31),
		rand.Intn(24),
		rand.Intn(60),
		rand.Intn(60),
		int(rand.Int31()),
		time.UTC,
	)
}

func TestMetricJSONRoundTrip(t *testing.T) {
	rand := rand.New(rand.NewSource(0))
	f := func(name, prog string, kind Kind, keys []string, val, ti, tns int64) bool {
		m := NewMetric(name, prog, kind, Int, keys...)
		labels := make([]string, 0)
		for range keys {
			if l, ok := quick.Value(reflect.TypeOf(name), rand); ok {
				labels = append(labels, l.String())
			} else {
				t.Errorf("failed to create value for labels")
				break
			}
		}
		d, _ := m.GetDatum(labels...)
		datum.SetInt(d, val, timeGenerator(rand))

		j, e := json.Marshal(m)
		if e != nil {
			t.Errorf("json.Marshal failed: %s\n", e)
			return false
		}

		r := newMetric(0)
		e = json.Unmarshal(j, &r)
		if e != nil {
			t.Errorf("json.Unmarshal failed: %s\n", e)
			return false
		}

		if diff := testutil.Diff(m, r, testutil.IgnoreUnexported(sync.RWMutex{})); diff != "" {
			t.Errorf("Round trip wasn't stable:\n%s", diff)
			return false
		}
		return true
	}
	q := &quick.Config{MaxCount: 100000}
	if testing.Short() {
		q.MaxCountScale = 0.01
	}
	if err := quick.Check(f, q); err != nil {
		t.Error(err)
	}
}

func TestTimer(t *testing.T) {
	m := NewMetric("test", "prog", Timer, Int)
	n := NewMetric("test", "prog", Timer, Int)
	diff := testutil.Diff(m, n, testutil.IgnoreUnexported(sync.RWMutex{}))
	if diff != "" {
		t.Errorf("Identical metrics not the same:\n%s", diff)
	}
	d, _ := m.GetDatum()
	datum.IncIntBy(d, 1, time.Now().UTC())
	lv := m.FindLabelValueOrNil([]string{})
	if lv == nil {
		t.Errorf("couldn't find labelvalue")
	}
	newD := lv.Value
	if newD == nil {
		t.Errorf("new_d is nil")
	}
	if newD.ValueString() != "1" {
		t.Errorf("value not 1")
	}
}

func TestRemoveMetricLabelValue(t *testing.T) {
	m := NewMetric("test", "prog", Counter, Int, "a", "b", "c")
	_, e := m.GetDatum("a", "a", "a")
	if e != nil {
		t.Errorf("Getdatum failed: %s", e)
	}
	lv := m.FindLabelValueOrNil([]string{"a", "a", "a"})
	if lv == nil {
		t.Errorf("coidln't find labelvalue")
	}
	e = m.RemoveDatum("a", "a", "a")
	if e != nil {
		t.Errorf("couldn't remove datum: %s", e)
	}
	lv = m.FindLabelValueOrNil([]string{"a", "a", "a"})
	if lv != nil {
		t.Errorf("label value still exists")
	}
}
<|endoftext|>


<commit_before>// Package nametransform encrypts and decrypts filenames.
package nametransform

import (
	"bytes"
	"crypto/aes"
	"encoding/base64"
	"path/filepath"
	"syscall"

	"github.com/rfjakob/eme"

	"github.com/rfjakob/gocryptfs/internal/tlog"
)

const (
	// Like ext4, we allow at most 255 bytes for a file name.
	NameMax = 255
)

// NameTransformer is an interface used to transform filenames.
type NameTransformer interface {
	DecryptName(cipherName string, iv []byte) (string, error)
	EncryptName(plainName string, iv []byte) string
	EncryptAndHashName(name string, iv []byte) (string, error)
	HashLongName(name string) string
	WriteLongNameAt(dirfd int, hashName string, plainName string) error
	B64EncodeToString(src []byte) string
	B64DecodeString(s string) ([]byte, error)
}

// NameTransform is used to transform filenames.
type NameTransform struct {
	emeCipher *eme.EMECipher
	longNames bool
	// B64 = either base64.URLEncoding or base64.RawURLEncoding, depending
	// on the Raw64 feature flag
	B64 *base64.Encoding
	// Patterns to bypass decryption
	BadnamePatterns []string
}

// New returns a new NameTransform instance.
func New(e *eme.EMECipher, longNames bool, raw64 bool) *NameTransform {
	b64 := base64.URLEncoding
	if raw64 {
		b64 = base64.RawURLEncoding
	}
	return &NameTransform{
		emeCipher: e,
		longNames: longNames,
		B64:       b64,
	}
}

// DecryptName calls decryptName to try and decrypt a base64-encoded encrypted
// filename "cipherName", and failing that checks if it can be bypassed
func (n *NameTransform) DecryptName(cipherName string, iv []byte) (string, error) {
	res, err := n.decryptName(cipherName, iv)
	if err != nil {
		for _, pattern := range n.BadnamePatterns {
			match, err := filepath.Match(pattern, cipherName)
			if err == nil && match { // Pattern should have been validated already
				//find longest decryptable substring
				for charpos := len(cipherName) - 1; charpos > 0; charpos-- {
					res, err = n.decryptName(cipherName[:charpos], iv)
					if err == nil {
						return res + cipherName[charpos:] + " GOCRYPTFS_BAD_NAME", nil
					}
				}
				return cipherName + " GOCRYPTFS_BAD_NAME", nil
			}
		}
	}
	return res, err
}

// decryptName decrypts a base64-encoded encrypted filename "cipherName" using the
// initialization vector "iv".
func (n *NameTransform) decryptName(cipherName string, iv []byte) (string, error) {
	bin, err := n.B64.DecodeString(cipherName)
	if err != nil {
		return "", err
	}
	if len(bin) == 0 {
		tlog.Warn.Printf("DecryptName: empty input")
		return "", syscall.EBADMSG
	}
	if len(bin)%aes.BlockSize != 0 {
		tlog.Debug.Printf("DecryptName %q: decoded length %d is not a multiple of 16", cipherName, len(bin))
		return "", syscall.EBADMSG
	}
	bin = n.emeCipher.Decrypt(iv, bin)
	bin, err = unPad16(bin)
	if err != nil {
		tlog.Debug.Printf("DecryptName: unPad16 error detail: %v", err)
		// unPad16 returns detailed errors including the position of the
		// incorrect bytes. Kill the padding oracle by lumping everything into
		// a generic error.
		return "", syscall.EBADMSG
	}
	// A name can never contain a null byte or "/". Make sure we never return those
	// to the kernel, even when we read a corrupted (or fuzzed) filesystem.
	if bytes.Contains(bin, []byte{0}) || bytes.Contains(bin, []byte("/")) {
		return "", syscall.EBADMSG
	}
	// The name should never be "." or "..".
	if bytes.Equal(bin, []byte(".")) || bytes.Equal(bin, []byte("..")) {
		return "", syscall.EBADMSG
	}
	plain := string(bin)
	return plain, err
}

// EncryptName encrypts "plainName", returns a base64-encoded "cipherName64",
// encrypted using EME (https://github.com/rfjakob/eme).
//
// This function is exported because in some cases, fusefrontend needs access
// to the full (not hashed) name if longname is used.
func (n *NameTransform) EncryptName(plainName string, iv []byte) (cipherName64 string) {
	bin := []byte(plainName)
	bin = pad16(bin)
	bin = n.emeCipher.Encrypt(iv, bin)
	cipherName64 = n.B64.EncodeToString(bin)
	return cipherName64
}

// B64EncodeToString returns a Base64-encoded string
func (n *NameTransform) B64EncodeToString(src []byte) string {
	return n.B64.EncodeToString(src)
}

// B64DecodeString decodes a Base64-encoded string
func (n *NameTransform) B64DecodeString(s string) ([]byte, error) {
	return n.B64.DecodeString(s)
}
<commit_msg>badname: stop trying to decrypt at length 22<commit_after>// Package nametransform encrypts and decrypts filenames.
package nametransform

import (
	"bytes"
	"crypto/aes"
	"encoding/base64"
	"path/filepath"
	"syscall"

	"github.com/rfjakob/eme"

	"github.com/rfjakob/gocryptfs/internal/tlog"
)

const (
	// Like ext4, we allow at most 255 bytes for a file name.
	NameMax = 255
)

// NameTransformer is an interface used to transform filenames.
type NameTransformer interface {
	DecryptName(cipherName string, iv []byte) (string, error)
	EncryptName(plainName string, iv []byte) string
	EncryptAndHashName(name string, iv []byte) (string, error)
	HashLongName(name string) string
	WriteLongNameAt(dirfd int, hashName string, plainName string) error
	B64EncodeToString(src []byte) string
	B64DecodeString(s string) ([]byte, error)
}

// NameTransform is used to transform filenames.
type NameTransform struct {
	emeCipher *eme.EMECipher
	longNames bool
	// B64 = either base64.URLEncoding or base64.RawURLEncoding, depending
	// on the Raw64 feature flag
	B64 *base64.Encoding
	// Patterns to bypass decryption
	BadnamePatterns []string
}

// New returns a new NameTransform instance.
func New(e *eme.EMECipher, longNames bool, raw64 bool) *NameTransform {
	b64 := base64.URLEncoding
	if raw64 {
		b64 = base64.RawURLEncoding
	}
	return &NameTransform{
		emeCipher: e,
		longNames: longNames,
		B64:       b64,
	}
}

// DecryptName calls decryptName to try and decrypt a base64-encoded encrypted
// filename "cipherName", and failing that checks if it can be bypassed
func (n *NameTransform) DecryptName(cipherName string, iv []byte) (string, error) {
	res, err := n.decryptName(cipherName, iv)
	if err != nil {
		for _, pattern := range n.BadnamePatterns {
			match, err := filepath.Match(pattern, cipherName)
			if err == nil && match { // Pattern should have been validated already
				// Find longest decryptable substring
				// At least 16 bytes due to AES --> at least 22 characters in base64
				nameMin := n.B64.EncodedLen(aes.BlockSize)
				for charpos := len(cipherName) - 1; charpos >= nameMin; charpos-- {
					res, err = n.decryptName(cipherName[:charpos], iv)
					if err == nil {
						return res + cipherName[charpos:] + " GOCRYPTFS_BAD_NAME", nil
					}
				}
				return cipherName + " GOCRYPTFS_BAD_NAME", nil
			}
		}
	}
	return res, err
}

// decryptName decrypts a base64-encoded encrypted filename "cipherName" using the
// initialization vector "iv".
func (n *NameTransform) decryptName(cipherName string, iv []byte) (string, error) {
	bin, err := n.B64.DecodeString(cipherName)
	if err != nil {
		return "", err
	}
	if len(bin) == 0 {
		tlog.Warn.Printf("DecryptName: empty input")
		return "", syscall.EBADMSG
	}
	if len(bin)%aes.BlockSize != 0 {
		tlog.Debug.Printf("DecryptName %q: decoded length %d is not a multiple of 16", cipherName, len(bin))
		return "", syscall.EBADMSG
	}
	bin = n.emeCipher.Decrypt(iv, bin)
	bin, err = unPad16(bin)
	if err != nil {
		tlog.Debug.Printf("DecryptName: unPad16 error detail: %v", err)
		// unPad16 returns detailed errors including the position of the
		// incorrect bytes. Kill the padding oracle by lumping everything into
		// a generic error.
		return "", syscall.EBADMSG
	}
	// A name can never contain a null byte or "/". Make sure we never return those
	// to the kernel, even when we read a corrupted (or fuzzed) filesystem.
	if bytes.Contains(bin, []byte{0}) || bytes.Contains(bin, []byte("/")) {
		return "", syscall.EBADMSG
	}
	// The name should never be "." or "..".
	if bytes.Equal(bin, []byte(".")) || bytes.Equal(bin, []byte("..")) {
		return "", syscall.EBADMSG
	}
	plain := string(bin)
	return plain, err
}

// EncryptName encrypts "plainName", returns a base64-encoded "cipherName64",
// encrypted using EME (https://github.com/rfjakob/eme).
//
// This function is exported because in some cases, fusefrontend needs access
// to the full (not hashed) name if longname is used.
func (n *NameTransform) EncryptName(plainName string, iv []byte) (cipherName64 string) {
	bin := []byte(plainName)
	bin = pad16(bin)
	bin = n.emeCipher.Encrypt(iv, bin)
	cipherName64 = n.B64.EncodeToString(bin)
	return cipherName64
}

// B64EncodeToString returns a Base64-encoded string
func (n *NameTransform) B64EncodeToString(src []byte) string {
	return n.B64.EncodeToString(src)
}

// B64DecodeString decodes a Base64-encoded string
func (n *NameTransform) B64DecodeString(s string) ([]byte, error) {
	return n.B64.DecodeString(s)
}
<|endoftext|>