<commit_before>package caching

import (
	"fmt"
	"github.com/boltdb/bolt"
	log "github.com/cloudfoundry-community/firehose-to-syslog/logging"
	cfClient "github.com/cloudfoundry-community/go-cfclient"
	json "github.com/pquerna/ffjson/ffjson"
)

type App struct {
	Name      string
	Guid      string
	SpaceName string
	SpaceGuid string
	OrgName   string
	OrgGuid   string
}

var gcfClient *cfClient.Client
var appdb *bolt.DB

func CreateBucket() {
	appdb.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		return nil

	})

}

func FillDatabase(listApps []App) {

	for _, app := range listApps {
		appdb.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
			if err != nil {
				return fmt.Errorf("create bucket: %s", err)
			}

			serialize, err := json.Marshal(app)

			if err != nil {
				return fmt.Errorf("Error Marshaling data: %s", err)
			}
			err = b.Put([]byte(app.Guid), serialize)

			if err != nil {
				return fmt.Errorf("Error inserting data: %s", err)
			}
			return nil
		})

	}

}

func GetAppByGuid(appGuid string) []App {
	var apps []App
	app := gcfClient.AppByGuid(appGuid)
	apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
	FillDatabase(apps)
	return apps

}

func GetAllApp() []App {

	log.LogStd("Retrieving Apps for Cache...", false)
	var apps []App

	defer func() {
		if r := recover(); r != nil {
			log.LogError("Recovered in caching.GetAllApp()", r)
		}
	}()

	for _, app := range gcfClient.ListApps() {
		log.LogStd(fmt.Sprintf("App [%s] Found...", app.Name), false)
		apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
	}

	FillDatabase(apps)

	log.LogStd(fmt.Sprintf("Found [%d] Apps!", len(apps)), false)

	return apps
}

func GetAppInfo(appGuid string) App {

	defer func() {
		if r := recover(); r != nil {
			log.LogError(fmt.Sprintf("Recovered from panic retrieving App Info for App Guid: %s", appGuid), r)
		}
	}()

	var d []byte
	var app App
	appdb.View(func(tx *bolt.Tx) error {
		log.LogStd(fmt.Sprintf("Looking for App %s in Cache!\n", appGuid), false)
		b := tx.Bucket([]byte("AppBucket"))
		d = b.Get([]byte(appGuid))
		return nil
	})
	err := json.Unmarshal([]byte(d), &app)
	if err != nil {
		return App{}
	}
	return app
}

func SetCfClient(cfClient *cfClient.Client) {
	gcfClient = cfClient

}

func SetAppDb(db *bolt.DB) {
	appdb = db
}
<commit_msg>Fix potential panic<commit_after>package caching

import (
	"fmt"
	"github.com/boltdb/bolt"
	log "github.com/cloudfoundry-community/firehose-to-syslog/logging"
	cfClient "github.com/cloudfoundry-community/go-cfclient"
	json "github.com/pquerna/ffjson/ffjson"
)

type App struct {
	Name      string
	Guid      string
	SpaceName string
	SpaceGuid string
	OrgName   string
	OrgGuid   string
}

var gcfClient *cfClient.Client
var appdb *bolt.DB

func CreateBucket() {
	appdb.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		return nil

	})

}

func FillDatabase(listApps []App) {

	for _, app := range listApps {
		appdb.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte("AppBucket"))
			if err != nil {
				return fmt.Errorf("create bucket: %s", err)
			}

			serialize, err := json.Marshal(app)

			if err != nil {
				return fmt.Errorf("Error Marshaling data: %s", err)
			}
			err = b.Put([]byte(app.Guid), serialize)

			if err != nil {
				return fmt.Errorf("Error inserting data: %s", err)
			}
			return nil
		})

	}

}

func GetAppByGuid(appGuid string) []App {
	var apps []App
	app := gcfClient.AppByGuid(appGuid)
	apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
	FillDatabase(apps)
	return apps

}

func GetAllApp() []App {

	log.LogStd("Retrieving Apps for Cache...", false)
	var apps []App

	defer func() {
		if r := recover(); r != nil {
			log.LogError("Recovered in caching.GetAllApp()", r)
		}
	}()

	for _, app := range gcfClient.ListApps() {
		log.LogStd(fmt.Sprintf("App [%s] Found...", app.Name), false)
		apps = append(apps, App{app.Name, app.Guid, app.SpaceData.Entity.Name, app.SpaceData.Entity.Guid, app.SpaceData.Entity.OrgData.Entity.Name, app.SpaceData.Entity.OrgData.Entity.Guid})
	}

	FillDatabase(apps)

	log.LogStd(fmt.Sprintf("Found [%d] Apps!", len(apps)), false)

	return apps
}

func GetAppInfo(appGuid string) App {

	defer func() {
		if r := recover(); r != nil {
			log.LogError(fmt.Sprintf("Recovered from panic retrieving App Info for App Guid: %s", appGuid), r)
		}
	}()

	var d []byte
	var app App
	appdb.View(func(tx *bolt.Tx) error {
		log.LogStd(fmt.Sprintf("Looking for App %s in Cache!\n", appGuid), false)
		b := tx.Bucket([]byte("AppBucket"))
		d = cloneBytes(b.Get([]byte(appGuid)))
		return nil
	})
	err := json.Unmarshal([]byte(d), &app)
	if err != nil {
		return App{}
	}
	return app
}

// cloneBytes returns a copy of a given slice.
func cloneBytes(v []byte) []byte {
	var clone = make([]byte, len(v))
	copy(clone, v)
	return clone
}

func SetCfClient(cfClient *cfClient.Client) {
	gcfClient = cfClient

}

func SetAppDb(db *bolt.DB) {
	appdb = db
}
<|endoftext|>


<commit_before>// Package attachconsole is meant to be used as a side-effect only
// import. It attaches the calling process to its parent's console.
// It is most useful for an application built with
//	-ldflags "-H windowsgui"
// flags so that it won't have a console of itself. After importing
// this package, if the application is run from the command prompt
// window, the console output will appear in the command prompt window
// instead of discarded.
package attachconsole // import "github.com/minux/windows/attachconsole"

import "syscall"

func attach() error {
	const ATTACH_PARENT_PROCESS = ^uintptr(0)
	proc := syscall.MustLoadDLL("kernel32.dll").MustFindProc("AttachConsole")
	_, _, err := proc.Call(ATTACH_PARENT_PROCESS)
	return err
}

func init() {
	attach()
}
<commit_msg>attachconsole: add copyright header<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package attachconsole is meant to be used as a side-effect only
// import. It attaches the calling process to its parent's console.
// It is most useful for an application built with
//	-ldflags "-H windowsgui"
// flags so that it won't have a console of itself. After importing
// this package, if the application is run from the command prompt
// window, the console output will appear in the command prompt window
// instead of discarded.
package attachconsole // import "github.com/minux/windows/attachconsole"

import "syscall"

func attach() error {
	const ATTACH_PARENT_PROCESS = ^uintptr(0)
	proc := syscall.MustLoadDLL("kernel32.dll").MustFindProc("AttachConsole")
	_, _, err := proc.Call(ATTACH_PARENT_PROCESS)
	return err
}

func init() {
	attach()
}
<|endoftext|>


<commit_before>// Copyright © 2017 Makoto Ito
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileio

import (
	"bufio"
	"io"
	"os"
	"strings"
)

const (
	defaultBatch   = 10000
	defaultBufSize = 4096
)

var (
	rr *bufio.Reader
)

// Load reads the text in input file.
func Load(inputFile string, f func(words []string)) error {
	return loadBatch(inputFile, defaultBatch, toLowerWords(f))
}

func toLowerWords(f func(words []string)) func(sentences []string) {
	return func(lines []string) {
		for _, l := range lines {
			words := strings.Fields(strings.ToLower(l))
			f(words)
		}
	}
}

func loadBatch(inputFile string, batch int, f func(sentences []string)) error {
	file, err := os.Open(inputFile)

	if err != nil {
		return err
	}
	defer file.Close()

	rr = bufio.NewReader(file)

	j := 0
	for {
		lines := make([]string, batch)
		for i := 1; i <= batch; i++ {
			line, err := readLine()
			if err == io.EOF {
				f(lines)
				return nil
			} else if err != nil {
				return err
			}
			lines = append(lines, line)
		}
		f(lines)
		j++
	}
}

func readLine() (string, error) {
	buf := make([]byte, 0, defaultBufSize)
	for {
		line, isPrefix, err := rr.ReadLine()
		if err != nil {
			return "", err
		}

		buf = append(buf, line...)
		if !isPrefix {
			break
		}
	}
	return string(buf), nil
}
<commit_msg>Fix slice size to 0<commit_after>// Copyright © 2017 Makoto Ito
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileio

import (
	"bufio"
	"io"
	"os"
	"strings"
)

const (
	defaultBatch   = 10000
	defaultBufSize = 4096
)

var (
	rr *bufio.Reader
)

// Load reads the text in input file.
func Load(inputFile string, f func(words []string)) error {
	return loadBatch(inputFile, defaultBatch, toLowerWords(f))
}

func toLowerWords(f func(words []string)) func(sentences []string) {
	return func(lines []string) {
		for _, l := range lines {
			words := strings.Fields(strings.ToLower(l))
			f(words)
		}
	}
}

func loadBatch(inputFile string, batch int, f func(sentences []string)) error {
	file, err := os.Open(inputFile)

	if err != nil {
		return err
	}
	defer file.Close()

	rr = bufio.NewReader(file)

	j := 0
	for {
		lines := make([]string, 0)
		for i := 1; i <= batch; i++ {
			line, err := readLine()
			if err == io.EOF {
				f(lines)
				return nil
			} else if err != nil {
				return err
			}
			lines = append(lines, line)
		}
		f(lines)
		j++
	}
}

func readLine() (string, error) {
	buf := make([]byte, 0, defaultBufSize)
	for {
		line, isPrefix, err := rr.ReadLine()
		if err != nil {
			return "", err
		}

		buf = append(buf, line...)
		if !isPrefix {
			break
		}
	}
	return string(buf), nil
}
<|endoftext|>


<commit_before>package genericoauth

import (
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/auth/verifier"
	"github.com/concourse/atc/db"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

const ProviderName = "oauth"

type NoopVerifier struct{}

func (v NoopVerifier) Verify(logger lager.Logger, client *http.Client) (bool, error) {
	return true, nil
}

func NewProvider(
	genericOAuth *db.GenericOAuth,
	redirectURL string,
) Provider {
	endpoint := oauth2.Endpoint{}
	if genericOAuth.AuthURL != "" && genericOAuth.TokenURL != "" {
		endpoint.AuthURL = genericOAuth.AuthURL
		endpoint.TokenURL = genericOAuth.TokenURL
	}

	var oauthVerifier verifier.Verifier
	if genericOAuth.Scope != "" {
		oauthVerifier = ScopeVerifier{scope: genericOAuth.Scope}
	} else {
		oauthVerifier = NoopVerifier{}
	}

	return Provider{
		Verifier: oauthVerifier,
		Config: ConfigOverride{
			Config: oauth2.Config{
				ClientID:     genericOAuth.ClientID,
				ClientSecret: genericOAuth.ClientSecret,
				Endpoint:     endpoint,
				RedirectURL:  redirectURL,
			},
			AuthURLParams: genericOAuth.AuthURLParams,
		},
	}
}

type Provider struct {
	verifier.Verifier
	Config ConfigOverride
}

type ConfigOverride struct {
	oauth2.Config
	AuthURLParams map[string]string
}

// oauth2.Config implements the required Provider methods:
// AuthCodeURL(string, ...oauth2.AuthCodeOption) string
// Exchange(context.Context, string) (*oauth2.Token, error)
// Client(context.Context, *oauth2.Token) *http.Client

// override the default Provider method implementation from
// oauth2.Config in order to pass the extra configured Auth
// URL parameters

func (provider Provider) AuthCodeURL(state string, opts ...oauth2.AuthCodeOption) string {
	for key, value := range provider.Config.AuthURLParams {
		opts = append(opts, oauth2.SetAuthURLParam(key, value))
	}
	return provider.Config.AuthCodeURL(state, opts...)
}

func (provider Provider) Exchange(ctx context.Context, code string) (*oauth2.Token, error) {
	return provider.Config.Exchange(ctx, code)
}

func (provider Provider) Client(ctx context.Context, t *oauth2.Token) *http.Client {
	return provider.Config.Client(ctx, t)
}

func (Provider) PreTokenClient() (*http.Client, error) {
	return &http.Client{
		Transport: &http.Transport{
			DisableKeepAlives: true,
		},
	}, nil
}
<commit_msg>Use NewScopeVerifier constructor<commit_after>package genericoauth

import (
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/auth/verifier"
	"github.com/concourse/atc/db"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

const ProviderName = "oauth"

type NoopVerifier struct{}

func (v NoopVerifier) Verify(logger lager.Logger, client *http.Client) (bool, error) {
	return true, nil
}

func NewProvider(
	genericOAuth *db.GenericOAuth,
	redirectURL string,
) Provider {
	endpoint := oauth2.Endpoint{}
	if genericOAuth.AuthURL != "" && genericOAuth.TokenURL != "" {
		endpoint.AuthURL = genericOAuth.AuthURL
		endpoint.TokenURL = genericOAuth.TokenURL
	}

	var oauthVerifier verifier.Verifier
	if genericOAuth.Scope != "" {
		oauthVerifier = NewScopeVerifier(genericOAuth.Scope)
	} else {
		oauthVerifier = NoopVerifier{}
	}

	return Provider{
		Verifier: oauthVerifier,
		Config: ConfigOverride{
			Config: oauth2.Config{
				ClientID:     genericOAuth.ClientID,
				ClientSecret: genericOAuth.ClientSecret,
				Endpoint:     endpoint,
				RedirectURL:  redirectURL,
			},
			AuthURLParams: genericOAuth.AuthURLParams,
		},
	}
}

type Provider struct {
	verifier.Verifier
	Config ConfigOverride
}

type ConfigOverride struct {
	oauth2.Config
	AuthURLParams map[string]string
}

// oauth2.Config implements the required Provider methods:
// AuthCodeURL(string, ...oauth2.AuthCodeOption) string
// Exchange(context.Context, string) (*oauth2.Token, error)
// Client(context.Context, *oauth2.Token) *http.Client

// override the default Provider method implementation from
// oauth2.Config in order to pass the extra configured Auth
// URL parameters

func (provider Provider) AuthCodeURL(state string, opts ...oauth2.AuthCodeOption) string {
	for key, value := range provider.Config.AuthURLParams {
		opts = append(opts, oauth2.SetAuthURLParam(key, value))
	}
	return provider.Config.AuthCodeURL(state, opts...)
}

func (provider Provider) Exchange(ctx context.Context, code string) (*oauth2.Token, error) {
	return provider.Config.Exchange(ctx, code)
}

func (provider Provider) Client(ctx context.Context, t *oauth2.Token) *http.Client {
	return provider.Config.Client(ctx, t)
}

func (Provider) PreTokenClient() (*http.Client, error) {
	return &http.Client{
		Transport: &http.Transport{
			DisableKeepAlives: true,
		},
	}, nil
}
<|endoftext|>


<commit_before>package connection

import (
	"fmt"
	"reflect"
	"strconv"
	"time"
)

//following two structs represent cipherscan output

type CipherscanOutput struct {
	Target         string                  `json:"target"`
	IP             string                  `json:"ip"`
	Timestamp      string                  `json:"utctimestamp"`
	ServerSide     string                  `json:"serverside"`
	CurvesFallback string                  `json:"curves_fallback"`
	CipherSuites   []CipherscanCiphersuite `json:"ciphersuite"`
}

type CipherscanCiphersuite struct {
	Cipher       string   `json:"cipher"`
	Protocols    []string `json:"protocols"`
	PubKey       []string `json:"pubkey"`
	SigAlg       []string `json:"sigalg"`
	Trusted      string   `json:"trusted"`
	TicketHint   string   `json:"ticket_hint"`
	OCSPStapling string   `json:"ocsp_stapling"`
	PFS          string   `json:"pfs"`
	Curves       []string `json:"curves,omitempty"`
}

//the following structs represent the output we want to provide to DB.

type Stored struct {
	ScanIP         string        `json:"scanIP"`
	ServerSide     bool          `json:"serverside,omitempty"`
	CipherSuite    []Ciphersuite `json:"ciphersuite,omitempty"`
	CurvesFallback bool          `json:"curvesFallback,omitempty"`
}

type Ciphersuite struct {
	Cipher       string   `json:"cipher,omitempty"`
	Protocols    []string `json:"protocols,omitempty"`
	PubKey       float64  `json:"pubkey,omitempty"`
	SigAlg       string   `json:"sigalg,omitempty"`
	TicketHint   string   `json:"ticket_hint,omitempty"`
	OCSPStapling bool     `json:"ocsp_stapling,omitempty"`
	PFS          string   `json:"pfs,omitempty"`
	Curves       []string `json:"curves,omitempty"`
}

func stringtoBool(s string) bool {
	if s == "True" {
		return true
	} else {
		return false
	}
}

func (c Stored) Equal(ci Stored) bool {

	if c.CurvesFallback != ci.CurvesFallback {
		return false
	}

	if c.ServerSide != ci.ServerSide {
		return false
	}

	for i, suite := range c.CipherSuite {

		if !suite.equal(ci.CipherSuite[i]) {
			return false
		}
	}

	return true
}

func (s Ciphersuite) equal(cs Ciphersuite) bool {

	if s.Cipher != cs.Cipher {
		return false
	}

	if s.OCSPStapling != cs.OCSPStapling {
		return false
	}

	if s.PFS != cs.PFS {
		return false
	}

	if s.PubKey != cs.PubKey {
		return false
	}

	if s.SigAlg != cs.SigAlg {
		return false
	}

	if !reflect.DeepEqual(s.Curves, cs.Curves) {
		return false
	}

	if !reflect.DeepEqual(s.Protocols, cs.Protocols) {
		return false
	}

	return true

}

func (s CipherscanOutput) convertTimestamp(t string) (time.Time, error) {

	layout := "2006-01-02T15:04:05.0Z"
	return time.Parse(layout, t)
}

// Stored creates a Stored struct from the CipherscanOutput struct
func (s CipherscanOutput) Stored() (Stored, error) {

	c := Stored{}

	var err error

	t, err := s.convertTimestamp(s.Timestamp)

	if err != nil {
		return c, err
	}

	c.ServerSide = stringtoBool(s.ServerSide)
	c.CurvesFallback = stringtoBool(s.CurvesFallback)
	c.ScanIP = s.IP

	for _, cipher := range s.CipherSuites {

		newcipher := Ciphersuite{}

		newcipher.Cipher = cipher.Cipher
		newcipher.OCSPStapling = stringtoBool(cipher.OCSPStapling)
		newcipher.PFS = cipher.PFS

		newcipher.Protocols = cipher.Protocols

		if len(cipher.PubKey) > 1 {
			return c, fmt.Errorf("Multiple PubKeys for ", s.Target, " at cipher :", cipher.Cipher)
		}

		if len(cipher.PubKey) > 0 {
			newcipher.PubKey, err = strconv.ParseFloat(cipher.PubKey[0], 64)
		} else {
			return c, fmt.Errorf("No Public Keys found")
		}

		if len(cipher.SigAlg) > 1 {

			return c, fmt.Errorf("Multiple SigAlgs for ", s.Target, " at cipher :", cipher.Cipher)
		}

		if len(cipher.SigAlg) > 0 {
			newcipher.SigAlg = cipher.SigAlg[0]
		} else {
			return c, fmt.Errorf("No Signature Algorithms found")
		}

		newcipher.TicketHint = cipher.TicketHint
		if err != nil {
			return c, err
		}

		newcipher.Curves = append(newcipher.Curves, cipher.Curves...)
		c.CipherSuite = append(c.CipherSuite, newcipher)
	}

	return c, nil
}
<commit_msg>removed current time calculation as it is no longer needed<commit_after>package connection

import (
	"fmt"
	"reflect"
	"strconv"
	"time"
)

//following two structs represent cipherscan output

type CipherscanOutput struct {
	Target         string                  `json:"target"`
	IP             string                  `json:"ip"`
	Timestamp      string                  `json:"utctimestamp"`
	ServerSide     string                  `json:"serverside"`
	CurvesFallback string                  `json:"curves_fallback"`
	CipherSuites   []CipherscanCiphersuite `json:"ciphersuite"`
}

type CipherscanCiphersuite struct {
	Cipher       string   `json:"cipher"`
	Protocols    []string `json:"protocols"`
	PubKey       []string `json:"pubkey"`
	SigAlg       []string `json:"sigalg"`
	Trusted      string   `json:"trusted"`
	TicketHint   string   `json:"ticket_hint"`
	OCSPStapling string   `json:"ocsp_stapling"`
	PFS          string   `json:"pfs"`
	Curves       []string `json:"curves,omitempty"`
}

//the following structs represent the output we want to provide to DB.

type Stored struct {
	ScanIP         string        `json:"scanIP"`
	ServerSide     bool          `json:"serverside,omitempty"`
	CipherSuite    []Ciphersuite `json:"ciphersuite,omitempty"`
	CurvesFallback bool          `json:"curvesFallback,omitempty"`
}

type Ciphersuite struct {
	Cipher       string   `json:"cipher,omitempty"`
	Protocols    []string `json:"protocols,omitempty"`
	PubKey       float64  `json:"pubkey,omitempty"`
	SigAlg       string   `json:"sigalg,omitempty"`
	TicketHint   string   `json:"ticket_hint,omitempty"`
	OCSPStapling bool     `json:"ocsp_stapling,omitempty"`
	PFS          string   `json:"pfs,omitempty"`
	Curves       []string `json:"curves,omitempty"`
}

func stringtoBool(s string) bool {
	if s == "True" {
		return true
	} else {
		return false
	}
}

func (c Stored) Equal(ci Stored) bool {

	if c.CurvesFallback != ci.CurvesFallback {
		return false
	}

	if c.ServerSide != ci.ServerSide {
		return false
	}

	for i, suite := range c.CipherSuite {

		if !suite.equal(ci.CipherSuite[i]) {
			return false
		}
	}

	return true
}

func (s Ciphersuite) equal(cs Ciphersuite) bool {

	if s.Cipher != cs.Cipher {
		return false
	}

	if s.OCSPStapling != cs.OCSPStapling {
		return false
	}

	if s.PFS != cs.PFS {
		return false
	}

	if s.PubKey != cs.PubKey {
		return false
	}

	if s.SigAlg != cs.SigAlg {
		return false
	}

	if !reflect.DeepEqual(s.Curves, cs.Curves) {
		return false
	}

	if !reflect.DeepEqual(s.Protocols, cs.Protocols) {
		return false
	}

	return true

}

func (s CipherscanOutput) convertTimestamp(t string) (time.Time, error) {

	layout := "2006-01-02T15:04:05.0Z"
	return time.Parse(layout, t)
}

// Stored creates a Stored struct from the CipherscanOutput struct
func (s CipherscanOutput) Stored() (Stored, error) {

	c := Stored{}

	var err error

	c.ServerSide = stringtoBool(s.ServerSide)
	c.CurvesFallback = stringtoBool(s.CurvesFallback)
	c.ScanIP = s.IP

	for _, cipher := range s.CipherSuites {

		newcipher := Ciphersuite{}

		newcipher.Cipher = cipher.Cipher
		newcipher.OCSPStapling = stringtoBool(cipher.OCSPStapling)
		newcipher.PFS = cipher.PFS

		newcipher.Protocols = cipher.Protocols

		if len(cipher.PubKey) > 1 {
			return c, fmt.Errorf("Multiple PubKeys for ", s.Target, " at cipher :", cipher.Cipher)
		}

		if len(cipher.PubKey) > 0 {
			newcipher.PubKey, err = strconv.ParseFloat(cipher.PubKey[0], 64)
		} else {
			return c, fmt.Errorf("No Public Keys found")
		}

		if len(cipher.SigAlg) > 1 {

			return c, fmt.Errorf("Multiple SigAlgs for ", s.Target, " at cipher :", cipher.Cipher)
		}

		if len(cipher.SigAlg) > 0 {
			newcipher.SigAlg = cipher.SigAlg[0]
		} else {
			return c, fmt.Errorf("No Signature Algorithms found")
		}

		newcipher.TicketHint = cipher.TicketHint
		if err != nil {
			return c, err
		}

		newcipher.Curves = append(newcipher.Curves, cipher.Curves...)
		c.CipherSuite = append(c.CipherSuite, newcipher)
	}

	return c, nil
}
<|endoftext|>


<commit_before>//@auther foolbread
//@time 2015-11-12
//@file aqua/connect_server/server/server.go
package server

import (
	aeer "aqua/common/error"
	anet "aqua/common/net"
	aproto "aqua/common/proto"
	"aqua/connect_server/config"
	"fbcommon/golog"
	"fmt"
	"net"
	"sync"
)

const ARRARY_LEN = 255
const (
	LOGIN_SUCCESS = 0
)

func InitServer() {
	golog.Info("initing connect server......")
	g_outserver = new(outerConnectServer)
	for i := 0; i < ARRARY_LEN; i++ {
		g_outserver.cons[i] = newOuterConnSet()
	}
	g_outserver.startListen()

	g_innerserver = new(innerConnectServer)
	g_innerserver.startListen()
}

var g_outserver *outerConnectServer
var g_innerserver *innerConnectServer

type outerConnSet struct {
	lo   *sync.RWMutex
	mcon map[string]*Client
}

func newOuterConnSet() *outerConnSet {
	r := new(outerConnSet)
	r.lo = new(sync.RWMutex)
	r.mcon = make(map[string]*Client)

	return r
}

type outerConnectServer struct {
	cons [ARRARY_LEN]*outerConnSet
}

func (s *outerConnectServer) startListen() {
	oaddr := config.GetConfig().GetOuterAddr()

	go s.outerServer(oaddr)
}

func (s *outerConnectServer) outerServer(a string) {
	li, err := net.Listen("tcp", a)
	if err != nil {
		golog.Critical(err)
	}

	golog.Info("start outer listen:", a)

	for {
		c, err := li.Accept()
		if err != nil {
			golog.Critical(err)
		}

		go s.handlerClientCon(c)
	}
}

func (s *outerConnectServer) handlerClientCon(c net.Conn) {
	defer c.Close()
	//handler login
	cli, err := s.handlerClientLogin(c)
	if err != nil {
		golog.Error(err)
		return
	}

	cli.run()
}

func (s *outerConnectServer) handlerClientLogin(c net.Conn) (*Client, error) {
	var buf [1024]byte
	var ret *Client = nil
	l, err := anet.RecvPacket(c, buf[:])
	if err != nil {
		return nil, err
	}

	req, err := aproto.UnmarshalLoginReq(buf[aproto.HEAD_LEN:l])
	if err != nil {
		return nil, err
	}

	//check token
	set := s.cons[req.Token[0]%ARRARY_LEN]
	key := fmt.Sprintf("%02X", req.Token)
	set.lo.RLock()
	_, ok := set.mcon[key]
	if ok {
		set.lo.RUnlock()
		return nil, aeer.ErrConnExsit
	}
	set.lo.RUnlock()

	set.lo.Lock()
	_, ok = set.mcon[key]
	if !ok {
		ret = newClient(req.Cid, req.Token, c)
		set.mcon[key] = ret
	}
	set.lo.Unlock()

	//construct login response
	d, err := aproto.MarshalLoginRes(req.Token, LOGIN_SUCCESS, req.Cid)
	if err != nil {
		s.exitConn(req.Token, key)
		return nil, err
	}

	err = anet.SendPacket(c, d)
	if err != nil {
		s.exitConn(req.Token, key)
		return nil, err
	}

	return ret, nil
}

func (s *outerConnectServer) exitConn(key []byte, keystr string) {
	set := s.cons[key[0]%ARRARY_LEN]

	set.lo.Lock()
	delete(set.mcon, keystr)
	set.lo.Unlock()

	golog.Info("token:", keystr, "connect exit!")
}

////////////////////////////////////////////////////////////////////////////////////
type innerConnectServer struct {
	server map[int][]*logicServer
	lo     *sync.RWMutex
}

func newInnerConnectServer() *innerConnectServer {
	r := new(innerConnectServer)
	r.server = make(map[int][]*logicServer)
	r.lo = new(sync.RWMutex)

	return r
}

func (s *innerConnectServer) startListen() {
	iaddr := config.GetConfig().GetInnerAddr()

	go s.innerServer(iaddr)
}

func (s *innerConnectServer) getServer(t int, key []byte) *logicServer {
	var ret *logicServer
	s.lo.RLock()
	v, ok := s.server[t]
	if ok {
		ret = v[int(key[0])%len(v)]
	}
	s.lo.RUnlock()

	return ret
}

func (s *innerConnectServer) innerServer(a string) {
	li, err := net.Listen("tcp", a)
	if err != nil {
		golog.Critical(err)
	}

	golog.Info("start inner listen:", a)

	for {
		c, err := li.Accept()
		if err != nil {
			golog.Critical(err)
		}

		go s.handlerLogicCon(c)
	}
}

func (s *innerConnectServer) handlerLogicCon(c net.Conn) {
	ser, err := s.handlerLogicLogin(c)
	if err != nil {
		golog.Error(err)
		return
	}

	ser.run()
}

func (s *innerConnectServer) handlerLogicLogin(c net.Conn) (*logicServer, error) {
	var buf [1024]byte
	n, err := anet.RecvPacket(c, buf[:])
	if err != nil {
		return nil, err
	}

	req, err := aproto.UnmarshalSvrRegisterReq(buf[:n])
	if err != nil {
		return nil, err
	}

	l := newLogicServer(int(req.ServiceType), c)

	s.lo.Lock()
	val := s.server[int(req.ServiceType)]
	val = append(val, l)
	s.lo.Unlock()

	return l, nil
}
<commit_msg>*  修改函数名字<commit_after>//@auther foolbread
//@time 2015-11-12
//@file aqua/connect_server/server/server.go
package server

import (
	aeer "aqua/common/error"
	anet "aqua/common/net"
	aproto "aqua/common/proto"
	"aqua/connect_server/config"
	"fbcommon/golog"
	"fmt"
	"net"
	"sync"
)

const ARRARY_LEN = 255
const (
	LOGIN_SUCCESS = 0
)

func InitServer() {
	golog.Info("initing connect server......")
	g_outserver = new(outerConnectServer)
	for i := 0; i < ARRARY_LEN; i++ {
		g_outserver.cons[i] = newOuterConnSet()
	}
	g_outserver.startListen()

	g_innerserver = new(innerConnectServer)
	g_innerserver.startListen()
}

var g_outserver *outerConnectServer
var g_innerserver *innerConnectServer

type outerConnSet struct {
	lo   *sync.RWMutex
	mcon map[string]*Client
}

func newOuterConnSet() *outerConnSet {
	r := new(outerConnSet)
	r.lo = new(sync.RWMutex)
	r.mcon = make(map[string]*Client)

	return r
}

type outerConnectServer struct {
	cons [ARRARY_LEN]*outerConnSet
}

func (s *outerConnectServer) startListen() {
	oaddr := config.GetConfig().GetOuterAddr()

	go s.outerServer(oaddr)
}

func (s *outerConnectServer) outerServer(a string) {
	li, err := net.Listen("tcp", a)
	if err != nil {
		golog.Critical(err)
	}

	golog.Info("start outer listen:", a)

	for {
		c, err := li.Accept()
		if err != nil {
			golog.Critical(err)
		}

		go s.handlerClientCon(c)
	}
}

func (s *outerConnectServer) handlerClientCon(c net.Conn) {
	defer c.Close()
	//handler login
	cli, err := s.handlerClientLogin(c)
	if err != nil {
		golog.Error(err)
		return
	}

	cli.run()
}

func (s *outerConnectServer) handlerClientLogin(c net.Conn) (*Client, error) {
	var buf [1024]byte
	var ret *Client = nil
	l, err := anet.RecvPacket(c, buf[:])
	if err != nil {
		return nil, err
	}

	req, err := aproto.UnmarshalLoginReq(buf[aproto.HEAD_LEN:l])
	if err != nil {
		return nil, err
	}

	//check token
	set := s.cons[req.Token[0]%ARRARY_LEN]
	key := fmt.Sprintf("%02X", req.Token)
	set.lo.RLock()
	_, ok := set.mcon[key]
	if ok {
		set.lo.RUnlock()
		return nil, aeer.ErrConnExsit
	}
	set.lo.RUnlock()

	set.lo.Lock()
	_, ok = set.mcon[key]
	if !ok {
		ret = newClient(req.Cid, req.Token, c)
		set.mcon[key] = ret
	}
	set.lo.Unlock()

	//construct login response
	d, err := aproto.MarshalLoginRes(req.Token, LOGIN_SUCCESS, req.Cid)
	if err != nil {
		s.exitClient(req.Token, key)
		return nil, err
	}

	err = anet.SendPacket(c, d)
	if err != nil {
		s.exitClient(req.Token, key)
		return nil, err
	}

	return ret, nil
}

func (s *outerConnectServer) exitClient(key []byte, keystr string) {
	set := s.cons[key[0]%ARRARY_LEN]

	set.lo.Lock()
	delete(set.mcon, keystr)
	set.lo.Unlock()

	golog.Info("token:", keystr, "connect exit!")
}

////////////////////////////////////////////////////////////////////////////////////
type innerConnectServer struct {
	server map[int][]*logicServer
	lo     *sync.RWMutex
}

func newInnerConnectServer() *innerConnectServer {
	r := new(innerConnectServer)
	r.server = make(map[int][]*logicServer)
	r.lo = new(sync.RWMutex)

	return r
}

func (s *innerConnectServer) startListen() {
	iaddr := config.GetConfig().GetInnerAddr()

	go s.innerServer(iaddr)
}

func (s *innerConnectServer) getServer(t int, key []byte) *logicServer {
	var ret *logicServer
	s.lo.RLock()
	v, ok := s.server[t]
	if ok {
		ret = v[int(key[0])%len(v)]
	}
	s.lo.RUnlock()

	return ret
}

func (s *innerConnectServer) innerServer(a string) {
	li, err := net.Listen("tcp", a)
	if err != nil {
		golog.Critical(err)
	}

	golog.Info("start inner listen:", a)

	for {
		c, err := li.Accept()
		if err != nil {
			golog.Critical(err)
		}

		go s.handlerLogicCon(c)
	}
}

func (s *innerConnectServer) handlerLogicCon(c net.Conn) {
	ser, err := s.handlerLogicLogin(c)
	if err != nil {
		golog.Error(err)
		return
	}

	ser.run()
}

func (s *innerConnectServer) handlerLogicLogin(c net.Conn) (*logicServer, error) {
	var buf [1024]byte
	n, err := anet.RecvPacket(c, buf[:])
	if err != nil {
		return nil, err
	}

	req, err := aproto.UnmarshalSvrRegisterReq(buf[:n])
	if err != nil {
		return nil, err
	}

	l := newLogicServer(int(req.ServiceType), c)

	s.lo.Lock()
	val := s.server[int(req.ServiceType)]
	val = append(val, l)
	s.lo.Unlock()

	return l, nil
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"path/filepath"

	"github.com/inconshreveable/log15"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/api/def"
	"polydawn.net/repeatr/rio"
)

// Run inputs
func ProvisionInputs(transmat rio.Transmat, inputs def.InputGroup, journal log15.Logger) map[string]rio.Arena {
	// start having all filesystems
	// input names are used as keys, so must be unique
	fsGather := make(chan map[string]materializerReport)
	for name, in := range inputs {
		go func(name string, in *def.Input) {
			try.Do(func() {
				journal.Info(fmt.Sprintf("Starting materialize for %s hash=%s", in.Type, in.Hash))
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(in.Warehouses))
				for i, wh := range in.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				arena := transmat.Materialize(
					rio.TransmatKind(in.Type),
					rio.CommitID(in.Hash),
					warehouses,
					journal,
				)
				// submit report
				journal.Info(fmt.Sprintf("Finished materialize for %s hash=%s", in.Type, in.Hash))
				fsGather <- map[string]materializerReport{
					name: {Arena: arena},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn(fmt.Sprintf("Errored during materialize for %s hash=%s", in.Type, in.Hash), "error", err.Message())
				fsGather <- map[string]materializerReport{
					name: {Err: err},
				}
			}).Done()
		}(name, in)
	}

	// (we don't have any output setup at this point, but if we do in the future, that'll be here.)

	// gather materialized inputs
	// any errors are re-raised immediately (TODO: this currently doesn't fan out smooth cancellations)
	nInputs := len(inputs)
	filesystems := make(map[string]rio.Arena, nInputs)
	for range inputs {
		for name, report := range <-fsGather {
			if report.Err != nil {
				panic(report.Err)
			}
			journal.Info(fmt.Sprintf("Input %d/%d ready", len(filesystems)+1, nInputs))
			filesystems[name] = report.Arena
		}
	}

	return filesystems
}

func AssembleFilesystem(
	assemblerFn rio.Assembler,
	rootPath string,
	inputs def.InputGroup,
	inputArenas map[string]rio.Arena,
	hostMounts []def.Mount,
	journal log15.Logger,
) rio.Assembly {
	journal.Info("All inputs acquired... starting assembly")
	// process inputs
	assemblyParts := make([]rio.AssemblyPart, 0, len(inputArenas))
	for name, arena := range inputArenas {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: arena.Path(),
			TargetPath: inputs[name].MountPath,
			Writable:   true, // TODO input config should have a word about this
		})
	}
	// process mounts
	for _, mount := range hostMounts {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: mount.SourcePath,
			TargetPath: mount.TargetPath,
			Writable:   mount.Writable,
			BareMount:  true,
		})
	}
	// assemmmmmmmmblllle
	assembly := assemblerFn(rootPath, assemblyParts)
	journal.Info("Assembly complete!")
	return assembly
}

type materializerReport struct {
	Arena rio.Arena     // if success
	Err   *errors.Error // subtype of input.Error.  (others are forbidden by contract and treated as fatal.)
}

func ProvisionOutputs(outputs def.OutputGroup, rootfs string, journal log15.Logger) {
	// We no longer make output MountPaths by default.
	// Originally, this seemed like a good idea, because it would be a
	//  consistent stance and allow us to use more complex (e.g. mount-powered)
	//   output shuttling concepts later without any fuss.
	// However, practically speaking, as a default, this has turned out not to work well.
	// Other tools in the world expect clear directories.
	// One example that makes this instantaneously game-over in the real
	//  world is git: `git clone [url] .` *won't work* if there are any other
	//   dirs already existing under `.`; and given how frequently projects
	//    put their build and test outputs in gitignored dirs under their
	//     project versioning root, we very frequently see outputs pointed there.
	//      That creates a lot of noise... so we're dropping the behavior.
}

// Run outputs
func PreserveOutputs(transmat rio.Transmat, outputs def.OutputGroup, rootfs string, journal log15.Logger) def.OutputGroup {
	// run commit on the outputs
	scanGather := make(chan map[string]scanReport)
	for name, out := range outputs {
		go func(name string, out *def.Output) {
			out.Filters = &def.Filters{}
			out.Filters.InitDefaultsOutput()
			filterOptions := rio.ConvertFilterConfig(*out.Filters)
			scanPath := filepath.Join(rootfs, out.MountPath)
			journal.Info(fmt.Sprintf("Starting scan on %q", scanPath))
			try.Do(func() {
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(out.Warehouses))
				for i, wh := range out.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				commitID := transmat.Scan(
					rio.TransmatKind(out.Type),
					scanPath,
					warehouses,
					journal,
					filterOptions...,
				)
				out.Hash = string(commitID)
				// submit report
				journal.Info(fmt.Sprintf("Finished scan on %q", scanPath))
				scanGather <- map[string]scanReport{
					name: {Output: out},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn(fmt.Sprintf("Errored scan on %q", scanPath), "error", err.Message())
				scanGather <- map[string]scanReport{
					name: {Err: err},
				}
			}).Done()
		}(name, out)
	}

	// gather reports
	results := def.OutputGroup{}
	for range outputs {
		for name, report := range <-scanGather {
			if report.Err != nil {
				panic(report.Err)
			}
			results[name] = report.Output
		}
	}

	return results
}

type scanReport struct {
	Output *def.Output   // now including the hash
	Err    *errors.Error // subtype of output.Error.  (others are forbidden by contract and treated as fatal.)
}
<commit_msg>Use more structured logging and less fmt'd messages in materialization.<commit_after>package util

import (
	"fmt"
	"path/filepath"

	"github.com/inconshreveable/log15"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/api/def"
	"polydawn.net/repeatr/rio"
)

// Run inputs
func ProvisionInputs(transmat rio.Transmat, inputs def.InputGroup, journal log15.Logger) map[string]rio.Arena {
	// start having all filesystems
	// input names are used as keys, so must be unique
	fsGather := make(chan map[string]materializerReport)
	for name, in := range inputs {
		go func(name string, in *def.Input) {
			journal := journal.New(
				"input", name,
				"type", in.Type,
				"hash", in.Hash,
			)
			try.Do(func() {
				journal.Info("Starting materialize")
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(in.Warehouses))
				for i, wh := range in.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				arena := transmat.Materialize(
					rio.TransmatKind(in.Type),
					rio.CommitID(in.Hash),
					warehouses,
					journal,
				)
				// submit report
				journal.Info("Finished materialize")
				fsGather <- map[string]materializerReport{
					name: {Arena: arena},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn("Error during materialize",
					"error", err.Message(),
				)
				fsGather <- map[string]materializerReport{
					name: {Err: err},
				}
			}).Done()
		}(name, in)
	}

	// (we don't have any output setup at this point, but if we do in the future, that'll be here.)

	// gather materialized inputs
	// any errors are re-raised immediately (TODO: this currently doesn't fan out smooth cancellations)
	nInputs := len(inputs)
	filesystems := make(map[string]rio.Arena, nInputs)
	for range inputs {
		for name, report := range <-fsGather {
			if report.Err != nil {
				panic(report.Err)
			}
			journal.Info(fmt.Sprintf("Input %d/%d ready", len(filesystems)+1, nInputs))
			filesystems[name] = report.Arena
		}
	}

	return filesystems
}

func AssembleFilesystem(
	assemblerFn rio.Assembler,
	rootPath string,
	inputs def.InputGroup,
	inputArenas map[string]rio.Arena,
	hostMounts []def.Mount,
	journal log15.Logger,
) rio.Assembly {
	journal.Info("All inputs acquired... starting assembly")
	// process inputs
	assemblyParts := make([]rio.AssemblyPart, 0, len(inputArenas))
	for name, arena := range inputArenas {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: arena.Path(),
			TargetPath: inputs[name].MountPath,
			Writable:   true, // TODO input config should have a word about this
		})
	}
	// process mounts
	for _, mount := range hostMounts {
		assemblyParts = append(assemblyParts, rio.AssemblyPart{
			SourcePath: mount.SourcePath,
			TargetPath: mount.TargetPath,
			Writable:   mount.Writable,
			BareMount:  true,
		})
	}
	// assemmmmmmmmblllle
	assembly := assemblerFn(rootPath, assemblyParts)
	journal.Info("Assembly complete!")
	return assembly
}

type materializerReport struct {
	Arena rio.Arena     // if success
	Err   *errors.Error // subtype of input.Error.  (others are forbidden by contract and treated as fatal.)
}

func ProvisionOutputs(outputs def.OutputGroup, rootfs string, journal log15.Logger) {
	// We no longer make output MountPaths by default.
	// Originally, this seemed like a good idea, because it would be a
	//  consistent stance and allow us to use more complex (e.g. mount-powered)
	//   output shuttling concepts later without any fuss.
	// However, practically speaking, as a default, this has turned out not to work well.
	// Other tools in the world expect clear directories.
	// One example that makes this instantaneously game-over in the real
	//  world is git: `git clone [url] .` *won't work* if there are any other
	//   dirs already existing under `.`; and given how frequently projects
	//    put their build and test outputs in gitignored dirs under their
	//     project versioning root, we very frequently see outputs pointed there.
	//      That creates a lot of noise... so we're dropping the behavior.
}

// Run outputs
func PreserveOutputs(transmat rio.Transmat, outputs def.OutputGroup, rootfs string, journal log15.Logger) def.OutputGroup {
	// run commit on the outputs
	scanGather := make(chan map[string]scanReport)
	for name, out := range outputs {
		go func(name string, out *def.Output) {
			out.Filters = &def.Filters{}
			out.Filters.InitDefaultsOutput()
			filterOptions := rio.ConvertFilterConfig(*out.Filters)
			scanPath := filepath.Join(rootfs, out.MountPath)
			journal.Info(fmt.Sprintf("Starting scan on %q", scanPath))
			try.Do(func() {
				// todo: create validity checking api for URIs, check them all before launching anything
				warehouses := make([]rio.SiloURI, len(out.Warehouses))
				for i, wh := range out.Warehouses {
					warehouses[i] = rio.SiloURI(wh)
				}
				// invoke transmat (blocking, potentially long time)
				commitID := transmat.Scan(
					rio.TransmatKind(out.Type),
					scanPath,
					warehouses,
					journal,
					filterOptions...,
				)
				out.Hash = string(commitID)
				// submit report
				journal.Info(fmt.Sprintf("Finished scan on %q", scanPath))
				scanGather <- map[string]scanReport{
					name: {Output: out},
				}
			}).Catch(rio.Error, func(err *errors.Error) {
				journal.Warn(fmt.Sprintf("Errored scan on %q", scanPath), "error", err.Message())
				scanGather <- map[string]scanReport{
					name: {Err: err},
				}
			}).Done()
		}(name, out)
	}

	// gather reports
	results := def.OutputGroup{}
	for range outputs {
		for name, report := range <-scanGather {
			if report.Err != nil {
				panic(report.Err)
			}
			results[name] = report.Output
		}
	}

	return results
}

type scanReport struct {
	Output *def.Output   // now including the hash
	Err    *errors.Error // subtype of output.Error.  (others are forbidden by contract and treated as fatal.)
}
<|endoftext|>


<commit_before>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"code.google.com/p/gorilla/mux"
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
	"syscall"
)

// Config holds configuration data used when creating a Proxy with New.
type Config struct {
	// HmacKey is a string to be used as the hmac key
	HmacKey         string
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList       []string
	// DenyList is a list of string represenstations of regex (not compiled
	// regex). The deny filter check occurs after the allow filter check
	// (if any).
	DenyList        []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize         int64
	// FollowRedirects is a boolean that specifies whether upstream redirects
	// are followed (10 depth) or not.
	FollowRedirects bool
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout  time.Duration
}

// Interface for Proxy to use for stats/metrics.
// This must be goroutine safe, as AddBytes and AddServed will be called from
// many goroutines.
type ProxyMetrics interface {
	AddBytes(bc int64)
	AddServed()
	//GetStats() (b uint64, c uint64)
}

// A Proxy is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support.
type Proxy struct {
	client    *http.Client
	hmacKey   []byte
	allowList []*regexp.Regexp
	denyList  []*regexp.Regexp
	maxSize   int64
	metrics   ProxyMetrics
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	Logger.Debugln("Request:", req.URL)
	if p.metrics != nil {
		go p.metrics.AddServed()
	}

	w.Header().Set("Server", ServerNameVer)

	vars := mux.Vars(req)
	surl, ok := DecodeUrl(&p.hmacKey, vars["sigHash"], vars["encodedUrl"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	Logger.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		Logger.Debugln(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out denyList urls based on regexes. Do this second
	// as denyList takes precedence
	for _, rgx := range p.denyList {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		Logger.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &ValidReqHeaders)
	if req.Header.Get("X-Forwarded-For") == "" {
		host, _, err := net.SplitHostPort(req.RemoteAddr)
		if err == nil && !addr1918match.MatchString(host) {
			nreq.Header.Add("X-Forwarded-For", host)
		}
	}
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", ServerNameVer)

	resp, err := p.client.Do(nreq)
	if err != nil {
		Logger.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.maxSize {
		Logger.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			Logger.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		Logger.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303, 307:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
		h.Set("X-Content-Type-Options", "nosniff")
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
	h.Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(resp.StatusCode)

	// since this uses io.Copy from the respBody, it is streaming
	// from the request to the response. This means it will nearly
	// always end up with a chunked response.
	// Change to the following to send whole body at once, and
	// read whole body at once too:
	//    body, err := ioutil.ReadAll(resp.Body)
	//    if err != nil {
	//        Logger.Println("Error writing response:", err)
	//    }
	//    w.Write(body)
	// Might use quite a bit of memory though. Untested.
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		// only log if not broken pipe, as this means the client terminated
		// conn for some reason.
		opErr, ok := err.(*net.OpError)
		if !ok || opErr.Err != syscall.EPIPE {
			Logger.Println("Error writing response:", err)
		}
		return
	}

	if p.metrics != nil {
		go p.metrics.AddBytes(bW)
	}
	Logger.Debugln(req, resp.StatusCode)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *Proxy) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// sets a proxy metrics (ProxyMetrics interface) for the proxy
func (p *Proxy) SetMetricsCollector(pm ProxyMetrics) {
	p.metrics = pm
}

// Returns a new Proxy. An error is returned if there was a failure
// to parse the regex from the passed Config.
func New(pc Config) (*Proxy, error) {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, pc.RequestTimeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(pc.RequestTimeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &Proxy{
		client:    client,
		hmacKey:   []byte(pc.HmacKey),
		allowList: allow,
		denyList:  deny,
		maxSize:   pc.MaxSize}, nil
}
<commit_msg>clean up comment<commit_after>// Package camoproxy provides an HTTP proxy server with content type
// restrictions as well as regex host allow and deny list support.
package camoproxy

import (
	"code.google.com/p/gorilla/mux"
	"errors"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
	"syscall"
)

// Config holds configuration data used when creating a Proxy with New.
type Config struct {
	// HmacKey is a string to be used as the hmac key
	HmacKey         string
	// AllowList is a list of string represenstations of regex (not compiled
	// regex) that are used as a whitelist filter. If an AllowList is present,
	// then anything not matching is dropped. If no AllowList is present,
	// no Allow filtering is done.
	AllowList       []string
	// DenyList is a list of string represenstations of regex (not compiled
	// regex). The deny filter check occurs after the allow filter check
	// (if any).
	DenyList        []string
	// MaxSize is the maximum valid image size response (in bytes).
	MaxSize         int64
	// FollowRedirects is a boolean that specifies whether upstream redirects
	// are followed (10 depth) or not.
	FollowRedirects bool
	// Request timeout is a timeout for fetching upstream data.
	RequestTimeout  time.Duration
}

// Interface for Proxy to use for stats/metrics.
// This must be goroutine safe, as AddBytes and AddServed will be called from
// many goroutines.
type ProxyMetrics interface {
	AddBytes(bc int64)
	AddServed()
	//GetStats() (b uint64, c uint64)
}

// A Proxy is a Camo like HTTP proxy, that provides content type
// restrictions as well as regex host allow and deny list support.
type Proxy struct {
	client    *http.Client
	hmacKey   []byte
	allowList []*regexp.Regexp
	denyList  []*regexp.Regexp
	maxSize   int64
	metrics   ProxyMetrics
}

// ServerHTTP handles the client request, validates the request is validly
// HMAC signed, filters based on the Allow/Deny list, and then proxies
// valid requests to the desired endpoint. Responses are filtered for
// proper image content types.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	Logger.Debugln("Request:", req.URL)
	if p.metrics != nil {
		go p.metrics.AddServed()
	}

	w.Header().Set("Server", ServerNameVer)

	vars := mux.Vars(req)
	surl, ok := DecodeUrl(&p.hmacKey, vars["sigHash"], vars["encodedUrl"])
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	Logger.Debugln("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		Logger.Debugln(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if allowList is set, require match
	matchFound := true
	if len(p.allowList) > 0 {
		matchFound = false
		for _, rgx := range p.allowList {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out denyList urls based on regexes. Do this second
	// as denyList takes precedence
	for _, rgx := range p.denyList {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		Logger.Debugln("Could not create NewRequest", err)
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	}

	// filter headers
	p.copyHeader(&nreq.Header, &req.Header, &ValidReqHeaders)
	if req.Header.Get("X-Forwarded-For") == "" {
		host, _, err := net.SplitHostPort(req.RemoteAddr)
		if err == nil && !addr1918match.MatchString(host) {
			nreq.Header.Add("X-Forwarded-For", host)
		}
	}
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", ServerNameVer)

	resp, err := p.client.Do(nreq)
	if err != nil {
		Logger.Debugln("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.maxSize {
		Logger.Debugln("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusNotFound)
		return
	}

	switch resp.StatusCode {
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			Logger.Debugln("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	case 300:
		Logger.Debugln("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303, 307:
		// if we get a redirect here, we either disabled following,
		// or followed until max depth and still got one (redirect loop)
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 304:
		h := w.Header()
		p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
		h.Set("X-Content-Type-Options", "nosniff")
		w.WriteHeader(304)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 500, 502, 503, 504:
		// upstream errors should probably just 502. client can try later.
		http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		return
	default:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	h := w.Header()
	p.copyHeader(&h, &resp.Header, &ValidRespHeaders)
	h.Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(resp.StatusCode)

	// since this uses io.Copy from the respBody, it is streaming
	// from the request to the response. This means it will nearly
	// always end up with a chunked response.
	// Change to the following to send whole body at once, and
	// read whole body at once too:
	//    body, err := ioutil.ReadAll(resp.Body)
	//    if err != nil {
	//        Logger.Println("Error writing response:", err)
	//    }
	//    w.Write(body)
	// Might use quite a bit of memory though. Untested.
	bW, err := io.Copy(w, resp.Body)
	if err != nil {
		// only log if not broken pipe. broken pipe means the client
		// terminated conn for some reason.
		opErr, ok := err.(*net.OpError)
		if !ok || opErr.Err != syscall.EPIPE {
			Logger.Println("Error writing response:", err)
		}
		return
	}

	if p.metrics != nil {
		go p.metrics.AddBytes(bW)
	}
	Logger.Debugln(req, resp.StatusCode)
}

// copy headers from src into dst
// empty filter map will result in no filtering being done
func (p *Proxy) copyHeader(dst, src *http.Header, filter *map[string]bool) {
	f := *filter
	filtering := false
	if len(f) > 0 {
		filtering = true
	}

	for k, vv := range *src {
		if x, ok := f[k]; filtering && (!ok || !x) {
			continue
		}
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

// sets a proxy metrics (ProxyMetrics interface) for the proxy
func (p *Proxy) SetMetricsCollector(pm ProxyMetrics) {
	p.metrics = pm
}

// Returns a new Proxy. An error is returned if there was a failure
// to parse the regex from the passed Config.
func New(pc Config) (*Proxy, error) {
	tr := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			c, err := net.DialTimeout(netw, addr, pc.RequestTimeout)
			if err != nil {
				return nil, err
			}
			// also set time limit on reading
			c.SetDeadline(time.Now().Add(pc.RequestTimeout))
			return c, nil
		}}

	// spawn an idle conn trimmer
	go func() {
		time.Sleep(5 * time.Minute)
		tr.CloseIdleConnections()
	}()

	// build/compile regex
	client := &http.Client{Transport: tr}
	if !pc.FollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errors.New("Not following redirect")
		}
	}

	allow := make([]*regexp.Regexp, 0)
	deny := make([]*regexp.Regexp, 0)

	var c *regexp.Regexp
	var err error
	for _, v := range pc.DenyList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		deny = append(deny, c)
	}
	for _, v := range pc.AllowList {
		c, err = regexp.Compile(v)
		if err != nil {
			return nil, err
		}
		allow = append(allow, c)
	}

	return &Proxy{
		client:    client,
		hmacKey:   []byte(pc.HmacKey),
		allowList: allow,
		denyList:  deny,
		maxSize:   pc.MaxSize}, nil
}
<|endoftext|>


<commit_before>package builtin

import (
	// "log"

	. "github.com/wanliu/flow/builtin/resolves"
	. "github.com/wanliu/flow/context"

	config "github.com/wanliu/flow/builtin/config"
)

// type ConfirmData {
// 	Resolve *Resolve
// 	ResolveType string

// 	Action string
// 	Value interface{}
// }

type AddressConfirm struct {
	Order *OrderResolve
	Value string
}

type Confirm struct {
	TryGetEntities

	expMins float64

	ExpMins <-chan float64

	Confirm <-chan Context
	Cancel  <-chan Context
	Expire  <-chan Context

	Out chan<- ReplyData
}

func NewConfirm() interface{} {
	return new(Confirm)
}

func (c *Confirm) OnExpMins(mins float64) {
	c.expMins = mins
}

func (c Confirm) OnExpire(ctx Context) {
	ctx.SetValue(config.CtxKeyConfirm, nil)
}

func (c *Confirm) OnConfirm(ctx Context) {
	currentOrder := ctx.Value(config.CtxKeyOrder)
	confirm := ctx.Value(config.CtxKeyConfirm)

	if currentOrder != nil && confirm != nil {
		cOrder := currentOrder.(OrderResolve)

		if cOrder.Expired(config.SesssionExpiredMinutes) {
			c.Out <- ReplyData{"当前没有正在进行中的订单", ctx}
			return
		}

		cConfirm := confirm.(AddressConfirm)

		if cOrder.Address == "" {
			cOrder.Address = cConfirm.Value

			if cOrder.Fulfiled() {
				ctx.SetValue(config.CtxKeyOrder, nil)
				ctx.SetValue(config.CtxKeyLastOrder, cOrder)
			}

			reply := "已经确认\"" + cConfirm.Value + "\"为收货地址" + "\n" + cOrder.Answer()
			c.Out <- ReplyData{reply, ctx}
		} else {
			reply := ReplyData{"确认操作已经过期", ctx}
			c.Out <- reply
		}
	} else {
		reply := ReplyData{"确认操作已经过期", ctx}
		c.Out <- reply
	}
}

func (c *Confirm) OnCancel(ctx Context) {
	currentOrder := ctx.Value(config.CtxKeyOrder)
	confirm := ctx.Value(config.CtxKeyConfirm)

	if currentOrder != nil && confirm != nil {
		cOrder := currentOrder.(OrderResolve)

		ctx.SetValue(config.CtxKeyConfirm, nil)

		reply := "取消操作完成"

		if cOrder.Address == "" {
			reply = reply + "，当前订单收货地址尚未确认，请输入收货地址"
		}

		c.Out <- ReplyData{reply, ctx}
	} else {
		reply := ReplyData{"取消操作已经过期", ctx}
		c.Out <- reply
	}
}
<commit_msg>fix unfulfiled address confirm<commit_after>package builtin

import (
	// "log"

	. "github.com/wanliu/flow/builtin/resolves"
	. "github.com/wanliu/flow/context"

	config "github.com/wanliu/flow/builtin/config"
)

// type ConfirmData {
// 	Resolve *Resolve
// 	ResolveType string

// 	Action string
// 	Value interface{}
// }

type AddressConfirm struct {
	Order *OrderResolve
	Value string
}

type Confirm struct {
	TryGetEntities

	expMins float64

	ExpMins <-chan float64

	Confirm <-chan Context
	Cancel  <-chan Context
	Expire  <-chan Context

	Out chan<- ReplyData
}

func NewConfirm() interface{} {
	return new(Confirm)
}

func (c *Confirm) OnExpMins(mins float64) {
	c.expMins = mins
}

func (c Confirm) OnExpire(ctx Context) {
	ctx.SetValue(config.CtxKeyConfirm, nil)
}

func (c *Confirm) OnConfirm(ctx Context) {
	currentOrder := ctx.Value(config.CtxKeyOrder)
	confirm := ctx.Value(config.CtxKeyConfirm)

	if currentOrder != nil && confirm != nil {
		cOrder := currentOrder.(OrderResolve)

		if cOrder.Expired(config.SesssionExpiredMinutes) {
			c.Out <- ReplyData{"当前没有正在进行中的订单", ctx}
			return
		}

		cConfirm := confirm.(AddressConfirm)

		if cOrder.Address == "" {
			cOrder.Address = cConfirm.Value

			if cOrder.Fulfiled() {
				ctx.SetValue(config.CtxKeyOrder, nil)
				ctx.SetValue(config.CtxKeyLastOrder, cOrder)
			} else {
				ctx.SetValue(config.CtxKeyOrder, cOrder)
			}

			reply := "已经确认\"" + cConfirm.Value + "\"为收货地址" + "\n" + cOrder.Answer()
			c.Out <- ReplyData{reply, ctx}
		} else {
			reply := ReplyData{"确认操作已经过期", ctx}
			c.Out <- reply
		}
	} else {
		reply := ReplyData{"确认操作已经过期", ctx}
		c.Out <- reply
	}
}

func (c *Confirm) OnCancel(ctx Context) {
	currentOrder := ctx.Value(config.CtxKeyOrder)
	confirm := ctx.Value(config.CtxKeyConfirm)

	if currentOrder != nil && confirm != nil {
		cOrder := currentOrder.(OrderResolve)

		ctx.SetValue(config.CtxKeyConfirm, nil)

		reply := "取消操作完成"

		if cOrder.Address == "" {
			reply = reply + "，当前订单收货地址尚未确认，请输入收货地址"
		}

		c.Out <- ReplyData{reply, ctx}
	} else {
		reply := ReplyData{"取消操作已经过期", ctx}
		c.Out <- reply
	}
}
<|endoftext|>


<commit_before>package detour

import (
	"strings"
	"sync"
)

type wlEntry struct {
	permanent bool
}

var (
	muWhitelist sync.RWMutex
	whitelist   = make(map[string]wlEntry)
)

// AddToWl adds a domain to whitelist, all subdomains of this domain
// are also considered to be in the whitelist.
func AddToWl(addr string, permanent bool) {
	muWhitelist.Lock()
	defer muWhitelist.Unlock()
	if add != "" {
		whitelist[addr] = wlEntry{permanent}
	}
}

//RemoveFromWl removes an addr from whitelist
func RemoveFromWl(addr string) {
	muWhitelist.Lock()
	defer muWhitelist.Unlock()
	delete(whitelist, addr)
}

//DumpWhitelist dumps the whitelist for other usage
func DumpWhitelist() (wl []string) {
	wl = make([]string, 1)
	muWhitelist.Lock()
	defer muWhitelist.Unlock()
	for k, v := range whitelist {
		if v.permanent {
			wl = append(wl, k)
		}
	}
	return
}

func whitelisted(addr string) (in bool) {
	muWhitelist.RLock()
	defer muWhitelist.RUnlock()
	for ; addr != ""; addr = getParentDomain(addr) {
		_, in = whitelist[addr]
		if in {
			return
		}
	}
	return
}

func wlTemporarily(addr string) bool {
	muWhitelist.RLock()
	defer muWhitelist.RUnlock()
	// temporary domains are always full ones, just check map
	p, ok := whitelist[addr]
	return ok && p.permanent == false
}

func getParentDomain(addr string) string {
	parts := strings.SplitN(addr, ".", 2)
	if len(parts) < 2 {
		return ""
	}
	return parts[1]
}
<commit_msg>Fixing variable name.<commit_after>package detour

import (
	"strings"
	"sync"
)

type wlEntry struct {
	permanent bool
}

var (
	muWhitelist sync.RWMutex
	whitelist   = make(map[string]wlEntry)
)

// AddToWl adds a domain to whitelist, all subdomains of this domain
// are also considered to be in the whitelist.
func AddToWl(addr string, permanent bool) {
	muWhitelist.Lock()
	defer muWhitelist.Unlock()
	if addr != "" {
		whitelist[addr] = wlEntry{permanent}
	}
}

//RemoveFromWl removes an addr from whitelist
func RemoveFromWl(addr string) {
	muWhitelist.Lock()
	defer muWhitelist.Unlock()
	delete(whitelist, addr)
}

//DumpWhitelist dumps the whitelist for other usage
func DumpWhitelist() (wl []string) {
	wl = make([]string, 1)
	muWhitelist.Lock()
	defer muWhitelist.Unlock()
	for k, v := range whitelist {
		if v.permanent {
			wl = append(wl, k)
		}
	}
	return
}

func whitelisted(addr string) (in bool) {
	muWhitelist.RLock()
	defer muWhitelist.RUnlock()
	for ; addr != ""; addr = getParentDomain(addr) {
		_, in = whitelist[addr]
		if in {
			return
		}
	}
	return
}

func wlTemporarily(addr string) bool {
	muWhitelist.RLock()
	defer muWhitelist.RUnlock()
	// temporary domains are always full ones, just check map
	p, ok := whitelist[addr]
	return ok && p.permanent == false
}

func getParentDomain(addr string) string {
	parts := strings.SplitN(addr, ".", 2)
	if len(parts) < 2 {
		return ""
	}
	return parts[1]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/meisterluk/dupfiles-go/api"
	"github.com/meisterluk/dupfiles-go/match"
	"github.com/meisterluk/dupfiles-go/traversal"
)

func main() {
	var conf api.Config
	conf.HashAlgorithm = "sha512"
	conf.HashSpec.Content = true
	conf.HashSpec.Relpath = true

	bases := make([]api.Source, 0, 5)

	if os.Args[1] == "-v" || os.Args[1] == "--version" {
		log.Println("version 0.1 basic")
		os.Exit(0)
	}

	if len(os.Args)%2 == 0 {
		log.Println("Usage: ./dupfiles <NAME1> <PATH1> [<NAME2> <PATH2>]+")
		log.Fatal("Error: number of command-line arguments must be even")
	}

	if len(os.Args) == 1 {
		log.Print("usage: dupfiles <name> <path> [<name> <path>]*")
		os.Exit(0)
	}

	// create file system root instances
	for i := 1; i < len(os.Args); i = i + 2 {
		bases = append(bases, api.Source{Path: os.Args[i+1], Name: os.Args[i]})
	}

	// get ready for traversal
	trees := make([]api.Tree, 0, len(bases))
	treePtrs := make([]*api.Tree, 0, 5)
	for _, base := range bases {
		t := api.Tree{}
		trees = append(trees, t)
		treePtrs = append(treePtrs, &t)
		err := traversal.DFSTraverse(&conf, &base, &t)
		if err != nil {
			log.Fatal(err)
		}
	}

	done := make(chan bool)
	eqChan := make(chan []*api.Entry)

	go func() {
		for eq := range eqChan {
			fmt.Printf("{ ")
			for _, e := range eq {
				fmt.Printf(" %s %s  ", e.Base, e.Path)
			}
			fmt.Printf("}\n")
		}
		done <- true
	}()

	err := match.Match(&conf, treePtrs, eqChan)
	if err != nil {
		log.Fatal(err)
	}
	close(eqChan)
	<-done
}
<commit_msg>Put main routine is separate Main function. Improves testability.<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/meisterluk/dupfiles-go/api"
	"github.com/meisterluk/dupfiles-go/match"
	"github.com/meisterluk/dupfiles-go/traversal"
)

// Main implements the main routine but is independent of os.Args
func Main(args []string) {
	var conf api.Config
	conf.HashAlgorithm = "sha512"
	conf.HashSpec.Content = true
	conf.HashSpec.Relpath = true

	bases := make([]api.Source, 0, 5)

	// create file system root instances
	for i := 1; i < len(args); i = i + 2 {
		bases = append(bases, api.Source{Path: args[i+1], Name: args[i]})
	}

	// get ready for traversal
	trees := make([]api.Tree, 0, len(bases))
	treePtrs := make([]*api.Tree, 0, 5)
	for _, base := range bases {
		t := api.Tree{}
		trees = append(trees, t)
		treePtrs = append(treePtrs, &t)
		err := traversal.DFSTraverse(&conf, &base, &t)
		if err != nil {
			log.Fatal(err)
		}
	}

	done := make(chan bool)
	eqChan := make(chan []*api.Entry)

	go func() {
		for eq := range eqChan {
			fmt.Printf("{ ")
			for _, e := range eq {
				fmt.Printf(" %s %s  ", e.Base, e.Path)
			}
			fmt.Printf("}\n")
		}
		done <- true
	}()

	err := match.Match(&conf, treePtrs, eqChan)
	if err != nil {
		log.Fatal(err)
	}
	close(eqChan)
	<-done
}

func main() {
	if os.Args[1] == "-v" || os.Args[1] == "--version" {
		log.Println("version 0.1 basic")
		os.Exit(0)
	}

	if len(os.Args)%2 == 0 {
		log.Println("Usage: ./dupfiles <NAME1> <PATH1> [<NAME2> <PATH2>]+")
		log.Fatal("Error: number of command-line arguments must be even")
	}

	if len(os.Args) == 1 {
		log.Print("usage: dupfiles <name> <path> [<name> <path>]*")
		os.Exit(0)
	}

	Main(os.Args)
}
<|endoftext|>


<commit_before>// Package v1alpha1 implements all the required types and methods for parsing
// resources for v1alpha1 versioned ClusterServiceVersions.
package v1alpha1

import (
	"encoding/json"

	"github.com/coreos/go-semver/semver"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// GroupVersion is the version used in the Scheme for ClusterServiceVersions.
const GroupVersion = "v1alpha1"

// NamedInstallStrategy represents the block of an ClusterServiceVersion resource
// where the install strategy is specified.
type NamedInstallStrategy struct {
	StrategyName    string          `json:"strategy"`
	StrategySpecRaw json.RawMessage `json:"spec"`
}

// CustomResourceDefinitions declares all of the CRDs managed or required by
// an operator being ran by ClusterServiceVersion.
//
// If the CRD is present in the Owned list, it is implicitly required.
type CustomResourceDefinitions struct {
	Owned    []string
	Required []string
}

// ClusterServiceVersionSpec declarations tell the ALM how to install an operator
// that can manage apps for given version and AppType.
type ClusterServiceVersionSpec struct {
	InstallStrategy           NamedInstallStrategy      `json:"install"`
	Version                   semver.Version            `json:"version"`
	Maturity                  string                    `json:"maturity"`
	CustomResourceDefinitions CustomResourceDefinitions `json:"customresourcedefinitions"`
	Permissions               []string                  `json:"permissions"`
	DisplayName               string                    `json:"displayName"`
	Description               string                    `json:"description"`
	Keywords                  []string                  `json:"keywords"`
	Maintainers               []Maintainer              `json:"maintainers"`
	Links                     []AppLink                 `json:"links"`
	Icon                      Icon                      `json:"iconURL"`

	// Map of string keys and values that can be used to organize and categorize
	// (scope and select) objects. May match selectors of replication controllers
	// and services.
	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
	// +optional
	Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"`

	// Annotations is an unstructured key value map stored with a resource that may be
	// set by external tools to store and retrieve arbitrary metadata. They are not
	// queryable and should be preserved when modifying objects.
	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
	// +optional
	Annotations map[string]string `json:"annotations,omitempty" protobuf:"bytes,12,rep,name=annotations"`

	// Label selector for pods. Existing ReplicaSets whose pods are
	// selected by this will be the ones affected by this deployment.
	// +optional
	Selector *metav1.LabelSelector `json:"selector,omitempty" protobuf:"bytes,2,opt,name=selector"`
}

type Maintainer struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type AppLink struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

type Icon struct {
	Data      string `json:"base64data"`
	MediaType string `json:"mediatype"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// ClusterServiceVersion is a Custom Resource of type `ClusterServiceVersionSpec`.
type ClusterServiceVersion struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Spec   ClusterServiceVersionSpec `json:"spec"`
	Status metav1.Status             `json:"status"`
}

// ClusterServiceVersionList represents a list of ClusterServiceVersions.
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type ClusterServiceVersionList struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Items []ClusterServiceVersion `json:"items"`
}
<commit_msg>csv: add json tags to CRDs<commit_after>// Package v1alpha1 implements all the required types and methods for parsing
// resources for v1alpha1 versioned ClusterServiceVersions.
package v1alpha1

import (
	"encoding/json"

	"github.com/coreos/go-semver/semver"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// GroupVersion is the version used in the Scheme for ClusterServiceVersions.
const GroupVersion = "v1alpha1"

// NamedInstallStrategy represents the block of an ClusterServiceVersion resource
// where the install strategy is specified.
type NamedInstallStrategy struct {
	StrategyName    string          `json:"strategy"`
	StrategySpecRaw json.RawMessage `json:"spec"`
}

// CustomResourceDefinitions declares all of the CRDs managed or required by
// an operator being ran by ClusterServiceVersion.
//
// If the CRD is present in the Owned list, it is implicitly required.
type CustomResourceDefinitions struct {
	Owned    []string `json:"owned"`
	Required []string `json:"required"`
}

// ClusterServiceVersionSpec declarations tell the ALM how to install an operator
// that can manage apps for given version and AppType.
type ClusterServiceVersionSpec struct {
	InstallStrategy           NamedInstallStrategy      `json:"install"`
	Version                   semver.Version            `json:"version"`
	Maturity                  string                    `json:"maturity"`
	CustomResourceDefinitions CustomResourceDefinitions `json:"customresourcedefinitions"`
	Permissions               []string                  `json:"permissions"`
	DisplayName               string                    `json:"displayName"`
	Description               string                    `json:"description"`
	Keywords                  []string                  `json:"keywords"`
	Maintainers               []Maintainer              `json:"maintainers"`
	Links                     []AppLink                 `json:"links"`
	Icon                      Icon                      `json:"iconURL"`

	// Map of string keys and values that can be used to organize and categorize
	// (scope and select) objects. May match selectors of replication controllers
	// and services.
	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
	// +optional
	Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"`

	// Annotations is an unstructured key value map stored with a resource that may be
	// set by external tools to store and retrieve arbitrary metadata. They are not
	// queryable and should be preserved when modifying objects.
	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
	// +optional
	Annotations map[string]string `json:"annotations,omitempty" protobuf:"bytes,12,rep,name=annotations"`

	// Label selector for pods. Existing ReplicaSets whose pods are
	// selected by this will be the ones affected by this deployment.
	// +optional
	Selector *metav1.LabelSelector `json:"selector,omitempty" protobuf:"bytes,2,opt,name=selector"`
}

type Maintainer struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type AppLink struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

type Icon struct {
	Data      string `json:"base64data"`
	MediaType string `json:"mediatype"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// ClusterServiceVersion is a Custom Resource of type `ClusterServiceVersionSpec`.
type ClusterServiceVersion struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Spec   ClusterServiceVersionSpec `json:"spec"`
	Status metav1.Status             `json:"status"`
}

// ClusterServiceVersionList represents a list of ClusterServiceVersions.
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type ClusterServiceVersionList struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Items []ClusterServiceVersion `json:"items"`
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package unix_test

import (
	"bytes"
	"go/build"
	"net"
	"os"
	"syscall"
	"testing"

	"golang.org/x/sys/unix"
)

// TestSCMCredentials tests the sending and receiving of credentials
// (PID, UID, GID) in an ancillary message between two UNIX
// sockets. The SO_PASSCRED socket option is enabled on the sending
// socket for this to work.
func TestSCMCredentials(t *testing.T) {
	socketTypeTests := []struct {
		socketType int
		dataLen    int
	}{
		{
			unix.SOCK_STREAM,
			1,
		}, {
			unix.SOCK_DGRAM,
			0,
		},
	}

	for _, tt := range socketTypeTests {
		if tt.socketType == unix.SOCK_DGRAM && !atLeast1p10() {
			t.Log("skipping DGRAM test on pre-1.10")
			continue
		}

		fds, err := unix.Socketpair(unix.AF_LOCAL, tt.socketType, 0)
		if err != nil {
			t.Fatalf("Socketpair: %v", err)
		}
		defer unix.Close(fds[0])
		defer unix.Close(fds[1])

		err = unix.SetsockoptInt(fds[0], unix.SOL_SOCKET, unix.SO_PASSCRED, 1)
		if err != nil {
			t.Fatalf("SetsockoptInt: %v", err)
		}

		srvFile := os.NewFile(uintptr(fds[0]), "server")
		defer srvFile.Close()
		srv, err := net.FileConn(srvFile)
		if err != nil {
			t.Errorf("FileConn: %v", err)
			return
		}
		defer srv.Close()

		cliFile := os.NewFile(uintptr(fds[1]), "client")
		defer cliFile.Close()
		cli, err := net.FileConn(cliFile)
		if err != nil {
			t.Errorf("FileConn: %v", err)
			return
		}
		defer cli.Close()

		var ucred unix.Ucred
		ucred.Pid = int32(os.Getpid() - 1)
		ucred.Uid = uint32(os.Getuid())
		ucred.Gid = uint32(os.Getgid())
		oob := unix.UnixCredentials(&ucred)
		_, _, err = cli.(*net.UnixConn).WriteMsgUnix(nil, oob, nil)
		if op, ok := err.(*net.OpError); ok {
			err = op.Err
		}
		if sys, ok := err.(*os.SyscallError); ok {
			err = sys.Err
		}
		if err != syscall.EPERM {
			t.Fatalf("WriteMsgUnix failed with %v, want EPERM", err)
		}

		// Fix the PID.
		ucred.Pid = int32(os.Getpid())
		oob = unix.UnixCredentials(&ucred)

		// On SOCK_STREAM, this is internally going to send a dummy byte
		n, oobn, err := cli.(*net.UnixConn).WriteMsgUnix(nil, oob, nil)
		if err != nil {
			t.Fatalf("WriteMsgUnix: %v", err)
		}
		if n != 0 {
			t.Fatalf("WriteMsgUnix n = %d, want 0", n)
		}
		if oobn != len(oob) {
			t.Fatalf("WriteMsgUnix oobn = %d, want %d", oobn, len(oob))
		}

		oob2 := make([]byte, 10*len(oob))
		n, oobn2, flags, _, err := srv.(*net.UnixConn).ReadMsgUnix(nil, oob2)
		if err != nil {
			t.Fatalf("ReadMsgUnix: %v", err)
		}
		if flags != 0 {
			t.Fatalf("ReadMsgUnix flags = 0x%x, want 0", flags)
		}
		if n != tt.dataLen {
			t.Fatalf("ReadMsgUnix n = %d, want %d", n, tt.dataLen)
		}
		if oobn2 != oobn {
			// without SO_PASSCRED set on the socket, ReadMsgUnix will
			// return zero oob bytes
			t.Fatalf("ReadMsgUnix oobn = %d, want %d", oobn2, oobn)
		}
		oob2 = oob2[:oobn2]
		if !bytes.Equal(oob, oob2) {
			t.Fatal("ReadMsgUnix oob bytes don't match")
		}

		scm, err := unix.ParseSocketControlMessage(oob2)
		if err != nil {
			t.Fatalf("ParseSocketControlMessage: %v", err)
		}
		newUcred, err := unix.ParseUnixCredentials(&scm[0])
		if err != nil {
			t.Fatalf("ParseUnixCredentials: %v", err)
		}
		if *newUcred != ucred {
			t.Fatalf("ParseUnixCredentials = %+v, want %+v", newUcred, ucred)
		}
	}
}

// atLeast1p10 reports whether we are running on Go 1.10 or later.
func atLeast1p10() bool {
	for _, ver := range build.Default.ReleaseTags {
		if ver == "go1.10" {
			return true
		}
	}
	return false
}
<commit_msg>unix: fix TestSCMCredentials to not fail when root<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package unix_test

import (
	"bytes"
	"go/build"
	"net"
	"os"
	"testing"

	"golang.org/x/sys/unix"
)

// TestSCMCredentials tests the sending and receiving of credentials
// (PID, UID, GID) in an ancillary message between two UNIX
// sockets. The SO_PASSCRED socket option is enabled on the sending
// socket for this to work.
func TestSCMCredentials(t *testing.T) {
	socketTypeTests := []struct {
		socketType int
		dataLen    int
	}{
		{
			unix.SOCK_STREAM,
			1,
		}, {
			unix.SOCK_DGRAM,
			0,
		},
	}

	for _, tt := range socketTypeTests {
		if tt.socketType == unix.SOCK_DGRAM && !atLeast1p10() {
			t.Log("skipping DGRAM test on pre-1.10")
			continue
		}

		fds, err := unix.Socketpair(unix.AF_LOCAL, tt.socketType, 0)
		if err != nil {
			t.Fatalf("Socketpair: %v", err)
		}
		defer unix.Close(fds[0])
		defer unix.Close(fds[1])

		err = unix.SetsockoptInt(fds[0], unix.SOL_SOCKET, unix.SO_PASSCRED, 1)
		if err != nil {
			t.Fatalf("SetsockoptInt: %v", err)
		}

		srvFile := os.NewFile(uintptr(fds[0]), "server")
		defer srvFile.Close()
		srv, err := net.FileConn(srvFile)
		if err != nil {
			t.Errorf("FileConn: %v", err)
			return
		}
		defer srv.Close()

		cliFile := os.NewFile(uintptr(fds[1]), "client")
		defer cliFile.Close()
		cli, err := net.FileConn(cliFile)
		if err != nil {
			t.Errorf("FileConn: %v", err)
			return
		}
		defer cli.Close()

		var ucred unix.Ucred
		ucred.Pid = int32(os.Getpid())
		ucred.Uid = uint32(os.Getuid())
		ucred.Gid = uint32(os.Getgid())
		oob := unix.UnixCredentials(&ucred)

		// On SOCK_STREAM, this is internally going to send a dummy byte
		n, oobn, err := cli.(*net.UnixConn).WriteMsgUnix(nil, oob, nil)
		if err != nil {
			t.Fatalf("WriteMsgUnix: %v", err)
		}
		if n != 0 {
			t.Fatalf("WriteMsgUnix n = %d, want 0", n)
		}
		if oobn != len(oob) {
			t.Fatalf("WriteMsgUnix oobn = %d, want %d", oobn, len(oob))
		}

		oob2 := make([]byte, 10*len(oob))
		n, oobn2, flags, _, err := srv.(*net.UnixConn).ReadMsgUnix(nil, oob2)
		if err != nil {
			t.Fatalf("ReadMsgUnix: %v", err)
		}
		if flags != 0 {
			t.Fatalf("ReadMsgUnix flags = 0x%x, want 0", flags)
		}
		if n != tt.dataLen {
			t.Fatalf("ReadMsgUnix n = %d, want %d", n, tt.dataLen)
		}
		if oobn2 != oobn {
			// without SO_PASSCRED set on the socket, ReadMsgUnix will
			// return zero oob bytes
			t.Fatalf("ReadMsgUnix oobn = %d, want %d", oobn2, oobn)
		}
		oob2 = oob2[:oobn2]
		if !bytes.Equal(oob, oob2) {
			t.Fatal("ReadMsgUnix oob bytes don't match")
		}

		scm, err := unix.ParseSocketControlMessage(oob2)
		if err != nil {
			t.Fatalf("ParseSocketControlMessage: %v", err)
		}
		newUcred, err := unix.ParseUnixCredentials(&scm[0])
		if err != nil {
			t.Fatalf("ParseUnixCredentials: %v", err)
		}
		if *newUcred != ucred {
			t.Fatalf("ParseUnixCredentials = %+v, want %+v", newUcred, ucred)
		}
	}
}

// atLeast1p10 reports whether we are running on Go 1.10 or later.
func atLeast1p10() bool {
	for _, ver := range build.Default.ReleaseTags {
		if ver == "go1.10" {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package freecache

import (
	"encoding/binary"
	"sync/atomic"
	"github.com/spaolacci/murmur3"
	"time"
)

type CacheStatus struct {
	TimeStamp,
	TimeRange,
	ItemsCount,
	ExpiredCount int64
	hitCount    int64        `json:"-"`
	lookupCount int64        `json:"-"`
	HitRate,
	AvgLookupPerSecond,
	AvgHitPerSecond float64
}

func getCurrTimestamp() int64 {
	return int64(time.Now().Unix())
}

type Cache struct {
	segments   [256]segment
	hitCount   int64
	missCount  int64
	lastStatus CacheStatus
}

func hashFunc(data []byte) uint64 {
	return murmur3.Sum64(data)
}

// The cache size will be set to 512KB at minimum.
// If the size is set relatively large, you should call
// `debug.SetGCPercent()`, set it to a much smaller value
// to limit the memory consumption and GC pause time.
func NewCache(size int) (cache *Cache) {
	if size < 512*1024 {
		size = 512 * 1024
	}
	cache = new(Cache)
	for i := 0; i < 256; i++ {
		cache.segments[i] = newSegment(size/256, i)
	}
	cache.lastStatus = CacheStatus{TimeStamp: getCurrTimestamp()}
	return
}

// If the key is larger than 65535 or value is larger than 1/1024 of the cache size,
// the entry will not be written to the cache. expireSeconds <= 0 means no expire,
// but it can be evicted when cache is full.
func (cache *Cache) Set(key, value []byte, expireSeconds int) (err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	err = cache.segments[segId].set(key, value, hashVal, expireSeconds)
	return
}

// Get the value or not found error.
func (cache *Cache) Get(key []byte) (value []byte, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	value, err = cache.segments[segId].get(key, hashVal)
	if err == nil {
		atomic.AddInt64(&cache.hitCount, 1)
	} else {
		atomic.AddInt64(&cache.missCount, 1)
	}
	return
}

func (cache *Cache) TTL(key []byte) (timeLeft uint32, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	timeLeft, err = cache.segments[segId].ttl(key, hashVal)
	return
}

func (cache *Cache) Del(key []byte) (affected bool) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	affected = cache.segments[segId].del(key, hashVal)
	return
}

func (cache *Cache) SetInt(key int64, value []byte, expireSeconds int) (err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Set(bKey[:], value, expireSeconds)
}

func (cache *Cache) GetInt(key int64) (value []byte, err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Get(bKey[:])
}

func (cache *Cache) DelInt(key int64) (affected bool) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Del(bKey[:])
}

func (cache *Cache) EvacuateCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalEvacuate)
	}
	return
}

func (cache *Cache) ExpiredCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalExpired)
	}
	return
}

func (cache *Cache) EntryCount() (entryCount int64) {
	for i := 0; i < 256; i++ {
		entryCount += atomic.LoadInt64(&cache.segments[i].entryCount)
	}
	return
}

func (cache *Cache) HitCount() int64 {
	return atomic.LoadInt64(&cache.hitCount)
}

func (cache *Cache) LookupCount() int64 {
	return atomic.LoadInt64(&cache.hitCount) + atomic.LoadInt64(&cache.missCount)
}

func (cache *Cache) HitRate() float64 {
	lookupCount := cache.LookupCount()
	if lookupCount == 0 {
		return 0
	} else {
		return float64(cache.HitCount()) / float64(lookupCount)
	}
}

func (cache *Cache) OverwriteCount() (overwriteCount int64) {
	for i := 0; i < 256; i++ {
		overwriteCount += atomic.LoadInt64(&cache.segments[i].overwrites)
	}
	return
}

func (cache *Cache) Clear() {
	for i := 0; i < 256; i++ {
		seg := cache.segments[i]
		seg.lock.Lock()
		cache.segments[i] = newSegment(len(cache.segments[i].rb.data), i)
		seg.lock.Unlock()
	}
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.TimeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.hitCount = 0
	cache.lastStatus.lookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) ResetStatistics() {
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	for i := 0; i < 256; i++ {
		cache.segments[i].lock.Lock()
		cache.segments[i].resetStatistics()
		cache.segments[i].lock.Unlock()
	}
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.TimeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.hitCount = 0
	cache.lastStatus.lookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) GetStatistics() *CacheStatus {
	now := getCurrTimestamp()
	currentStatus := CacheStatus{TimeStamp: now, TimeRange: now - cache.lastStatus.TimeStamp}
	itemsCount := cache.EntryCount()
	expiredCount := cache.ExpiredCount()
	hitCount := cache.HitCount()
	lookupCount := cache.LookupCount()
	currentStatus.ExpiredCount = expiredCount
	currentStatus.ItemsCount = itemsCount
	if currentStatus.TimeRange > 0 {
		currentStatus.hitCount = hitCount - cache.lastStatus.hitCount
		currentStatus.lookupCount = lookupCount - cache.lastStatus.lookupCount
		currentStatus.AvgLookupPerSecond = float64(currentStatus.lookupCount) / float64(currentStatus.TimeRange)
		currentStatus.AvgHitPerSecond = float64(currentStatus.hitCount) / float64(currentStatus.TimeRange)
		if currentStatus.lookupCount > 0 {
			currentStatus.HitRate = float64(currentStatus.hitCount) / float64(currentStatus.lookupCount)
		} else {
			currentStatus.HitRate = 0.0
		}
		cache.lastStatus.TimeStamp = now
		cache.lastStatus.TimeRange = 0
		cache.lastStatus.ExpiredCount = expiredCount
		cache.lastStatus.ItemsCount = itemsCount
		cache.lastStatus.hitCount = hitCount
		cache.lastStatus.lookupCount = lookupCount
		cache.lastStatus.HitRate = 0

	}
	return &currentStatus
}
<commit_msg>状态值 Fix<commit_after>package freecache

import (
	"encoding/binary"
	"sync/atomic"
	"github.com/spaolacci/murmur3"
	"time"
)

type CacheStatus struct {
	TimeStamp,
	TimeRange,
	ItemsCount,
	ExpiredCount int64
	hitCount    int64        `json:"-"`
	lookupCount int64        `json:"-"`
	HitRate,
	AvgLookupPerSecond,
	AvgHitPerSecond float64
}

func getCurrTimestamp() int64 {
	return int64(time.Now().Unix())
}

type Cache struct {
	segments   [256]segment
	hitCount   int64
	missCount  int64
	lastStatus CacheStatus
}

func hashFunc(data []byte) uint64 {
	return murmur3.Sum64(data)
}

// The cache size will be set to 512KB at minimum.
// If the size is set relatively large, you should call
// `debug.SetGCPercent()`, set it to a much smaller value
// to limit the memory consumption and GC pause time.
func NewCache(size int) (cache *Cache) {
	if size < 512*1024 {
		size = 512 * 1024
	}
	cache = new(Cache)
	for i := 0; i < 256; i++ {
		cache.segments[i] = newSegment(size/256, i)
	}
	cache.lastStatus = CacheStatus{TimeStamp: getCurrTimestamp()}
	return
}

// If the key is larger than 65535 or value is larger than 1/1024 of the cache size,
// the entry will not be written to the cache. expireSeconds <= 0 means no expire,
// but it can be evicted when cache is full.
func (cache *Cache) Set(key, value []byte, expireSeconds int) (err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	err = cache.segments[segId].set(key, value, hashVal, expireSeconds)
	return
}

// Get the value or not found error.
func (cache *Cache) Get(key []byte) (value []byte, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	value, err = cache.segments[segId].get(key, hashVal)
	if err == nil {
		atomic.AddInt64(&cache.hitCount, 1)
	} else {
		atomic.AddInt64(&cache.missCount, 1)
	}
	return
}

func (cache *Cache) TTL(key []byte) (timeLeft uint32, err error) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	timeLeft, err = cache.segments[segId].ttl(key, hashVal)
	return
}

func (cache *Cache) Del(key []byte) (affected bool) {
	hashVal := hashFunc(key)
	segId := hashVal & 255
	affected = cache.segments[segId].del(key, hashVal)
	return
}

func (cache *Cache) SetInt(key int64, value []byte, expireSeconds int) (err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Set(bKey[:], value, expireSeconds)
}

func (cache *Cache) GetInt(key int64) (value []byte, err error) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Get(bKey[:])
}

func (cache *Cache) DelInt(key int64) (affected bool) {
	var bKey [8]byte
	binary.LittleEndian.PutUint64(bKey[:], uint64(key))
	return cache.Del(bKey[:])
}

func (cache *Cache) HistoricalEvacuateCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalEvacuate)
	}
	return
}

func (cache *Cache) HistoricalExpiredCount() (count int64) {
	for i := 0; i < 256; i++ {
		count += atomic.LoadInt64(&cache.segments[i].totalExpired)
	}
	return
}

func (cache *Cache) HistoricalOverwriteCount() (overwriteCount int64) {
	for i := 0; i < 256; i++ {
		overwriteCount += atomic.LoadInt64(&cache.segments[i].overwrites)
	}
	return
}

func (cache *Cache) EntryCount() (entryCount int64) {
	for i := 0; i < 256; i++ {
		entryCount += atomic.LoadInt64(&cache.segments[i].entryCount)
	}
	return
}

func (cache *Cache) ExpiredCount() (deleteCount int64) {
	for i := 0; i < 256; i++ {
		deleteCount += atomic.LoadInt64(&cache.segments[i].totalCount) - atomic.LoadInt64(&cache.segments[i].entryCount)
	}
	return
}

func (cache *Cache) HitCount() int64 {
	return atomic.LoadInt64(&cache.hitCount)
}

func (cache *Cache) LookupCount() int64 {
	return atomic.LoadInt64(&cache.hitCount) + atomic.LoadInt64(&cache.missCount)
}

func (cache *Cache) HitRate() float64 {
	lookupCount := cache.LookupCount()
	if lookupCount == 0 {
		return 0
	} else {
		return float64(cache.HitCount()) / float64(lookupCount)
	}
}

func (cache *Cache) Clear() {
	for i := 0; i < 256; i++ {
		seg := cache.segments[i]
		seg.lock.Lock()
		cache.segments[i] = newSegment(len(cache.segments[i].rb.data), i)
		seg.lock.Unlock()
	}
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.TimeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.hitCount = 0
	cache.lastStatus.lookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) ResetStatistics() {
	atomic.StoreInt64(&cache.hitCount, 0)
	atomic.StoreInt64(&cache.missCount, 0)
	for i := 0; i < 256; i++ {
		cache.segments[i].lock.Lock()
		cache.segments[i].resetStatistics()
		cache.segments[i].lock.Unlock()
	}
	cache.lastStatus.TimeStamp = getCurrTimestamp()
	cache.lastStatus.TimeRange = 0
	cache.lastStatus.ExpiredCount = 0
	cache.lastStatus.ItemsCount = 0
	cache.lastStatus.hitCount = 0
	cache.lastStatus.lookupCount = 0
	cache.lastStatus.HitRate = 0
}

func (cache *Cache) GetStatistics() *CacheStatus {
	now := getCurrTimestamp()
	currentStatus := CacheStatus{TimeStamp: now, TimeRange: now - cache.lastStatus.TimeStamp}
	itemsCount := cache.EntryCount()
	expiredCount := cache.ExpiredCount()
	hitCount := cache.HitCount()
	lookupCount := cache.LookupCount()
	currentStatus.ExpiredCount = expiredCount
	currentStatus.ItemsCount = itemsCount
	if currentStatus.TimeRange > 0 {
		currentStatus.hitCount = hitCount - cache.lastStatus.hitCount
		currentStatus.lookupCount = lookupCount - cache.lastStatus.lookupCount
		currentStatus.AvgLookupPerSecond = float64(currentStatus.lookupCount) / float64(currentStatus.TimeRange)
		currentStatus.AvgHitPerSecond = float64(currentStatus.hitCount) / float64(currentStatus.TimeRange)
		if currentStatus.lookupCount > 0 {
			currentStatus.HitRate = float64(currentStatus.hitCount) / float64(currentStatus.lookupCount)
		} else {
			currentStatus.HitRate = 0.0
		}
		cache.lastStatus.TimeStamp = now
		cache.lastStatus.TimeRange = 0
		cache.lastStatus.ExpiredCount = expiredCount
		cache.lastStatus.ItemsCount = itemsCount
		cache.lastStatus.hitCount = hitCount
		cache.lastStatus.lookupCount = lookupCount
		cache.lastStatus.HitRate = 0

	}
	return &currentStatus
}
<|endoftext|>