<commit_before>package of10

import (
	"net"

	. "github.com/oshothebig/goflow/openflow"
)

type FeaturesRequest struct {
	Header
}

type FeaturesReply struct {
	Header
	DatapathId   DatapathId
	Buffers      uint32
	Tables       uint8
	pad          [3]uint8
	Capabilities Capability
	Actions      ActionType
	Ports        []PhysicalPort
}

type DatapathId uint64
type Capability uint32

const (
	OFPC_FLOW_STATS Capability = 1 << iota
	OFPC_TABLE_STATS
	OFPC_PORT_STATS
	OFPC_STP
	OFPC_RESERVED
	OFPC_IP_REASM
	OFPC_QUEUE_STATS
	OFPC_ARP_MATCH_IP
)

var Capabilities = struct {
	FlowStats    Capability
	TableStats   Capability
	PortStats    Capability
	Stp          Capability
	Reserved     Capability
	IpReassemble Capability
	QueueStats   Capability
	ArpMatchIp   Capability
}{
	OFPC_FLOW_STATS,
	OFPC_TABLE_STATS,
	OFPC_PORT_STATS,
	OFPC_STP,
	OFPC_RESERVED,
	OFPC_IP_REASM,
	OFPC_QUEUE_STATS,
	OFPC_ARP_MATCH_IP,
}

type SwitchConfig struct {
	Header
	Flags          ConfigFlag
	MissSendLength uint16
}

type ConfigFlag uint16

const (
	OFPC_FRAG_NORMAL ConfigFlag = iota
	OFPC_FRAG_DROP
	OFPC_FRAG_REASM
	OFPC_FRAG_MASK
)

var ConfigFlags = struct {
	FragmentNormal     ConfigFlag
	FragmentDrop       ConfigFlag
	FragmentReassemble ConfigFlag
	FragmentMask       ConfigFlag
}{
	OFPC_FRAG_NORMAL,
	OFPC_FRAG_DROP,
	OFPC_FRAG_REASM,
	OFPC_FRAG_MASK,
}

type FlowMod struct {
	Header
	Match       Match
	Cookie      Cookie
	Command     FlowModCommand
	IdleTimeout uint16
	HardTimeout uint16
	Priority    uint16
	BufferId    BufferId
	OutPort     PortNumber
	Flags       FlowModFlag
	Actions     []Action
}

type Cookie uint64
type FlowModCommand uint16
type BufferId uint32
type FlowModFlag uint16

const (
	OFPFC_ADD FlowModCommand = iota
	OFPFC_MODIFY
	OFPFC_MODIFY_STRICT
	OFPFC_DELETE
	OFPFC_DELETE_STRICT
)

var FlowModCommands = struct {
	Add          FlowModCommand
	Modify       FlowModCommand
	ModifyStrict FlowModCommand
	Delete       FlowModCommand
	DeleteStrict FlowModCommand
}{
	OFPFC_ADD,
	OFPFC_MODIFY,
	OFPFC_MODIFY_STRICT,
	OFPFC_DELETE,
	OFPFC_DELETE_STRICT,
}

const (
	OFPFF_SEND_FLOW_REM FlowModFlag = 1 << iota
	OFPFF_CHECK_OVERLAP
	OFPFF_EMERG
)

var FlowModFlags = struct {
	SendFlowRemoved FlowModFlag
	CheckOverlap    FlowModFlag
	Emergency       FlowModFlag
}{
	OFPFF_SEND_FLOW_REM,
	OFPFF_CHECK_OVERLAP,
	OFPFF_EMERG,
}

type PortMod struct {
	Header
	PortNumber      PortNumber
	HardwareAddress net.HardwareAddr
	Config          PortConfig
	Mask            PortConfig
	Advertise       PortFeature
	pad             [4]uint8
}

type QueueGetConfigRequest struct {
	Header
	Port PortNumber
	pad  [2]uint8
}

type QueueGetConfigReply struct {
	Header
	Port   PortNumber
	pad    [6]uint8
	Queues []PacketQueue
}

type StatsRequest struct {
	Header
	Type  StatsType
	Flags uint16
	Body  []uint8
}

type StatsReply struct {
	Header
	Type  StatsType
	Flags uint16
	Body  []uint8
}

type StatsType uint16

const (
	OFPST_DESC StatsType = iota
	OFPST_FLOW
	OFPST_AGGREGATE
	OFPST_TABLE
	OFPST_PORT
	OFPST_QUEUE
	OFPST_VENDOR StatsType = 0xffff
)

var StatsTypes = struct {
	Description StatsType
	Flow        StatsType
	Aggregate   StatsType
	Table       StatsType
	Port        StatsType
	Queue       StatsType
	Vendor      StatsType
}{
	OFPST_DESC,
	OFPST_FLOW,
	OFPST_AGGREGATE,
	OFPST_TABLE,
	OFPST_PORT,
	OFPST_QUEUE,
	OFPST_VENDOR,
}

type DescriptionStats struct {
	Manufacturer []uint8
	Hardware     []uint8
	Software     []uint8
	SerialNumber []uint8
	Datapath     []uint8
}

type FlowStatsRequest struct {
	Match   Match
	TalbeId uint8
	pad     uint8
	OutPort PortNumber
}

type FlowStatsReply struct {
	Length          uint16
	TableId         uint8
	pad             [1]uint8
	Match           Match
	DurationSec     uint32
	DurationNanoSec uint32
	Priority        uint16
	IdleTimeout     uint16
	HardTimeout     uint16
	pad2            [6]uint8
	Cookie          Cookie
	PacketCount     uint64
	ByteCount       uint64
	Actions         []ActionHeader
}

type AggregateStatsRequest struct {
	Match   Match
	TableId uint8
	pad     [1]uint8
	OutPort PortNumber
}

type AggregateStatsReply struct {
	PacketCount uint64
	ByteCount   uint64
	FlowCount   uint64
	pad         [4]uint8
}

type TableStatsReply struct {
	TableId      uint8
	pad          [3]uint8
	Name         []uint8
	Wildcards    Wildcard
	MaxEntries   uint32
	ActiveCount  uint32
	LookupCount  uint32
	MatchedCount uint32
}

type PortStatsRequest struct {
	PortNumber PortNumber
	pad        [6]uint8
}

type PortStatsReply struct {
	PortNumber      PortNumber
	pad             [6]uint8
	RxPackets       uint64
	TxPackets       uint64
	RxBytes         uint64
	TxBytes         uint64
	RxDropped       uint64
	TxDropped       uint64
	RxErrors        uint64
	TxErrors        uint64
	RxFrameErrors   uint64
	RxOverrunErrors uint64
	RxCrcErrors     uint64
	Collisions      uint64
}

type QueueStatsRequest struct {
	PortNumber PortNumber
	pad        [2]uint8
	QueueId    uint32
}

type QueueStatsReply struct {
	PortNumber PortNumber
	pad        [2]uint8
	QueueId    uint32
	TxBytes    uint64
	TxPackets  uint64
	TxErrors   uint64
}

type PacketOut struct {
	Header
	BufferId      uint32
	InPort        PortNumber
	ActionsLength uint16
	Actions       []ActionHeader
	Data          []uint8
}

type BarrierRequest struct {
	Header
}

type BarrierReply struct {
	Header
}

type PacketIn struct {
	Header
	BufferId    uint32
	TotalLength uint16
	InPort      PortNumber
	Reason      PacketInReason
	pad         [1]uint8
	Data        []uint8
}

type PacketInReason uint8

const (
	OFPR_NO_MATCH PacketInReason = iota
	OFPR_ACTION
)

var PacketInReasons = struct {
	NoMatch PacketInReason
	Action  PacketInReason
}{
	OFPR_NO_MATCH,
	OFPR_ACTION,
}

type FlowRemoved struct {
	Header
	Match           Match
	Cookie          Cookie
	Priority        uint16
	Reason          FlowRemovedReason
	pad             [1]uint8
	DurationSec     uint32
	DurationNanoSec uint32
	IdleTimeout     uint16
	pad2            [2]uint8
	PacketCount     uint64
	ByteCount       uint64
}

type FlowRemovedReason uint8

const (
	OFPRR_IDLE_TIMEOUT FlowRemovedReason = iota
	OFPRR_HARD_TIMEOUT
	OFPRR_DELETE
)

var FlowRemovedReasons = struct {
	IdleTimeout FlowRemovedReason
	HardTimeout FlowRemovedReason
	Delete      FlowRemovedReason
}{
	OFPRR_IDLE_TIMEOUT,
	OFPRR_HARD_TIMEOUT,
	OFPRR_DELETE,
}
<commit_msg>Declare PortStatus struct<commit_after>package of10

import (
	"net"

	. "github.com/oshothebig/goflow/openflow"
)

type FeaturesRequest struct {
	Header
}

type FeaturesReply struct {
	Header
	DatapathId   DatapathId
	Buffers      uint32
	Tables       uint8
	pad          [3]uint8
	Capabilities Capability
	Actions      ActionType
	Ports        []PhysicalPort
}

type DatapathId uint64
type Capability uint32

const (
	OFPC_FLOW_STATS Capability = 1 << iota
	OFPC_TABLE_STATS
	OFPC_PORT_STATS
	OFPC_STP
	OFPC_RESERVED
	OFPC_IP_REASM
	OFPC_QUEUE_STATS
	OFPC_ARP_MATCH_IP
)

var Capabilities = struct {
	FlowStats    Capability
	TableStats   Capability
	PortStats    Capability
	Stp          Capability
	Reserved     Capability
	IpReassemble Capability
	QueueStats   Capability
	ArpMatchIp   Capability
}{
	OFPC_FLOW_STATS,
	OFPC_TABLE_STATS,
	OFPC_PORT_STATS,
	OFPC_STP,
	OFPC_RESERVED,
	OFPC_IP_REASM,
	OFPC_QUEUE_STATS,
	OFPC_ARP_MATCH_IP,
}

type SwitchConfig struct {
	Header
	Flags          ConfigFlag
	MissSendLength uint16
}

type ConfigFlag uint16

const (
	OFPC_FRAG_NORMAL ConfigFlag = iota
	OFPC_FRAG_DROP
	OFPC_FRAG_REASM
	OFPC_FRAG_MASK
)

var ConfigFlags = struct {
	FragmentNormal     ConfigFlag
	FragmentDrop       ConfigFlag
	FragmentReassemble ConfigFlag
	FragmentMask       ConfigFlag
}{
	OFPC_FRAG_NORMAL,
	OFPC_FRAG_DROP,
	OFPC_FRAG_REASM,
	OFPC_FRAG_MASK,
}

type FlowMod struct {
	Header
	Match       Match
	Cookie      Cookie
	Command     FlowModCommand
	IdleTimeout uint16
	HardTimeout uint16
	Priority    uint16
	BufferId    BufferId
	OutPort     PortNumber
	Flags       FlowModFlag
	Actions     []Action
}

type Cookie uint64
type FlowModCommand uint16
type BufferId uint32
type FlowModFlag uint16

const (
	OFPFC_ADD FlowModCommand = iota
	OFPFC_MODIFY
	OFPFC_MODIFY_STRICT
	OFPFC_DELETE
	OFPFC_DELETE_STRICT
)

var FlowModCommands = struct {
	Add          FlowModCommand
	Modify       FlowModCommand
	ModifyStrict FlowModCommand
	Delete       FlowModCommand
	DeleteStrict FlowModCommand
}{
	OFPFC_ADD,
	OFPFC_MODIFY,
	OFPFC_MODIFY_STRICT,
	OFPFC_DELETE,
	OFPFC_DELETE_STRICT,
}

const (
	OFPFF_SEND_FLOW_REM FlowModFlag = 1 << iota
	OFPFF_CHECK_OVERLAP
	OFPFF_EMERG
)

var FlowModFlags = struct {
	SendFlowRemoved FlowModFlag
	CheckOverlap    FlowModFlag
	Emergency       FlowModFlag
}{
	OFPFF_SEND_FLOW_REM,
	OFPFF_CHECK_OVERLAP,
	OFPFF_EMERG,
}

type PortMod struct {
	Header
	PortNumber      PortNumber
	HardwareAddress net.HardwareAddr
	Config          PortConfig
	Mask            PortConfig
	Advertise       PortFeature
	pad             [4]uint8
}

type QueueGetConfigRequest struct {
	Header
	Port PortNumber
	pad  [2]uint8
}

type QueueGetConfigReply struct {
	Header
	Port   PortNumber
	pad    [6]uint8
	Queues []PacketQueue
}

type StatsRequest struct {
	Header
	Type  StatsType
	Flags uint16
	Body  []uint8
}

type StatsReply struct {
	Header
	Type  StatsType
	Flags uint16
	Body  []uint8
}

type StatsType uint16

const (
	OFPST_DESC StatsType = iota
	OFPST_FLOW
	OFPST_AGGREGATE
	OFPST_TABLE
	OFPST_PORT
	OFPST_QUEUE
	OFPST_VENDOR StatsType = 0xffff
)

var StatsTypes = struct {
	Description StatsType
	Flow        StatsType
	Aggregate   StatsType
	Table       StatsType
	Port        StatsType
	Queue       StatsType
	Vendor      StatsType
}{
	OFPST_DESC,
	OFPST_FLOW,
	OFPST_AGGREGATE,
	OFPST_TABLE,
	OFPST_PORT,
	OFPST_QUEUE,
	OFPST_VENDOR,
}

type DescriptionStats struct {
	Manufacturer []uint8
	Hardware     []uint8
	Software     []uint8
	SerialNumber []uint8
	Datapath     []uint8
}

type FlowStatsRequest struct {
	Match   Match
	TalbeId uint8
	pad     uint8
	OutPort PortNumber
}

type FlowStatsReply struct {
	Length          uint16
	TableId         uint8
	pad             [1]uint8
	Match           Match
	DurationSec     uint32
	DurationNanoSec uint32
	Priority        uint16
	IdleTimeout     uint16
	HardTimeout     uint16
	pad2            [6]uint8
	Cookie          Cookie
	PacketCount     uint64
	ByteCount       uint64
	Actions         []ActionHeader
}

type AggregateStatsRequest struct {
	Match   Match
	TableId uint8
	pad     [1]uint8
	OutPort PortNumber
}

type AggregateStatsReply struct {
	PacketCount uint64
	ByteCount   uint64
	FlowCount   uint64
	pad         [4]uint8
}

type TableStatsReply struct {
	TableId      uint8
	pad          [3]uint8
	Name         []uint8
	Wildcards    Wildcard
	MaxEntries   uint32
	ActiveCount  uint32
	LookupCount  uint32
	MatchedCount uint32
}

type PortStatsRequest struct {
	PortNumber PortNumber
	pad        [6]uint8
}

type PortStatsReply struct {
	PortNumber      PortNumber
	pad             [6]uint8
	RxPackets       uint64
	TxPackets       uint64
	RxBytes         uint64
	TxBytes         uint64
	RxDropped       uint64
	TxDropped       uint64
	RxErrors        uint64
	TxErrors        uint64
	RxFrameErrors   uint64
	RxOverrunErrors uint64
	RxCrcErrors     uint64
	Collisions      uint64
}

type QueueStatsRequest struct {
	PortNumber PortNumber
	pad        [2]uint8
	QueueId    uint32
}

type QueueStatsReply struct {
	PortNumber PortNumber
	pad        [2]uint8
	QueueId    uint32
	TxBytes    uint64
	TxPackets  uint64
	TxErrors   uint64
}

type PacketOut struct {
	Header
	BufferId      uint32
	InPort        PortNumber
	ActionsLength uint16
	Actions       []ActionHeader
	Data          []uint8
}

type BarrierRequest struct {
	Header
}

type BarrierReply struct {
	Header
}

type PacketIn struct {
	Header
	BufferId    uint32
	TotalLength uint16
	InPort      PortNumber
	Reason      PacketInReason
	pad         [1]uint8
	Data        []uint8
}

type PacketInReason uint8

const (
	OFPR_NO_MATCH PacketInReason = iota
	OFPR_ACTION
)

var PacketInReasons = struct {
	NoMatch PacketInReason
	Action  PacketInReason
}{
	OFPR_NO_MATCH,
	OFPR_ACTION,
}

type FlowRemoved struct {
	Header
	Match           Match
	Cookie          Cookie
	Priority        uint16
	Reason          FlowRemovedReason
	pad             [1]uint8
	DurationSec     uint32
	DurationNanoSec uint32
	IdleTimeout     uint16
	pad2            [2]uint8
	PacketCount     uint64
	ByteCount       uint64
}

type FlowRemovedReason uint8

const (
	OFPRR_IDLE_TIMEOUT FlowRemovedReason = iota
	OFPRR_HARD_TIMEOUT
	OFPRR_DELETE
)

var FlowRemovedReasons = struct {
	IdleTimeout FlowRemovedReason
	HardTimeout FlowRemovedReason
	Delete      FlowRemovedReason
}{
	OFPRR_IDLE_TIMEOUT,
	OFPRR_HARD_TIMEOUT,
	OFPRR_DELETE,
}

type PortStatus struct {
	Header
	Reason      PortStatusReason
	pad         [7]uint8
	Description PhysicalPort
}

type PortStatusReason uint8

const (
	OFPPR_ADD PortStatusReason = iota
	OFPPR_DELETE
	OFPPR_MODIFY
)

var PortStatusReasons = struct {
	Add    PortStatusReason
	Delete PortStatusReason
	Modify PortStatusReason
}{
	OFPPR_ADD,
	OFPPR_DELETE,
	OFPPR_MODIFY,
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"runtime"
)

var hostsFile string
var Log = fmt.Println

func init() {
	if runtime.GOOS == "windows" {
		hostsFile = "C:\\Windows\\System32\\drivers\\etc\\hosts"
	} else {
		hostsFile = "/etc/hosts"
	}

}

func main() {

	var (
		resp *http.Response
		re   *regexp.Regexp
		err  error
	)

	urlList := []string{
		"http://googleips-google.stor.sinaapp.com/hosts",
		"http://blog.my-eclipse.cn/hosts.txt",
	}

	for _, url := range urlList {

		resp, err = http.Get(url)
		if err != nil {
			log.Fatal(err)
		}
		defer resp.Body.Close()

		if resp.StatusCode == http.StatusOK {
			fmt.Println("Update your hosts......")
			break
		} else {
			continue
		}
	}

	if resp.StatusCode != http.StatusOK {
		fmt.Println("Sorry :(\nUpdate your hosts fail, program will exit.\n")
		os.Exit(-1)
	}
	hosts, err := ioutil.ReadFile(hostsFile)
	hostsContext := string(hosts)
	var pat = "(?s)#old hosts start.*#old hosts end"

	re, _ = regexp.Compile(pat)
	oldStr := re.FindAllStringSubmatch(string(hostsContext), -1)
	if len(oldStr) == 0 {
		hostsContext = "\n#old hosts start\n" + hostsContext + "\n#old hosts end\n"
	} else {
		hostsContext = oldStr[0][0]
	}

	file, err := os.OpenFile(hostsFile, os.O_RDWR|os.O_CREATE, os.ModePerm)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	file.WriteString(hostsContext)
	file.WriteString("\n \n")

	buf := make([]byte, 10240)
	for {
		numBytes, _ := resp.Body.Read(buf)
		if numBytes == 0 {
			break
		}
		file.WriteString(string(buf[:numBytes]))
	}
	fmt.Println("\nUpdate the hosts success, press ENTER to exit!")
	fmt.Scanln()
}
<commit_msg>	modified:   update_hosts.go<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"runtime"
)

var hostsFile string

func init() {
	if runtime.GOOS == "windows" {
		hostsFile = "C:\\Windows\\System32\\drivers\\etc\\hosts"
	} else {
		hostsFile = "/etc/hosts"
	}
}

func main() {

	var (
		resp *http.Response
		re   *regexp.Regexp
		err  error
	)

	urlList := []string{
		"http://googleips-google.stor.sinaapp.com/hosts",
		"http://blog.my-eclipse.cn/hosts.txt",
	}

	for _, url := range urlList {

		resp, err = http.Get(url)
		if err != nil {
			log.Fatal(err)
		}
		defer resp.Body.Close()

		if resp.StatusCode == http.StatusOK {
			fmt.Println("Update your hosts......")
			break
		} else {
			continue
		}
	}

	if resp.StatusCode != http.StatusOK {
		fmt.Println("Sorry :(\nUpdate your hosts fail, program will exit.\n")
		os.Exit(-1)
	}
	hosts, err := ioutil.ReadFile(hostsFile)
	hostsContext := string(hosts)
	var pat = "(?s)#old hosts start.*#old hosts end"

	re, _ = regexp.Compile(pat)
	oldStr := re.FindAllStringSubmatch(string(hostsContext), -1)
	if len(oldStr) == 0 {
		hostsContext = "\n#old hosts start\n" + hostsContext + "\n#old hosts end\n"
	} else {
		hostsContext = oldStr[0][0]
	}

	file, err := os.OpenFile(hostsFile, os.O_RDWR|os.O_CREATE, os.ModePerm)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	file.WriteString(hostsContext)
	file.WriteString("\n \n")

	buf := make([]byte, 10240)
	for {
		numBytes, _ := resp.Body.Read(buf)
		if numBytes == 0 {
			break
		}
		file.WriteString(string(buf[:numBytes]))
	}
	fmt.Println("\nUpdate the hosts success, press ENTER to exit!")
	fmt.Scanln()
}
<|endoftext|>


<commit_before>// prc_authority
package openpgp

import (
	"encoding/json"
	"errors"
	//	"io"
	//	"io/ioutil"
	//	"os"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"
	//	"time"
)

type Authority struct {
	HkpAddr           string   //Address in string format of the machine of authority
	DomainsUnderAuth  []string //Domains which are under this authority
	EmailOfAuthority  string   //e.g. admin@authorize.com
	AsciiPubKeyOfAuth string   //Ascii Armored Public Key of Auhtority
	//NOTE:Most probably the peer itself will be authority for keys directly uploaded to it.
}

func (auth Authority) GetString() (stringOfAuth string, err error) {
	if auth.HkpAddr != "" {
		stringOfAuth += auth.HkpAddr

		for _, dsua := range auth.DomainsUnderAuth {
			stringOfAuth += dsua
		}
		if auth.EmailOfAuthority == "" || auth.AsciiPubKeyOfAuth == "" {
			err = errors.New("Authority doesnt contain Email or Public Key")
		}
		stringOfAuth += auth.EmailOfAuthority + auth.AsciiPubKeyOfAuth
	} else {
		err = errors.New("Authority doesnt contain HKP Address")
		return
	}
	return
}

type AuthForDomain struct {
	domain    string
	authority Authority
}

func GetAuthForAllDomains(remoteStatesInJSON string) (authsForDomains []AuthForDomain, err error) {

	authOwn, _ := GetOwnAuthority()

	for _, domn := range authOwn.DomainsUnderAuth {
		authsForDomains = append(authsForDomains, AuthForDomain{domain: domn, authority: authOwn})
	}

	rmtAuthsForDom, err3 := GetAuthsForDomFromStates(remoteStatesInJSON)
	if err3 != nil {
		err = err3
		return
	}
	for _, authsFrDom := range rmtAuthsForDom {
		authsForDomains = append(authsForDomains, authsFrDom)
	}

	return
}

type ExplicitAuth struct {
	Emails []string //Emails that we are explicitly authorizing ourselves to handle.
}

func GetExplicitAuths(email string) (err error) {
	defer func() {
		if err != nil {
			log.Println(" Explicit Auth Verification for ", email, " failed: ", err)
		}
	}()
	//1.Read JSON
	//2.Get expAuth Struct object
	//3.single and regular Expression
	//4.Verify return err or nil

	str, err1 := GetFileContentsFromConfig("authority.ExplicitAuthFile")
	if err1 != nil {
		err = err1
		return
	}

	var exAuths []ExplicitAuth

	err = json.Unmarshal([]byte(str), &exAuths)
	if err != nil {
		return
	}
	found := false
	for _, excAuth := range exAuths {
		for _, excEmail := range excAuth.Emails {
			//First do simple matching,If no match then do regexp matching.
			if email == excEmail {
				found = true
				break
			}
			//Do regexp matching
			found, err = regexp.MatchString(excEmail, email)
			if err != nil {
				continue
			}
			if found {
				break
			}
		}
	}
	if found {
		return
	} else {
		err = errors.New("Not in our authority")
		return
	}
	return

}

func GetAuthForDomain(domainKey string, remoteStatesInJSON string) (authority Authority, err error) {
	allauths, err1 := GetAuthForAllDomains(remoteStatesInJSON)
	if err1 != nil {
		err = err1
		return
	}
	for _, auth := range allauths {
		if auth.domain == domainKey {
			authority = auth.authority
			return
		}
	}
	err = errors.New("Could Not Find Authority for that domain")
	return
}

func GetAuthForHkpAddr(HkpAddrKey string, remoteStatesInJSON string) (authority Authority, err error) {
	allauths, err1 := GetAuthForAllDomains(remoteStatesInJSON)
	if err1 != nil {
		err = err1
		return
	}
	for _, auth := range allauths {
		if auth.authority.HkpAddr == HkpAddrKey {
			authority = auth.authority
			return
		}
	}
	err = errors.New("Could Not Find Authority for that address")
	return
}
func GetAuthForEmail(email string, remoteStatesInJSON string) (auth Authority, err error) {
	splits := strings.Split(email, "@")
	domain := splits[len(splits)-1]
	auth, err = GetAuthForDomain(domain, remoteStatesInJSON)
	return
}

func GetOwnAuthority() (auth Authority, err error) {
	ownAddr := Config().GetString("authority.ownAddr")
	if ownAddr == "" {
		err = errors.New("No OwnAddress Set in Config File")
		return
	}
	domns := Config().GetStrings("authority.domainsUnderAuth")

	if len(domns) < 1 {
		err = errors.New("No doamins under own Found")
	}

	eml := Config().GetString("authority.email")
	if eml == "" {
		err = errors.New("No Email Set for own authority")
	}
	pubKey, err1 := GetFileContentsFromConfig("authority.pubKeyPath")
	if err1 != nil {
		err = err1
		return
	}
	auth = Authority{
		HkpAddr:           ownAddr, //Address in string format of the machine of authority
		DomainsUnderAuth:  domns,   //Domains which are under this authority
		EmailOfAuthority:  eml,     //e.g. admin@authorize.com
		AsciiPubKeyOfAuth: pubKey}
	return
}

func IsUnderAuth(publicKey Pubkey) (err error) {

	email, err1 := GetEmailFromPubKey(publicKey)
	if err1 != nil {
		err = err1
		return
	}

	ownAuth, err1 := GetOwnAuthority()
	if err1 != nil {
		//		fmt.Println(err1)
		err = err1
		return
	}

	underAuth := false
	splits := strings.Split(email, "@")
	domain := splits[len(splits)-1]

	msg := "Sorry. This server can upload/delete keys of the following domains only : "
	for _, dom := range ownAuth.DomainsUnderAuth {
		msg += dom + " "
		if dom == domain {
			underAuth = true
			break
		}
	}
	if underAuth == false {
		err = GetExplicitAuths(email)
		if err != nil {
			err = errors.New(msg)
		}
		return
	}
	return

}

type PksAuthTXT struct {
	authAddr       string //Web address of key server which has authority over this domain
	keyFingerPrint string //Long Fingerprint of Pub key of authority
	since          int64  //Time in Seconds since 1970
}

func GetPksAuthTXTRecord(domain string) (pksAuth PksAuthTXT, err error) {
	txts, err1 := net.LookupTXT(domain)
	found := false
	if err1 != nil {
		log.Println(err1)
		err = err1
		return
	}
	net.LookupAddr(domain)

	for _, r := range txts {
		log.Println("TXT of " + domain + " : " + r)
		rcds := strings.Split(r, "=")
		if len(rcds) != 2 {
			log.Println("Not Found = in TXT Record of " + domain)
		} else if rcds[0] != "pks" {
			log.Println("pks not found in TXT Record of " + domain)
		} else {
			rcds = strings.Split(rcds[1], " ")
			if len(rcds) != 3 {
				log.Println("Not Enough values in TXT Record of domain " + domain)
			} else {
				pksAuth.authAddr = rcds[0]
				//fmt.Println("pks_Addr=", rcds[0])
				//fmt.Println("key_FingerPrint=", rcds[1])
				pksAuth.keyFingerPrint = rcds[1]
				since, err2 := strconv.Atoi(rcds[2])
				if err2 != nil {
					log.Println(err2)
					err = err2
					return
				}
				pksAuth.since = int64(since)
				found = true
				break
				//fmt.Println("time=", time.Unix(int64(since), 0))
			}

		}
	}
	if found == false {
		err = errors.New("TXT Record of domain " + domain + "doesnt contain valid pksAuth Record")
		return

	}
	return
}
<commit_msg>explicit is now only one object,same in json too<commit_after>// prc_authority
package openpgp

import (
	"encoding/json"
	"errors"
	//	"io"
	//	"io/ioutil"
	//	"os"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"
	//	"time"
)

type Authority struct {
	HkpAddr           string   //Address in string format of the machine of authority
	DomainsUnderAuth  []string //Domains which are under this authority
	EmailOfAuthority  string   //e.g. admin@authorize.com
	AsciiPubKeyOfAuth string   //Ascii Armored Public Key of Auhtority
	//NOTE:Most probably the peer itself will be authority for keys directly uploaded to it.
}

func (auth Authority) GetString() (stringOfAuth string, err error) {
	if auth.HkpAddr != "" {
		stringOfAuth += auth.HkpAddr

		for _, dsua := range auth.DomainsUnderAuth {
			stringOfAuth += dsua
		}
		if auth.EmailOfAuthority == "" || auth.AsciiPubKeyOfAuth == "" {
			err = errors.New("Authority doesnt contain Email or Public Key")
		}
		stringOfAuth += auth.EmailOfAuthority + auth.AsciiPubKeyOfAuth
	} else {
		err = errors.New("Authority doesnt contain HKP Address")
		return
	}
	return
}

type AuthForDomain struct {
	domain    string
	authority Authority
}

func GetAuthForAllDomains(remoteStatesInJSON string) (authsForDomains []AuthForDomain, err error) {

	authOwn, _ := GetOwnAuthority()

	for _, domn := range authOwn.DomainsUnderAuth {
		authsForDomains = append(authsForDomains, AuthForDomain{domain: domn, authority: authOwn})
	}

	rmtAuthsForDom, err3 := GetAuthsForDomFromStates(remoteStatesInJSON)
	if err3 != nil {
		err = err3
		return
	}
	for _, authsFrDom := range rmtAuthsForDom {
		authsForDomains = append(authsForDomains, authsFrDom)
	}

	return
}

type ExplicitAuth struct {
	Emails []string //Emails that we are explicitly authorizing ourselves to handle.
}

func GetExplicitAuths(email string) (err error) {
	defer func() {
		if err != nil {
			log.Println(" Explicit Auth Verification for ", email, " failed: ", err)
		}
	}()
	//1.Read JSON
	//2.Get expAuth Struct object
	//3.single and regular Expression
	//4.Verify return err or nil

	str, err1 := GetFileContentsFromConfig("authority.ExplicitAuthFile")
	if err1 != nil {
		err = err1
		return
	}

	var exAuth ExplicitAuth

	err = json.Unmarshal([]byte(str), &exAuth)
	if err != nil {
		return
	}
	found := false

	for _, excEmail := range exAuth.Emails {
		//First do simple matching,If no match then do regexp matching.
		if email == excEmail {
			found = true
			break
		}
		//Do regexp matching
		found, err = regexp.MatchString(excEmail, email)
		if err != nil {
			continue
		}
		if found {
			break
		}
	}

	if found {
		return
	} else {
		err = errors.New("Not in our authority")
		return
	}
	return

}

func GetAuthForDomain(domainKey string, remoteStatesInJSON string) (authority Authority, err error) {
	allauths, err1 := GetAuthForAllDomains(remoteStatesInJSON)
	if err1 != nil {
		err = err1
		return
	}
	for _, auth := range allauths {
		if auth.domain == domainKey {
			authority = auth.authority
			return
		}
	}
	err = errors.New("Could Not Find Authority for that domain")
	return
}

func GetAuthForHkpAddr(HkpAddrKey string, remoteStatesInJSON string) (authority Authority, err error) {
	allauths, err1 := GetAuthForAllDomains(remoteStatesInJSON)
	if err1 != nil {
		err = err1
		return
	}
	for _, auth := range allauths {
		if auth.authority.HkpAddr == HkpAddrKey {
			authority = auth.authority
			return
		}
	}
	err = errors.New("Could Not Find Authority for that address")
	return
}
func GetAuthForEmail(email string, remoteStatesInJSON string) (auth Authority, err error) {
	splits := strings.Split(email, "@")
	domain := splits[len(splits)-1]
	auth, err = GetAuthForDomain(domain, remoteStatesInJSON)
	return
}

func GetOwnAuthority() (auth Authority, err error) {
	ownAddr := Config().GetString("authority.ownAddr")
	if ownAddr == "" {
		err = errors.New("No OwnAddress Set in Config File")
		return
	}
	domns := Config().GetStrings("authority.domainsUnderAuth")

	if len(domns) < 1 {
		err = errors.New("No doamins under own Found")
	}

	eml := Config().GetString("authority.email")
	if eml == "" {
		err = errors.New("No Email Set for own authority")
	}
	pubKey, err1 := GetFileContentsFromConfig("authority.pubKeyPath")
	if err1 != nil {
		err = err1
		return
	}
	auth = Authority{
		HkpAddr:           ownAddr, //Address in string format of the machine of authority
		DomainsUnderAuth:  domns,   //Domains which are under this authority
		EmailOfAuthority:  eml,     //e.g. admin@authorize.com
		AsciiPubKeyOfAuth: pubKey}
	return
}

func IsUnderAuth(publicKey Pubkey) (err error) {

	email, err1 := GetEmailFromPubKey(publicKey)
	if err1 != nil {
		err = err1
		return
	}

	ownAuth, err1 := GetOwnAuthority()
	if err1 != nil {
		//		fmt.Println(err1)
		err = err1
		return
	}

	underAuth := false
	splits := strings.Split(email, "@")
	domain := splits[len(splits)-1]

	msg := "Sorry. This server can upload/delete keys of the following domains only : "
	for _, dom := range ownAuth.DomainsUnderAuth {
		msg += dom + " "
		if dom == domain {
			underAuth = true
			break
		}
	}
	if underAuth == false {
		err = GetExplicitAuths(email)
		if err != nil {
			err = errors.New(msg)
		}
		return
	}
	return

}

type PksAuthTXT struct {
	authAddr       string //Web address of key server which has authority over this domain
	keyFingerPrint string //Long Fingerprint of Pub key of authority
	since          int64  //Time in Seconds since 1970
}

func GetPksAuthTXTRecord(domain string) (pksAuth PksAuthTXT, err error) {
	txts, err1 := net.LookupTXT(domain)
	found := false
	if err1 != nil {
		log.Println(err1)
		err = err1
		return
	}
	net.LookupAddr(domain)

	for _, r := range txts {
		log.Println("TXT of " + domain + " : " + r)
		rcds := strings.Split(r, "=")
		if len(rcds) != 2 {
			log.Println("Not Found = in TXT Record of " + domain)
		} else if rcds[0] != "pks" {
			log.Println("pks not found in TXT Record of " + domain)
		} else {
			rcds = strings.Split(rcds[1], " ")
			if len(rcds) != 3 {
				log.Println("Not Enough values in TXT Record of domain " + domain)
			} else {
				pksAuth.authAddr = rcds[0]
				//fmt.Println("pks_Addr=", rcds[0])
				//fmt.Println("key_FingerPrint=", rcds[1])
				pksAuth.keyFingerPrint = rcds[1]
				since, err2 := strconv.Atoi(rcds[2])
				if err2 != nil {
					log.Println(err2)
					err = err2
					return
				}
				pksAuth.since = int64(since)
				found = true
				break
				//fmt.Println("time=", time.Unix(int64(since), 0))
			}

		}
	}
	if found == false {
		err = errors.New("TXT Record of domain " + domain + "doesnt contain valid pksAuth Record")
		return

	}
	return
}
<|endoftext|>


<commit_before>package siad

// wallet.go contains things like signatures and scans the blockchain for
// available funds that can be spent.

import (
	"errors"

	"github.com/NebulousLabs/Andromeda/siacore"
	"github.com/NebulousLabs/Andromeda/signatures"
)

// Contains a secret key, the spend conditions associated with that key, the
// address associated with those spend conditions, and a list of outputs that
// the wallet knows how to spend.
type Wallet struct {
	State *siacore.State

	SecretKey       signatures.SecretKey
	SpendConditions siacore.SpendConditions

	OwnedOutputs         map[siacore.OutputID]struct{} // A list of outputs spendable by this wallet.
	SpentOutputs         map[siacore.OutputID]struct{} // A list of outputs spent by this wallet which may not yet be in the blockchain.
	OpenFreezeConditions map[siacore.BlockHeight]int   // A list of all heights at which freeze conditions are being used.

	// Host variables.
	HostSettings HostAnnouncement
}

// Most of the parameters are already in the file contract, but what's not
// specified is how much of the ContractFund comes from the client, and how
// much comes from the host. This specifies how much the client is to add to
// the contract.
type FileContractParameters struct {
	Transaction        siacore.Transaction
	FileContractIndex  int
	ClientContribution siacore.Currency
}

// Wallet.FreezeConditions
func (w *Wallet) FreezeConditions(unlockHeight siacore.BlockHeight) (fc siacore.SpendConditions) {
	fc = w.SpendConditions
	fc.TimeLock = unlockHeight
	return
}

// Creates a new wallet that can receive and spend coins.
func CreateWallet() (w *Wallet, err error) {
	w = new(Wallet)

	var pk signatures.PublicKey
	w.SecretKey, pk, err = signatures.GenerateKeyPair()
	w.SpendConditions.PublicKeys = append(w.SpendConditions.PublicKeys, pk)
	w.SpendConditions.NumSignatures = 1

	w.OwnedOutputs = make(map[siacore.OutputID]struct{})
	w.SpentOutputs = make(map[siacore.OutputID]struct{})
	w.OpenFreezeConditions = make(map[siacore.BlockHeight]int)

	return
}

// Scans all unspent transactions and adds the ones that are spendable by this
// wallet.
func (w *Wallet) Scan() {
	w.OwnedOutputs = make(map[siacore.OutputID]struct{})

	// Check for owned outputs from the standard SpendConditions.
	scanAddresses := make(map[siacore.CoinAddress]struct{})
	scanAddresses[w.SpendConditions.CoinAddress()] = struct{}{}

	// I'm not sure that it's the wallet's job to deal with freeze conditions.
	/*
		for height, _ := range w.OpenFreezeConditions {
			if height < State.Height() {
				freezeConditions := w.SpendConditions
				freezeConditions.TimeLock = height
				scanAddresses[freezeConditions.CoinAddress()] = struct{}{}
			}
		}
	*/

	// Get the matching set of outputs and add them to the OwnedOutputs map.
	outputs := w.State.ScanOutputs(scanAddresses)
	for _, output := range outputs {
		w.OwnedOutputs[output] = struct{}{}
	}
}

// fundTransaction() adds `amount` Currency to the inputs, creating a refund
// output for any excess.
func (w *Wallet) FundTransaction(amount siacore.Currency, t *siacore.Transaction) (err error) {
	// Check that a nonzero amount of coins is being sent.
	if amount == siacore.Currency(0) {
		err = errors.New("cannot send 0 coins")
		return
	}

	// Add to the list of inputs until enough funds have been allocated.
	total := siacore.Currency(0)
	var newInputs []siacore.Input
	for id, _ := range w.OwnedOutputs {
		// Check that the output has not already been assigned somewhere else.
		_, exists := w.SpentOutputs[id]
		if exists {
			continue
		}

		// Check that the output exists.
		var output siacore.Output
		output, err = w.State.Output(id)
		if err != nil {
			continue
		}

		// Create an input to add to the transaction.
		newInput := siacore.Input{
			OutputID:        id,
			SpendConditions: w.SpendConditions,
		}
		newInputs = append(newInputs, newInput)

		// Add the value of the output to the total and see if we've hit a
		// sufficient amount.
		total += output.Value
		if total >= amount {
			break
		}
	}

	// Check that the sum of the inputs is sufficient to complete the
	// transaction.
	if total < amount {
		err = errors.New("insufficient funds")
		return
	}

	// Add all of the inputs to the transaction.
	t.Inputs = append(t.Inputs, newInputs...)

	// Add all of the inputs to the spent outputs map.
	for _, input := range newInputs {
		w.SpentOutputs[input.OutputID] = struct{}{}
	}

	// Add a refund output to the transaction if needed.
	if total-amount > 0 {
		t.Outputs = append(t.Outputs, siacore.Output{Value: total - amount, SpendHash: w.SpendConditions.CoinAddress()})
	}

	return
}

// Wallet.signTransaction() takes a transaction and adds a signature for every input
// that the wallet understands how to spend.
func (w *Wallet) SignTransaction(t *siacore.Transaction) (err error) {
	for i, input := range t.Inputs {
		// If we recognize the input as something we are able to sign, we sign
		// the input.
		if input.SpendConditions.CoinAddress() == w.SpendConditions.CoinAddress() {
			txnSig := siacore.TransactionSignature{
				InputID: input.OutputID,
			}
			t.Signatures = append(t.Signatures, txnSig)

			sigHash := t.SigHash(i)
			t.Signatures[i].Signature, err = signatures.SignBytes(sigHash[:], w.SecretKey)
			if err != nil {
				return
			}
		}
	}

	return
}

// Wallet.SpendCoins creates a transaction sending 'amount' to 'address', and
// allocateding 'minerFee' as a miner fee. The transaction is submitted to the
// miner pool, but is also returned.
func (w *Wallet) SpendCoins(amount, minerFee siacore.Currency, address siacore.CoinAddress) (t siacore.Transaction, err error) {
	// Scan blockchain for outputs.
	w.Scan()

	// Add `amount` of free coins to the transaction.
	err = w.FundTransaction(amount+minerFee, &t)
	if err != nil {
		return
	}

	// Add the miner fee.
	t.MinerFees = append(t.MinerFees, minerFee)

	// Add the output to `address`.
	t.Outputs = append(t.Outputs, siacore.Output{Value: amount, SpendHash: address})

	// Sign each input.
	err = w.SignTransaction(&t)
	if err != nil {
		return
	}

	err = w.State.AcceptTransaction(t)
	if err != nil {
		return
	}

	return
}
<commit_msg>remove freeze conditions from standard wallet<commit_after>package siad

// wallet.go contains things like signatures and scans the blockchain for
// available funds that can be spent.

import (
	"errors"

	"github.com/NebulousLabs/Andromeda/siacore"
	"github.com/NebulousLabs/Andromeda/signatures"
)

// Contains a secret key, the spend conditions associated with that key, the
// address associated with those spend conditions, and a list of outputs that
// the wallet knows how to spend.
type Wallet struct {
	State *siacore.State

	SecretKey       signatures.SecretKey
	SpendConditions siacore.SpendConditions

	OwnedOutputs map[siacore.OutputID]struct{} // A list of outputs spendable by this wallet.
	SpentOutputs map[siacore.OutputID]struct{} // A list of outputs spent by this wallet which may not yet be in the blockchain.
}

// Most of the parameters are already in the file contract, but what's not
// specified is how much of the ContractFund comes from the client, and how
// much comes from the host. This specifies how much the client is to add to
// the contract.
type FileContractParameters struct {
	Transaction        siacore.Transaction
	FileContractIndex  int
	ClientContribution siacore.Currency
}

// Creates a new wallet that can receive and spend coins.
func CreateWallet() (w *Wallet, err error) {
	w = new(Wallet)

	var pk signatures.PublicKey
	w.SecretKey, pk, err = signatures.GenerateKeyPair()
	w.SpendConditions.PublicKeys = append(w.SpendConditions.PublicKeys, pk)
	w.SpendConditions.NumSignatures = 1

	w.OwnedOutputs = make(map[siacore.OutputID]struct{})
	w.SpentOutputs = make(map[siacore.OutputID]struct{})

	return
}

// Scans all unspent transactions and adds the ones that are spendable by this
// wallet.
func (w *Wallet) Scan() {
	w.OwnedOutputs = make(map[siacore.OutputID]struct{})

	// Check for owned outputs from the standard SpendConditions.
	scanAddresses := make(map[siacore.CoinAddress]struct{})
	scanAddresses[w.SpendConditions.CoinAddress()] = struct{}{}

	// Get the matching set of outputs and add them to the OwnedOutputs map.
	outputs := w.State.ScanOutputs(scanAddresses)
	for _, output := range outputs {
		w.OwnedOutputs[output] = struct{}{}
	}
}

// fundTransaction() adds `amount` Currency to the inputs, creating a refund
// output for any excess.
func (w *Wallet) FundTransaction(amount siacore.Currency, t *siacore.Transaction) (err error) {
	// Check that a nonzero amount of coins is being sent.
	if amount == siacore.Currency(0) {
		err = errors.New("cannot send 0 coins")
		return
	}

	// Add to the list of inputs until enough funds have been allocated.
	total := siacore.Currency(0)
	var newInputs []siacore.Input
	for id, _ := range w.OwnedOutputs {
		// Check that the output has not already been assigned somewhere else.
		_, exists := w.SpentOutputs[id]
		if exists {
			continue
		}

		// Check that the output exists.
		var output siacore.Output
		output, err = w.State.Output(id)
		if err != nil {
			continue
		}

		// Create an input to add to the transaction.
		newInput := siacore.Input{
			OutputID:        id,
			SpendConditions: w.SpendConditions,
		}
		newInputs = append(newInputs, newInput)

		// Add the value of the output to the total and see if we've hit a
		// sufficient amount.
		total += output.Value
		if total >= amount {
			break
		}
	}

	// Check that the sum of the inputs is sufficient to complete the
	// transaction.
	if total < amount {
		err = errors.New("insufficient funds")
		return
	}

	// Add all of the inputs to the transaction.
	t.Inputs = append(t.Inputs, newInputs...)

	// Add all of the inputs to the spent outputs map.
	for _, input := range newInputs {
		w.SpentOutputs[input.OutputID] = struct{}{}
	}

	// Add a refund output to the transaction if needed.
	if total-amount > 0 {
		t.Outputs = append(t.Outputs, siacore.Output{Value: total - amount, SpendHash: w.SpendConditions.CoinAddress()})
	}

	return
}

// Wallet.signTransaction() takes a transaction and adds a signature for every input
// that the wallet understands how to spend.
func (w *Wallet) SignTransaction(t *siacore.Transaction) (err error) {
	for i, input := range t.Inputs {
		// If we recognize the input as something we are able to sign, we sign
		// the input.
		if input.SpendConditions.CoinAddress() == w.SpendConditions.CoinAddress() {
			txnSig := siacore.TransactionSignature{
				InputID: input.OutputID,
			}
			t.Signatures = append(t.Signatures, txnSig)

			sigHash := t.SigHash(i)
			t.Signatures[i].Signature, err = signatures.SignBytes(sigHash[:], w.SecretKey)
			if err != nil {
				return
			}
		}
	}

	return
}

// Wallet.SpendCoins creates a transaction sending 'amount' to 'address', and
// allocateding 'minerFee' as a miner fee. The transaction is submitted to the
// miner pool, but is also returned.
func (w *Wallet) SpendCoins(amount, minerFee siacore.Currency, address siacore.CoinAddress) (t siacore.Transaction, err error) {
	// Scan blockchain for outputs.
	w.Scan()

	// Add `amount` of free coins to the transaction.
	err = w.FundTransaction(amount+minerFee, &t)
	if err != nil {
		return
	}

	// Add the miner fee.
	t.MinerFees = append(t.MinerFees, minerFee)

	// Add the output to `address`.
	t.Outputs = append(t.Outputs, siacore.Output{Value: amount, SpendHash: address})

	// Sign each input.
	err = w.SignTransaction(&t)
	if err != nil {
		return
	}

	err = w.State.AcceptTransaction(t)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2016 Andrew O'Neill

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt

import (
	"fmt"

	"golang.org/x/net/context"

	"github.com/boltdb/bolt"
	storage "github.com/foolusion/choices/elwinstorage"
	"github.com/golang/protobuf/proto"
)

var (
	environmentStaging    = []byte("staging")
	environmentProduction = []byte("production")
)

type Server struct {
	db *bolt.DB
}

func NewServer(file string) (*Server, error) {
	db, err := bolt.Open(file, 0600, nil)
	if err != nil {
		return nil, err
	}

	if err := db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucket(environmentStaging); err != nil {
			if err != bolt.ErrBucketExists {
				return err
			}
		}
		if _, err := tx.CreateBucket(environmentProduction); err != nil {
			if err != bolt.ErrBucketExists {
				return err
			}
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return &Server{db: db}, nil
}

func (s *Server) Close() error {
	return s.db.Close()
}

// All returns all the namespaces for a given environment.
func (s *Server) All(ctx context.Context, r *storage.AllRequest) (*storage.AllReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	ar := &storage.AllReply{}
	if err := s.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket(env).Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var ns storage.Namespace
			if err := proto.Unmarshal(v, &ns); err != nil {
				return err
			}
			ar.Namespaces = append(ar.Namespaces, &ns)
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return ar, nil
}

// Create creates a namespace in the given environment.
func (s *Server) Create(ctx context.Context, r *storage.CreateRequest) (*storage.CreateReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	ns := r.Namespace
	if ns == nil {
		return nil, fmt.Errorf("namespace is nil")
	}

	pns, err := proto.Marshal(ns)
	if err != nil {
		return nil, err
	}
	cr := &storage.CreateReply{}
	if err := s.db.Update(func(tx *bolt.Tx) error {
		err := tx.Bucket(env).Put([]byte(ns.Name), pns)
		if err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	cr.Namespace = ns
	return cr, nil
}

// Read returns the namespace matching the supplied name from the given
// environment.
func (s *Server) Read(ctx context.Context, r *storage.ReadRequest) (*storage.ReadReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	if len(r.Name) == 0 {
		return nil, fmt.Errorf("name is empty")
	}

	ns := storage.Namespace{}
	resp := &storage.ReadReply{}
	if err := s.db.View(func(tx *bolt.Tx) error {
		buf := tx.Bucket(env).Get([]byte(r.Name))
		if buf == nil {
			return fmt.Errorf("key not found")
		}
		if err := proto.Unmarshal(buf, &ns); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	resp.Namespace = &ns
	return resp, nil
}

// Update replaces the namespace in the given environment with the namespace
// supplied.
func (s *Server) Update(ctx context.Context, r *storage.UpdateRequest) (*storage.UpdateReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	ns := r.GetNamespace()
	if ns == nil {
		return nil, fmt.Errorf("namespace is nil")
	}

	pns, err := proto.Marshal(ns)
	if err != nil {
		return nil, err
	}
	resp := &storage.UpdateReply{}
	if err := s.db.Update(func(tx *bolt.Tx) error {
		err := tx.Bucket(env).Put([]byte(ns.Name), pns)
		if err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	resp.Namespace = ns
	return resp, nil
}

// Delete deletes the namespace from the given environment.
func (s *Server) Delete(ctx context.Context, r *storage.DeleteRequest) (*storage.DeleteReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	if len(r.Name) == 0 {
		return nil, fmt.Errorf("name is empty")
	}

	ns := storage.Namespace{}
	resp := &storage.DeleteReply{}
	if err := s.db.Update(func(tx *bolt.Tx) error {
		buf := tx.Bucket(env).Get([]byte(r.Name))
		if buf == nil {
			return fmt.Errorf("key not found")
		}
		if err := proto.Unmarshal(buf, &ns); err != nil {
			return err
		}
		return tx.Bucket(env).Delete([]byte(r.Name))
	}); err != nil {
		return nil, err
	}

	resp.Namespace = &ns
	return resp, nil
}

func envFromStorageRequest(e storage.Environment) []byte {
	switch e {
	case storage.Environment_Staging:
		return environmentStaging
	case storage.Environment_Production:
		return environmentProduction
	default:
		return environmentStaging
	}
	return environmentStaging
}
<commit_msg>storage/bolt: grpc error codes for not found requests<commit_after>// Copyright 2016 Andrew O'Neill

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt

import (
	"fmt"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"golang.org/x/net/context"

	"github.com/boltdb/bolt"
	storage "github.com/foolusion/choices/elwinstorage"
	"github.com/golang/protobuf/proto"
)

var (
	environmentStaging    = []byte("staging")
	environmentProduction = []byte("production")
)

type Server struct {
	db *bolt.DB
}

func NewServer(file string) (*Server, error) {
	db, err := bolt.Open(file, 0600, nil)
	if err != nil {
		return nil, err
	}

	if err := db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucket(environmentStaging); err != nil {
			if err != bolt.ErrBucketExists {
				return err
			}
		}
		if _, err := tx.CreateBucket(environmentProduction); err != nil {
			if err != bolt.ErrBucketExists {
				return err
			}
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return &Server{db: db}, nil
}

func (s *Server) Close() error {
	return s.db.Close()
}

// All returns all the namespaces for a given environment.
func (s *Server) All(ctx context.Context, r *storage.AllRequest) (*storage.AllReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	ar := &storage.AllReply{}
	if err := s.db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket(env).Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var ns storage.Namespace
			if err := proto.Unmarshal(v, &ns); err != nil {
				return err
			}
			ar.Namespaces = append(ar.Namespaces, &ns)
		}
		return nil
	}); err != nil {
		return nil, err
	}
	return ar, nil
}

// Create creates a namespace in the given environment.
func (s *Server) Create(ctx context.Context, r *storage.CreateRequest) (*storage.CreateReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	ns := r.Namespace
	if ns == nil {
		return nil, fmt.Errorf("namespace is nil")
	}

	pns, err := proto.Marshal(ns)
	if err != nil {
		return nil, err
	}
	cr := &storage.CreateReply{}
	if err := s.db.Update(func(tx *bolt.Tx) error {
		err := tx.Bucket(env).Put([]byte(ns.Name), pns)
		if err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	cr.Namespace = ns
	return cr, nil
}

// Read returns the namespace matching the supplied name from the given
// environment.
func (s *Server) Read(ctx context.Context, r *storage.ReadRequest) (*storage.ReadReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	if len(r.Name) == 0 {
		return nil, fmt.Errorf("name is empty")
	}

	ns := storage.Namespace{}
	resp := &storage.ReadReply{}
	if err := s.db.View(func(tx *bolt.Tx) error {
		buf := tx.Bucket(env).Get([]byte(r.Name))
		if buf == nil {
			return grpc.Errorf(codes.NotFound, "key not found")
		}
		if err := proto.Unmarshal(buf, &ns); err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	resp.Namespace = &ns
	return resp, nil
}

// Update replaces the namespace in the given environment with the namespace
// supplied.
func (s *Server) Update(ctx context.Context, r *storage.UpdateRequest) (*storage.UpdateReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	ns := r.GetNamespace()
	if ns == nil {
		return nil, fmt.Errorf("namespace is nil")
	}

	pns, err := proto.Marshal(ns)
	if err != nil {
		return nil, err
	}
	resp := &storage.UpdateReply{}
	if err := s.db.Update(func(tx *bolt.Tx) error {
		err := tx.Bucket(env).Put([]byte(ns.Name), pns)
		if err != nil {
			return err
		}
		return nil
	}); err != nil {
		return nil, err
	}
	resp.Namespace = ns
	return resp, nil
}

// Delete deletes the namespace from the given environment.
func (s *Server) Delete(ctx context.Context, r *storage.DeleteRequest) (*storage.DeleteReply, error) {
	if r == nil {
		return nil, fmt.Errorf("request is nil")
	}
	env := envFromStorageRequest(r.Environment)

	if len(r.Name) == 0 {
		return nil, fmt.Errorf("name is empty")
	}

	ns := storage.Namespace{}
	resp := &storage.DeleteReply{}
	if err := s.db.Update(func(tx *bolt.Tx) error {
		buf := tx.Bucket(env).Get([]byte(r.Name))
		if buf == nil {
			return grpc.Errorf(codes.NotFound, "key not found")
		}
		if err := proto.Unmarshal(buf, &ns); err != nil {
			return err
		}
		return tx.Bucket(env).Delete([]byte(r.Name))
	}); err != nil {
		return nil, err
	}

	resp.Namespace = &ns
	return resp, nil
}

func envFromStorageRequest(e storage.Environment) []byte {
	switch e {
	case storage.Environment_Staging:
		return environmentStaging
	case storage.Environment_Production:
		return environmentProduction
	default:
		return environmentStaging
	}
	return environmentStaging
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"strings"
)

func operationClean(pkg *build.Package) {
	filepath.Walk(pkg.Dir, func(filename string, info os.FileInfo, err error) error {
		if err != nil {
			fmt.Printf("error walking pkg dir to clean files: %v\n", err)
			os.Exit(1)
		}
		if info.IsDir() {
			return nil
		}
		verbosef("checking file '%s'\n", filename)
		if strings.HasSuffix(filename, ".rice-box.go") ||
			strings.HasSuffix(filename, ".rice-box.syso") {
			err := os.Remove(filename)
			if err != nil {
				fmt.Printf("error removing file (%s): %s\n", filename, err)
				os.Exit(-1)
			}
			verbosef("removed file '%s'\n", filename)
		}
		return nil
	})
}
<commit_msg>Fix clean after #68<commit_after>package main

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"strings"
)

func operationClean(pkg *build.Package) {
	filepath.Walk(pkg.Dir, func(filename string, info os.FileInfo, err error) error {
		if err != nil {
			fmt.Printf("error walking pkg dir to clean files: %v\n", err)
			os.Exit(1)
		}
		if info.IsDir() {
			return nil
		}
		verbosef("checking file '%s'\n", filename)
		if filepath.Base(filename) == "rice-box.go" ||
			strings.HasSuffix(filename, ".rice-box.go") ||
			strings.HasSuffix(filename, ".rice-box.syso") {
			err := os.Remove(filename)
			if err != nil {
				fmt.Printf("error removing file (%s): %s\n", filename, err)
				os.Exit(-1)
			}
			verbosef("removed file '%s'\n", filename)
		}
		return nil
	})
}
<|endoftext|>


<commit_before>package storage

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"cloud.google.com/go/storage"
	"go.skia.org/infra/fuzzer/go/common"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/gcs"
	"go.skia.org/infra/go/sklog"
)

// FuzzerGCSClient is the interface for all fuzzer-specific Google Cloud Storage (GCS)
// interactions. It embeds gcs.GCSClient to extend that functionality.
// See also go/fuzzer/tests.NewMockGCSClient() for a mock.
type FuzzerGCSClient interface {
	gcs.GCSClient
	// GetAllFuzzNamesInFolder returns all the fuzz names in a given GCS folder.  It basically
	// returns a list of all files that don't end with a .dump or .err, or error
	// if there was a problem.
	GetAllFuzzNamesInFolder(name string) (hashes []string, err error)
	// DeleteAllFilesInFolder recursively deletes anything in the given folder, i.e.
	// that starts with the given prefix. It can run on multiple go routines if processes
	// is set to > 1.
	DeleteAllFilesInFolder(folder string, processes int) error

	// DownloadAllFuzzes downloads all fuzzes of a given type "bad", "grey" at the specified
	// revision and returns a slice of all the paths on disk where they are. It can run on
	// multiple go routines if processes is set to > 1.
	DownloadAllFuzzes(downloadToPath, category, revision, architecture, fuzzType string, processes int) ([]string, error)
}

type fuzzerclient struct {
	gcs.GCSClient
}

func NewFuzzerGCSClient(s *storage.Client, bucket string) FuzzerGCSClient {
	return &fuzzerclient{gcs.NewGCSClient(s, bucket)}
}

func (g *fuzzerclient) GetAllFuzzNamesInFolder(name string) (hashes []string, err error) {
	filter := func(item *storage.ObjectAttrs) {
		name := item.Name
		fuzzHash := name[strings.LastIndex(name, "/")+1:]
		if !common.IsNameOfFuzz(fuzzHash) {
			return
		}
		hashes = append(hashes, fuzzHash)
	}

	if err = g.AllFilesInDirectory(context.Background(), name, filter); err != nil {
		return hashes, fmt.Errorf("Problem getting fuzzes from folder %s: %s", name, err)
	}
	return hashes, nil
}

// See the FuzzerGCSClient for more information about DeleteAllFilesInFolder.
func (g *fuzzerclient) DeleteAllFilesInFolder(folder string, processes int) error {
	if processes <= 0 {
		processes = 1
	}
	errCount := int32(0)
	var wg sync.WaitGroup
	toDelete := make(chan string, 1000)
	for i := 0; i < processes; i++ {
		go g.deleteHelper(&wg, toDelete, &errCount)
	}
	del := func(item *storage.ObjectAttrs) {
		toDelete <- item.Name
	}
	if err := g.AllFilesInDirectory(context.Background(), folder, del); err != nil {
		return err
	}
	close(toDelete)
	wg.Wait()
	if errCount > 0 {
		return fmt.Errorf("There were one or more problems when deleting files in folder %q", folder)
	}
	return nil

}

// deleteHelper spins and waits for work to come in on the toDelete channel.  When it does, it
// uses the storage client to delete the file from the given bucket.
func (g *fuzzerclient) deleteHelper(wg *sync.WaitGroup, toDelete <-chan string, errCount *int32) {
	wg.Add(1)
	defer wg.Done()
	for file := range toDelete {
		if err := g.DeleteFile(context.Background(), file); err != nil {
			// Ignore 404 errors on deleting, as they are already gone.
			if !strings.Contains(err.Error(), "statuscode 404") {
				sklog.Errorf("Problem deleting gs://%s/%s: %s", g.Bucket(), file, err)
				atomic.AddInt32(errCount, 1)
			}
		}
	}
}

// See the FuzzerGCSClient for more information about DownloadAllFuzzes.
func (g *fuzzerclient) DownloadAllFuzzes(downloadPath, category, revision, architecture, fuzzType string, processes int) ([]string, error) {
	completedCount := int32(0)
	var wg sync.WaitGroup
	toDownload := make(chan string, 1000)
	for i := 0; i < processes; i++ {
		go g.downloadHelper(toDownload, downloadPath, &wg, &completedCount)
	}
	fuzzPaths := []string{}

	download := func(item *storage.ObjectAttrs) {
		name := item.Name
		if !common.IsNameOfFuzz(name) {
			return
		}
		fuzzHash := name[strings.LastIndex(name, "/")+1:]
		if fuzzHash == "" {
			return
		}
		fuzzPaths = append(fuzzPaths, filepath.Join(downloadPath, fuzzHash))
		toDownload <- item.Name
	}
	if err := g.AllFilesInDirectory(context.Background(), fmt.Sprintf("%s/%s/%s/%s", category, revision, architecture, fuzzType), download); err != nil {
		return nil, fmt.Errorf("Problem iterating through all files: %s", err)
	}
	close(toDownload)
	wg.Wait()

	return fuzzPaths, nil
}

// download starts a go routine that waits for files to download from Google Storage and downloads
// them to downloadPath.  When it is done (on error or when the channel is closed), it signals to
// the WaitGroup that it is done. It also logs the progress on downloading the fuzzes.
func (g *fuzzerclient) downloadHelper(toDownload <-chan string, downloadPath string, wg *sync.WaitGroup, completedCounter *int32) {
	wg.Add(1)
	defer wg.Done()
	for file := range toDownload {
		hash := file[strings.LastIndex(file, "/")+1:]
		onDisk := filepath.Join(downloadPath, hash)
		if !fileutil.FileExists(onDisk) {
			contents, err := g.GetFileContents(context.Background(), file)
			if err != nil {
				sklog.Warningf("Problem downloading fuzz %s, continuing anyway: %s", file, err)
				continue
			}
			if err = ioutil.WriteFile(onDisk, contents, 0644); err != nil && !os.IsExist(err) {
				sklog.Warningf("Problem writing fuzz to %s, continuing anyway: %s", onDisk, err)
			}
		}
		atomic.AddInt32(completedCounter, 1)
		if *completedCounter%100 == 0 {
			sklog.Infof("%d fuzzes downloaded", *completedCounter)
		}
	}
}
<commit_msg>Fix potential race conditions with wait groups<commit_after>package storage

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"cloud.google.com/go/storage"
	"go.skia.org/infra/fuzzer/go/common"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/gcs"
	"go.skia.org/infra/go/sklog"
)

// FuzzerGCSClient is the interface for all fuzzer-specific Google Cloud Storage (GCS)
// interactions. It embeds gcs.GCSClient to extend that functionality.
// See also go/fuzzer/tests.NewMockGCSClient() for a mock.
type FuzzerGCSClient interface {
	gcs.GCSClient
	// GetAllFuzzNamesInFolder returns all the fuzz names in a given GCS folder.  It basically
	// returns a list of all files that don't end with a .dump or .err, or error
	// if there was a problem.
	GetAllFuzzNamesInFolder(name string) (hashes []string, err error)
	// DeleteAllFilesInFolder recursively deletes anything in the given folder, i.e.
	// that starts with the given prefix. It can run on multiple go routines if processes
	// is set to > 1.
	DeleteAllFilesInFolder(folder string, processes int) error

	// DownloadAllFuzzes downloads all fuzzes of a given type "bad", "grey" at the specified
	// revision and returns a slice of all the paths on disk where they are. It can run on
	// multiple go routines if processes is set to > 1.
	DownloadAllFuzzes(downloadToPath, category, revision, architecture, fuzzType string, processes int) ([]string, error)
}

type fuzzerclient struct {
	gcs.GCSClient
}

func NewFuzzerGCSClient(s *storage.Client, bucket string) FuzzerGCSClient {
	return &fuzzerclient{gcs.NewGCSClient(s, bucket)}
}

func (g *fuzzerclient) GetAllFuzzNamesInFolder(name string) (hashes []string, err error) {
	filter := func(item *storage.ObjectAttrs) {
		name := item.Name
		fuzzHash := name[strings.LastIndex(name, "/")+1:]
		if !common.IsNameOfFuzz(fuzzHash) {
			return
		}
		hashes = append(hashes, fuzzHash)
	}

	if err = g.AllFilesInDirectory(context.Background(), name, filter); err != nil {
		return hashes, fmt.Errorf("Problem getting fuzzes from folder %s: %s", name, err)
	}
	return hashes, nil
}

// See the FuzzerGCSClient for more information about DeleteAllFilesInFolder.
func (g *fuzzerclient) DeleteAllFilesInFolder(folder string, processes int) error {
	if processes <= 0 {
		processes = 1
	}
	errCount := int32(0)
	var wg sync.WaitGroup
	toDelete := make(chan string, 1000)
	for i := 0; i < processes; i++ {
		wg.Add(1)
		go g.deleteHelper(&wg, toDelete, &errCount)
	}
	del := func(item *storage.ObjectAttrs) {
		toDelete <- item.Name
	}
	if err := g.AllFilesInDirectory(context.Background(), folder, del); err != nil {
		return err
	}
	close(toDelete)
	wg.Wait()
	if errCount > 0 {
		return fmt.Errorf("There were one or more problems when deleting files in folder %q", folder)
	}
	return nil

}

// deleteHelper spins and waits for work to come in on the toDelete channel.  When it does, it
// uses the storage client to delete the file from the given bucket.
func (g *fuzzerclient) deleteHelper(wg *sync.WaitGroup, toDelete <-chan string, errCount *int32) {
	defer wg.Done()
	for file := range toDelete {
		if err := g.DeleteFile(context.Background(), file); err != nil {
			// Ignore 404 errors on deleting, as they are already gone.
			if !strings.Contains(err.Error(), "statuscode 404") {
				sklog.Errorf("Problem deleting gs://%s/%s: %s", g.Bucket(), file, err)
				atomic.AddInt32(errCount, 1)
			}
		}
	}
}

// See the FuzzerGCSClient for more information about DownloadAllFuzzes.
func (g *fuzzerclient) DownloadAllFuzzes(downloadPath, category, revision, architecture, fuzzType string, processes int) ([]string, error) {
	completedCount := int32(0)
	var wg sync.WaitGroup
	toDownload := make(chan string, 1000)
	for i := 0; i < processes; i++ {
		wg.Add(1)
		go g.downloadHelper(toDownload, downloadPath, &wg, &completedCount)
	}
	fuzzPaths := []string{}

	download := func(item *storage.ObjectAttrs) {
		name := item.Name
		if !common.IsNameOfFuzz(name) {
			return
		}
		fuzzHash := name[strings.LastIndex(name, "/")+1:]
		if fuzzHash == "" {
			return
		}
		fuzzPaths = append(fuzzPaths, filepath.Join(downloadPath, fuzzHash))
		toDownload <- item.Name
	}
	if err := g.AllFilesInDirectory(context.Background(), fmt.Sprintf("%s/%s/%s/%s", category, revision, architecture, fuzzType), download); err != nil {
		return nil, fmt.Errorf("Problem iterating through all files: %s", err)
	}
	close(toDownload)
	wg.Wait()

	return fuzzPaths, nil
}

// download starts a go routine that waits for files to download from Google Storage and downloads
// them to downloadPath.  When it is done (on error or when the channel is closed), it signals to
// the WaitGroup that it is done. It also logs the progress on downloading the fuzzes.
func (g *fuzzerclient) downloadHelper(toDownload <-chan string, downloadPath string, wg *sync.WaitGroup, completedCounter *int32) {
	defer wg.Done()
	for file := range toDownload {
		hash := file[strings.LastIndex(file, "/")+1:]
		onDisk := filepath.Join(downloadPath, hash)
		if !fileutil.FileExists(onDisk) {
			contents, err := g.GetFileContents(context.Background(), file)
			if err != nil {
				sklog.Warningf("Problem downloading fuzz %s, continuing anyway: %s", file, err)
				continue
			}
			if err = ioutil.WriteFile(onDisk, contents, 0644); err != nil && !os.IsExist(err) {
				sklog.Warningf("Problem writing fuzz to %s, continuing anyway: %s", onDisk, err)
			}
		}
		i := atomic.AddInt32(completedCounter, 1)
		if i%100 == 0 {
			sklog.Infof("%d fuzzes downloaded", i)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/mattn/go-shellwords"
	"github.com/smira/commander"
)

func aptlyTaskRun(cmd *commander.Command, args []string) error {

	var err error
	var cmd_list [][]string

	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
		var text string
		cmd_args := []string{}

		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
			return fmt.Errorf("No such file, %s\n", filename)
		}

		fmt.Println("Reading file...\n")

		file, err := os.Open(filename)

		if err != nil {
			return err
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)

		for scanner.Scan() {
			text = strings.TrimSpace(scanner.Text()) + ","
			parsed_args, _ := shellwords.Parse(text)
			cmd_args = append(cmd_args, parsed_args...)
		}

		if err = scanner.Err(); err != nil {
			return err
		}

		if len(cmd_args) == 0 {
			return fmt.Errorf("The file is empty. Exiting...\n")
		}

		cmd_list = formatCommands(cmd_args)

	} else if len(args) == 0 {
		var text string
		cmd_args := []string{}

		fmt.Println("Please enter one command per line and leave one blank when finished.")

		reader := bufio.NewReader(os.Stdin)
		for {
			fmt.Printf("> ")
			text, _ = reader.ReadString('\n')
			if text == "\n" {
				break
			} else {
				text = strings.TrimSpace(text) + ","
				parsed_args, _ := shellwords.Parse(text)
				cmd_args = append(cmd_args, parsed_args...)
			}
		}

		if len(cmd_args) == 0 {
			return fmt.Errorf("Nothing entered. Exiting...\n")
		}

		cmd_list = formatCommands(cmd_args)

	} else {
		cmd_list = formatCommands(args)
	}

	commandErrored := false

	for i, command := range cmd_list {
		if !commandErrored {
			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
			context.Progress().Flush()

			returnCode := Run(RootCommand(), command, false)
			if returnCode != 0 {
				commandErrored = true
			}
			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
			CleanupContext()
		} else {
			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
		}
	}

	if commandErrored {
		err = fmt.Errorf("At least one command has reported an error\n")
	}

	return err
}

func formatCommands(args []string) [][]string {

	var cmd []string
	var cmd_array [][]string

	for _, s := range args {
		if s_trimmed := strings.TrimRight(s, ","); s_trimmed != s {
			cmd = append(cmd, s_trimmed)
			cmd_array = append(cmd_array, cmd)
			cmd = []string{}
		} else {
			cmd = append(cmd, s)
		}
	}

	if len(cmd) > 0 {
		cmd_array = append(cmd_array, cmd)
	}

	return cmd_array
}

func makeCmdTaskRun() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlyTaskRun,
		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
		Short:     "run aptly tasks",
		Long: `
Command helps origanise multiple aptly commands in one single aptly task, running as single thread.

Example:

  $ aptly task run
  > repo create local
  > repo add local pkg1
  > publish repo local
  > serve
  >

`,
	}

	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
	return cmd
}
<commit_msg>Style fixes. #96<commit_after>package cmd

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/mattn/go-shellwords"
	"github.com/smira/commander"
)

func aptlyTaskRun(cmd *commander.Command, args []string) error {
	var err error
	var cmd_list [][]string

	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
		var text string
		cmd_args := []string{}

		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
			return fmt.Errorf("No such file, %s\n", filename)
		}

		fmt.Println("Reading file...\n")

		file, err := os.Open(filename)

		if err != nil {
			return err
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)

		for scanner.Scan() {
			text = strings.TrimSpace(scanner.Text()) + ","
			parsed_args, _ := shellwords.Parse(text)
			cmd_args = append(cmd_args, parsed_args...)
		}

		if err = scanner.Err(); err != nil {
			return err
		}

		if len(cmd_args) == 0 {
			return fmt.Errorf("The file is empty. Exiting...\n")
		}

		cmd_list = formatCommands(cmd_args)
	} else if len(args) == 0 {
		var text string
		cmd_args := []string{}

		fmt.Println("Please enter one command per line and leave one blank when finished.")

		reader := bufio.NewReader(os.Stdin)
		for {
			fmt.Printf("> ")
			text, _ = reader.ReadString('\n')
			if text == "\n" {
				break
			} else {
				text = strings.TrimSpace(text) + ","
				parsed_args, _ := shellwords.Parse(text)
				cmd_args = append(cmd_args, parsed_args...)
			}
		}

		if len(cmd_args) == 0 {
			return fmt.Errorf("Nothing entered. Exiting...\n")
		}

		cmd_list = formatCommands(cmd_args)
	} else {
		cmd_list = formatCommands(args)
	}

	commandErrored := false

	for i, command := range cmd_list {
		if !commandErrored {
			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
			context.Progress().Flush()

			returnCode := Run(RootCommand(), command, false)
			if returnCode != 0 {
				commandErrored = true
			}
			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
			CleanupContext()
		} else {
			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
		}
	}

	if commandErrored {
		err = fmt.Errorf("At least one command has reported an error\n")
	}

	return err
}

func formatCommands(args []string) [][]string {
	var cmd []string
	var cmd_array [][]string

	for _, s := range args {
		if s_trimmed := strings.TrimRight(s, ","); s_trimmed != s {
			cmd = append(cmd, s_trimmed)
			cmd_array = append(cmd_array, cmd)
			cmd = []string{}
		} else {
			cmd = append(cmd, s)
		}
	}

	if len(cmd) > 0 {
		cmd_array = append(cmd_array, cmd)
	}

	return cmd_array
}

func makeCmdTaskRun() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlyTaskRun,
		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
		Short:     "run aptly tasks",
		Long: `
Command helps origanise multiple aptly commands in one single aptly task, running as single thread.

Example:

  $ aptly task run
  > repo create local
  > repo add local pkg1
  > publish repo local
  > serve
  >

`,
	}

	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
	return cmd
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

package ghttp

import (
    "os"
    "sync"
    "errors"
    "strings"
    "reflect"
    "runtime"
    "net/http"
    "gitee.com/johng/gf/g/os/glog"
    "gitee.com/johng/gf/g/os/gproc"
    "gitee.com/johng/gf/g/os/gcache"
    "gitee.com/johng/gf/g/util/gconv"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/container/gtype"
    "gitee.com/johng/gf/g/container/gqueue"
    "gitee.com/johng/gf/g/os/gspath"
    "gitee.com/johng/gf/g/os/gfile"
    "gitee.com/johng/gf/g/os/genv"
    "github.com/gorilla/websocket"
    "gitee.com/johng/gf/g/os/gtime"
    "time"
    "container/list"
)

const (
    gHTTP_METHODS              = "GET,PUT,POST,DELETE,PATCH,HEAD,CONNECT,OPTIONS,TRACE"
    gDEFAULT_SERVER            = "default"
    gDEFAULT_DOMAIN            = "default"
    gDEFAULT_METHOD            = "ALL"
    gDEFAULT_COOKIE_PATH       = "/"             // 默认path
    gDEFAULT_COOKIE_MAX_AGE    = 86400*365       // 默认cookie有效期(一年)
    gDEFAULT_SESSION_MAX_AGE   = 600             // 默认session有效期(600秒)
    gDEFAULT_SESSION_ID_NAME   = "gfsessionid"   // 默认存放Cookie中的SessionId名称
    gSERVER_STATUS_STOPPED     = 0               // Server状态：停止
    gSERVER_STATUS_RUNNING     = 1               // Server状态：运行
)

// ghttp.Server结构体
type Server struct {
    // 基本属性变量
    name             string                   // 服务名称，方便识别
    paths            *gspath.SPath            // 静态文件检索对象
    config           ServerConfig             // 配置对象
    status           int8                     // 当前服务器状态(0：未启动，1：运行中)
    servers          []*gracefulServer        // 底层http.Server列表
    methodsMap       map[string]bool          // 所有支持的HTTP Method(初始化时自动填充)
    servedCount      *gtype.Int               // 已经服务的请求数(4-8字节，不考虑溢出情况)，同时作为请求ID
    closeQueue       *gqueue.Queue            // 请求结束的关闭队列(存放的是需要异步关闭处理的*Request对象)
    // 服务注册相关
    handlerTree      map[string]interface{}   // 所有注册的回调函数(路由表，树型结构，哈希表+链表优先级匹配)
    handlerCache     *gcache.Cache            // 服务注册路由内存缓存
    hooksCache       *gcache.Cache            // 事件回调路由内存缓存
    // 自定义状态码回调
    hsmu             sync.RWMutex             // status handler互斥锁
    statusHandlerMap map[string]HandlerFunc   // 不同状态码下的注册处理方法(例如404状态时的处理方法)
    // COOKIE
    cookieMaxAge     *gtype.Int               // Cookie有效期
    cookies          *gmap.IntInterfaceMap    // 当前服务器正在服务(请求正在执行)的Cookie(每个请求一个Cookie对象)
    // SESSION
    sessionMaxAge    *gtype.Int               // Session有效期
    sessionIdName    *gtype.String            // SessionId名称
    sessions         *gcache.Cache            // Session内存缓存
    // 日志相关属性
    logPath          *gtype.String            // 存放日志的目录路径
    logHandler       *gtype.Interface         // 自定义日志处理回调方法
    errorLogEnabled  *gtype.Bool              // 是否开启error log
    accessLogEnabled *gtype.Bool              // 是否开启access log
    accessLogger     *glog.Logger             // access log日志对象
    errorLogger      *glog.Logger             // error log日志对象
}

// 路由对象
type Router struct {
    Uri      string       // 注册时的pattern - uri
    Method   string       // 注册时的pattern - method
    Domain   string       // 注册时的pattern - domain
    RegRule  string       // 路由规则解析后对应的正则表达式
    RegNames []string     // 路由规则解析后对应的变量名称数组
    Priority int          // 优先级，用于链表排序，值越大优先级越高
}

// 域名、URI与回调函数的绑定记录表
type handlerMap  map[string]*handlerItem

// http回调函数注册信息
type handlerItem struct {
    ctype    reflect.Type // 控制器类型(反射类型)
    fname    string       // 回调方法名称
    faddr    HandlerFunc  // 准确的执行方法内存地址(与以上两个参数二选一)
}

// 路由注册项(这里使用了非并发安全的list.List，因为该对象的使用统一是由htmu互斥锁保证并发安全)
type handlerRegisterItem struct {
    handler    *handlerItem             // 准确的执行方法内存地址
    hooks      map[string]*list.List   // 当前的事件回调注册，键名为事件名称，键值为事件执行方法链表
    router     *Router                  // 注册时绑定的路由对象
}

// 根据特定URL.Path解析后的路由检索结果项
type handlerParsedItem struct {
    item    *handlerRegisterItem        // 路由注册项
    values   map[string][]string        // 特定URL.Path的Router解析参数
}

// HTTP注册函数
type HandlerFunc func(r *Request)

// 文件描述符map
type listenerFdMap map[string]string

// Server表，用以存储和检索名称与Server对象之间的关联关系
var serverMapping = gmap.NewStringInterfaceMap()

// Web Socket默认配置
var wsUpgrader    = websocket.Upgrader{}

// Web Server已完成服务事件通道，当有事件时表示服务完成，当前进程退出
var doneChan      = make(chan struct{}, 1000)

// Web Server进程初始化
func init() {
    // 如果是完整重启，那么需要等待主进程销毁后，才开始执行监听，防止端口冲突
    if genv.Get(gADMIN_ACTION_RESTART_ENVKEY) != "" {
        if p, e := os.FindProcess(gproc.PPid()); e == nil {
            p.Kill()
            p.Wait()
        } else {
            glog.Error(e)
        }
    }

    // 信号量管理操作监听
    go handleProcessSignal()
    // 异步监听进程间消息
    go handleProcessMessage()
}

// 获取/创建一个默认配置的HTTP Server(默认监听端口是80)
// 单例模式，请保证name的唯一性
func GetServer(name...interface{}) (*Server) {
    sname := gDEFAULT_SERVER
    if len(name) > 0 {
        sname = gconv.String(name[0])
    }
    if s := serverMapping.Get(sname); s != nil {
        return s.(*Server)
    }
    s := &Server {
        name             : sname,
        paths            : gspath.New(),
        servers          : make([]*gracefulServer, 0),
        methodsMap       : make(map[string]bool),
        statusHandlerMap : make(map[string]HandlerFunc),
        handlerTree      : make(map[string]interface{}),
        handlerCache     : gcache.New(),
        hooksCache       : gcache.New(),
        cookies          : gmap.NewIntInterfaceMap(),
        sessions         : gcache.New(),
        cookieMaxAge     : gtype.NewInt(gDEFAULT_COOKIE_MAX_AGE),
        sessionMaxAge    : gtype.NewInt(gDEFAULT_SESSION_MAX_AGE),
        sessionIdName    : gtype.NewString(gDEFAULT_SESSION_ID_NAME),
        servedCount      : gtype.NewInt(),
        closeQueue       : gqueue.New(),
        logPath          : gtype.NewString(),
        accessLogEnabled : gtype.NewBool(),
        errorLogEnabled  : gtype.NewBool(true),
        accessLogger     : glog.New(),
        errorLogger      : glog.New(),
        logHandler       : gtype.NewInterface(),
    }
    s.errorLogger.SetBacktraceSkip(4)
    s.accessLogger.SetBacktraceSkip(4)
    // 设置路由解析缓存上限，使用LRU进行缓存淘汰
    s.hooksCache.SetCap(10000)
    s.handlerCache.SetCap(10000)
    for _, v := range strings.Split(gHTTP_METHODS, ",") {
        s.methodsMap[v] = true
    }
    s.SetConfig(defaultServerConfig)
    serverMapping.Set(sname, s)
    return s
}

// 作为守护协程异步执行(当同一进程中存在多个Web Server时，需要采用这种方式执行)
// 需要结合Wait方式一起使用
func (s *Server) Start() error {
    // 如果设置了静态文件目录，那么严格按照静态文件目录进行检索
    // 否则，默认使用当前可执行文件目录，并且如果是开发环境，默认也会添加main包的源码目录路径做为二级检索
    if s.config.ServerRoot != "" {
        s.paths.Set(s.config.ServerRoot)
    } else {
        s.paths.Set(gfile.SelfDir())
        if p := gfile.MainPkgPath(); gfile.Exists(p) {
            s.paths.Add(p)
        }
    }

    if s.status == gSERVER_STATUS_RUNNING {
        return errors.New("server is already running")
    }
    // 底层http server配置
    if s.config.Handler == nil {
        s.config.Handler = http.HandlerFunc(s.defaultHttpHandle)
    }

    // 启动http server
    reloaded := false
    fdMapStr := genv.Get(gADMIN_ACTION_RELOAD_ENVKEY)
    if len(fdMapStr) > 0 {
        sfm := bufferToServerFdMap([]byte(fdMapStr))
        if v, ok := sfm[s.name]; ok {
            s.startServer(v)
            reloaded = true
        }
    }
    if !reloaded {
        s.startServer(nil)
    }

    // 如果是子进程，那么服务开启后通知父进程销毁
    if gproc.IsChild() {
        gtime.SetTimeout(2*time.Second, func() {
            gproc.Send(gproc.PPid(), []byte("exit"), gADMIN_GPROC_COMM_GROUP)
        })
    }

    // 开启异步关闭队列处理循环
    s.startCloseQueueLoop()
    return nil
}

// 阻塞执行监听
func (s *Server) Run() error {
    if err := s.Start(); err != nil {
        return err
    }
    // 阻塞等待服务执行完成
    <- doneChan

    glog.Printfln("%d: all servers shutdown", gproc.Pid())
    return nil
}

// 阻塞等待所有Web Server停止，常用于多Web Server场景，以及需要将Web Server异步运行的场景
// 这是一个与进程相关的方法
func Wait() {
    // 阻塞等待服务执行完成
    <- doneChan

    glog.Printfln("%d: all servers shutdown", gproc.Pid())
}

// 开启底层Web Server执行
func (s *Server) startServer(fdMap listenerFdMap) {
    var httpsEnabled bool
    if len(s.config.HTTPSCertPath) > 0 && len(s.config.HTTPSKeyPath) > 0 {
        // ================
        // HTTPS
        // ================
        if len(s.config.HTTPSAddr) == 0 {
            if len(s.config.Addr) > 0 {
                s.config.HTTPSAddr = s.config.Addr
                s.config.Addr      = ""
            } else {
                s.config.HTTPSAddr = gDEFAULT_HTTPS_ADDR
            }
        }
        httpsEnabled = len(s.config.HTTPSAddr) > 0
        var array []string
        if v, ok := fdMap["https"]; ok && len(v) > 0 {
            array = strings.Split(v, ",")
        } else {
            array = strings.Split(s.config.HTTPSAddr, ",")
        }
        for _, v := range array {
            if len(v) == 0 {
                continue
            }
            fd    := 0
            addr  := v
            array := strings.Split(v, "#")
            if len(array) > 1 {
                addr = array[0]
                // windows系统不支持文件描述符传递socket通信平滑交接，因此只能完整重启
                if runtime.GOOS != "windows" {
                    fd = gconv.Int(array[1])
                }
            }
            if fd > 0 {
                s.servers = append(s.servers, s.newGracefulServer(addr, fd))
            } else {
                s.servers = append(s.servers, s.newGracefulServer(addr))
            }
            s.servers[len(s.servers) - 1].isHttps = true
        }
    }
    // ================
    // HTTP
    // ================
    // 当HTTPS服务未启用时，默认HTTP地址才会生效
    if !httpsEnabled && len(s.config.Addr) == 0 {
        s.config.Addr = gDEFAULT_HTTP_ADDR
    }
    var array []string
    if v, ok := fdMap["http"]; ok && len(v) > 0 {
        array = strings.Split(v, ",")
    } else {
        array = strings.Split(s.config.Addr, ",")
    }
    for _, v := range array {
        if len(v) == 0 {
            continue
        }
        fd    := 0
        addr  := v
        array := strings.Split(v, "#")
        if len(array) > 1 {
            addr = array[0]
            // windows系统不支持文件描述符传递socket通信平滑交接，因此只能完整重启
            if runtime.GOOS != "windows" {
                fd = gconv.Int(array[1])
            }
        }
        if fd > 0 {
            s.servers = append(s.servers, s.newGracefulServer(addr, fd))
        } else {
            s.servers = append(s.servers, s.newGracefulServer(addr))
        }
    }
    // 开始执行异步监听
    for _, v := range s.servers {
        go func(server *gracefulServer) {
            var err error
            if server.isHttps {
                err = server.ListenAndServeTLS(s.config.HTTPSCertPath, s.config.HTTPSKeyPath)
            } else {
                err = server.ListenAndServe()
            }
            // 如果非关闭错误，那么提示报错，否则认为是正常的服务关闭操作
            if err != nil && !strings.EqualFold(http.ErrServerClosed.Error(), err.Error()) {
                glog.Error(err)
            }
        }(v)
    }

    s.status = gSERVER_STATUS_RUNNING
}

// 获取当前监听的文件描述符信息，构造成map返回
func (s *Server) getListenerFdMap() map[string]string {
    m := map[string]string {
        "https" : "",
        "http"  : "",
    }
    // s.servers是从HTTPS到HTTP优先级遍历，解析的时候也应当按照这个顺序读取fd
    for _, v := range s.servers {
        str := v.addr + "#" + gconv.String(v.Fd()) + ","
        if v.isHttps {
            m["https"] += str
        } else {
            m["http"]  += str
        }
    }
    // 去掉末尾的","号
    if len(m["https"]) > 0 {
        m["https"] = m["https"][0 : len(m["https"]) - 1]
    }
    if len(m["http"]) > 0 {
        m["http"] = m["http"][0 : len(m["http"]) - 1]
    }

    return m
}
<commit_msg>修正ghttp包没有使用Server仍然初始化相关异步goroutine的问题<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

package ghttp

import (
    "os"
    "sync"
    "errors"
    "strings"
    "reflect"
    "runtime"
    "net/http"
    "gitee.com/johng/gf/g/os/glog"
    "gitee.com/johng/gf/g/os/gproc"
    "gitee.com/johng/gf/g/os/gcache"
    "gitee.com/johng/gf/g/util/gconv"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/container/gtype"
    "gitee.com/johng/gf/g/container/gqueue"
    "gitee.com/johng/gf/g/os/gspath"
    "gitee.com/johng/gf/g/os/gfile"
    "gitee.com/johng/gf/g/os/genv"
    "github.com/gorilla/websocket"
    "gitee.com/johng/gf/g/os/gtime"
    "time"
    "container/list"
)

const (
    gHTTP_METHODS              = "GET,PUT,POST,DELETE,PATCH,HEAD,CONNECT,OPTIONS,TRACE"
    gDEFAULT_SERVER            = "default"
    gDEFAULT_DOMAIN            = "default"
    gDEFAULT_METHOD            = "ALL"
    gDEFAULT_COOKIE_PATH       = "/"             // 默认path
    gDEFAULT_COOKIE_MAX_AGE    = 86400*365       // 默认cookie有效期(一年)
    gDEFAULT_SESSION_MAX_AGE   = 600             // 默认session有效期(600秒)
    gDEFAULT_SESSION_ID_NAME   = "gfsessionid"   // 默认存放Cookie中的SessionId名称
    gSERVER_STATUS_STOPPED     = 0               // Server状态：停止
    gSERVER_STATUS_RUNNING     = 1               // Server状态：运行
)

// ghttp.Server结构体
type Server struct {
    // 基本属性变量
    name             string                   // 服务名称，方便识别
    paths            *gspath.SPath            // 静态文件检索对象
    config           ServerConfig             // 配置对象
    status           int8                     // 当前服务器状态(0：未启动，1：运行中)
    servers          []*gracefulServer        // 底层http.Server列表
    methodsMap       map[string]bool          // 所有支持的HTTP Method(初始化时自动填充)
    servedCount      *gtype.Int               // 已经服务的请求数(4-8字节，不考虑溢出情况)，同时作为请求ID
    closeQueue       *gqueue.Queue            // 请求结束的关闭队列(存放的是需要异步关闭处理的*Request对象)
    // 服务注册相关
    handlerTree      map[string]interface{}   // 所有注册的回调函数(路由表，树型结构，哈希表+链表优先级匹配)
    handlerCache     *gcache.Cache            // 服务注册路由内存缓存
    hooksCache       *gcache.Cache            // 事件回调路由内存缓存
    // 自定义状态码回调
    hsmu             sync.RWMutex             // status handler互斥锁
    statusHandlerMap map[string]HandlerFunc   // 不同状态码下的注册处理方法(例如404状态时的处理方法)
    // COOKIE
    cookieMaxAge     *gtype.Int               // Cookie有效期
    cookies          *gmap.IntInterfaceMap    // 当前服务器正在服务(请求正在执行)的Cookie(每个请求一个Cookie对象)
    // SESSION
    sessionMaxAge    *gtype.Int               // Session有效期
    sessionIdName    *gtype.String            // SessionId名称
    sessions         *gcache.Cache            // Session内存缓存
    // 日志相关属性
    logPath          *gtype.String            // 存放日志的目录路径
    logHandler       *gtype.Interface         // 自定义日志处理回调方法
    errorLogEnabled  *gtype.Bool              // 是否开启error log
    accessLogEnabled *gtype.Bool              // 是否开启access log
    accessLogger     *glog.Logger             // access log日志对象
    errorLogger      *glog.Logger             // error log日志对象
}

// 路由对象
type Router struct {
    Uri      string       // 注册时的pattern - uri
    Method   string       // 注册时的pattern - method
    Domain   string       // 注册时的pattern - domain
    RegRule  string       // 路由规则解析后对应的正则表达式
    RegNames []string     // 路由规则解析后对应的变量名称数组
    Priority int          // 优先级，用于链表排序，值越大优先级越高
}

// 域名、URI与回调函数的绑定记录表
type handlerMap  map[string]*handlerItem

// http回调函数注册信息
type handlerItem struct {
    ctype    reflect.Type // 控制器类型(反射类型)
    fname    string       // 回调方法名称
    faddr    HandlerFunc  // 准确的执行方法内存地址(与以上两个参数二选一)
}

// 路由注册项(这里使用了非并发安全的list.List，因为该对象的使用统一是由htmu互斥锁保证并发安全)
type handlerRegisterItem struct {
    handler    *handlerItem             // 准确的执行方法内存地址
    hooks      map[string]*list.List   // 当前的事件回调注册，键名为事件名称，键值为事件执行方法链表
    router     *Router                  // 注册时绑定的路由对象
}

// 根据特定URL.Path解析后的路由检索结果项
type handlerParsedItem struct {
    item    *handlerRegisterItem        // 路由注册项
    values   map[string][]string        // 特定URL.Path的Router解析参数
}

// HTTP注册函数
type HandlerFunc func(r *Request)

// 文件描述符map
type listenerFdMap map[string]string

// Server表，用以存储和检索名称与Server对象之间的关联关系
var serverMapping    = gmap.NewStringInterfaceMap()

// Web Socket默认配置
var wsUpgrader       = websocket.Upgrader{}

// Web Server已完成服务事件通道，当有事件时表示服务完成，当前进程退出
var doneChan         = make(chan struct{}, 1000)

// 用于服务进程初始化，只能初始化一次，采用“懒初始化”(在server运行时才初始化)
var serverProcInited = gtype.NewBool()

// Web Server进程初始化.
// 注意该方法不能放置于包初始化方法init中，不使用ghttp.Server的功能便不能初始化对应的协程goroutine逻辑.
func serverProcInit() {
    if serverProcInited.Val() {
        return
    }
    serverProcInited.Set(true)
    // 如果是完整重启，那么需要等待主进程销毁后，才开始执行监听，防止端口冲突
    if genv.Get(gADMIN_ACTION_RESTART_ENVKEY) != "" {
        if p, e := os.FindProcess(gproc.PPid()); e == nil {
            p.Kill()
            p.Wait()
        } else {
            glog.Error(e)
        }
    }

    // 信号量管理操作监听
    go handleProcessSignal()
    // 异步监听进程间消息
    go handleProcessMessage()
}

// 获取/创建一个默认配置的HTTP Server(默认监听端口是80)
// 单例模式，请保证name的唯一性
func GetServer(name...interface{}) (*Server) {
    sname := gDEFAULT_SERVER
    if len(name) > 0 {
        sname = gconv.String(name[0])
    }
    if s := serverMapping.Get(sname); s != nil {
        return s.(*Server)
    }
    s := &Server {
        name             : sname,
        paths            : gspath.New(),
        servers          : make([]*gracefulServer, 0),
        methodsMap       : make(map[string]bool),
        statusHandlerMap : make(map[string]HandlerFunc),
        handlerTree      : make(map[string]interface{}),
        handlerCache     : gcache.New(),
        hooksCache       : gcache.New(),
        cookies          : gmap.NewIntInterfaceMap(),
        sessions         : gcache.New(),
        cookieMaxAge     : gtype.NewInt(gDEFAULT_COOKIE_MAX_AGE),
        sessionMaxAge    : gtype.NewInt(gDEFAULT_SESSION_MAX_AGE),
        sessionIdName    : gtype.NewString(gDEFAULT_SESSION_ID_NAME),
        servedCount      : gtype.NewInt(),
        closeQueue       : gqueue.New(),
        logPath          : gtype.NewString(),
        accessLogEnabled : gtype.NewBool(),
        errorLogEnabled  : gtype.NewBool(true),
        accessLogger     : glog.New(),
        errorLogger      : glog.New(),
        logHandler       : gtype.NewInterface(),
    }
    s.errorLogger.SetBacktraceSkip(4)
    s.accessLogger.SetBacktraceSkip(4)
    // 设置路由解析缓存上限，使用LRU进行缓存淘汰
    s.hooksCache.SetCap(10000)
    s.handlerCache.SetCap(10000)
    for _, v := range strings.Split(gHTTP_METHODS, ",") {
        s.methodsMap[v] = true
    }
    s.SetConfig(defaultServerConfig)
    serverMapping.Set(sname, s)
    return s
}

// 作为守护协程异步执行(当同一进程中存在多个Web Server时，需要采用这种方式执行)
// 需要结合Wait方式一起使用
func (s *Server) Start() error {
    serverProcInit()

    // 如果设置了静态文件目录，那么严格按照静态文件目录进行检索
    // 否则，默认使用当前可执行文件目录，并且如果是开发环境，默认也会添加main包的源码目录路径做为二级检索
    if s.config.ServerRoot != "" {
        s.paths.Set(s.config.ServerRoot)
    } else {
        s.paths.Set(gfile.SelfDir())
        if p := gfile.MainPkgPath(); gfile.Exists(p) {
            s.paths.Add(p)
        }
    }

    if s.status == gSERVER_STATUS_RUNNING {
        return errors.New("server is already running")
    }
    // 底层http server配置
    if s.config.Handler == nil {
        s.config.Handler = http.HandlerFunc(s.defaultHttpHandle)
    }

    // 启动http server
    reloaded := false
    fdMapStr := genv.Get(gADMIN_ACTION_RELOAD_ENVKEY)
    if len(fdMapStr) > 0 {
        sfm := bufferToServerFdMap([]byte(fdMapStr))
        if v, ok := sfm[s.name]; ok {
            s.startServer(v)
            reloaded = true
        }
    }
    if !reloaded {
        s.startServer(nil)
    }

    // 如果是子进程，那么服务开启后通知父进程销毁
    if gproc.IsChild() {
        gtime.SetTimeout(2*time.Second, func() {
            gproc.Send(gproc.PPid(), []byte("exit"), gADMIN_GPROC_COMM_GROUP)
        })
    }

    // 开启异步关闭队列处理循环
    s.startCloseQueueLoop()
    return nil
}

// 阻塞执行监听
func (s *Server) Run() error {
    if err := s.Start(); err != nil {
        return err
    }
    // 阻塞等待服务执行完成
    <- doneChan

    glog.Printfln("%d: all servers shutdown", gproc.Pid())
    return nil
}

// 阻塞等待所有Web Server停止，常用于多Web Server场景，以及需要将Web Server异步运行的场景
// 这是一个与进程相关的方法
func Wait() {
    // 阻塞等待服务执行完成
    <- doneChan

    glog.Printfln("%d: all servers shutdown", gproc.Pid())
}

// 开启底层Web Server执行
func (s *Server) startServer(fdMap listenerFdMap) {
    var httpsEnabled bool
    if len(s.config.HTTPSCertPath) > 0 && len(s.config.HTTPSKeyPath) > 0 {
        // ================
        // HTTPS
        // ================
        if len(s.config.HTTPSAddr) == 0 {
            if len(s.config.Addr) > 0 {
                s.config.HTTPSAddr = s.config.Addr
                s.config.Addr      = ""
            } else {
                s.config.HTTPSAddr = gDEFAULT_HTTPS_ADDR
            }
        }
        httpsEnabled = len(s.config.HTTPSAddr) > 0
        var array []string
        if v, ok := fdMap["https"]; ok && len(v) > 0 {
            array = strings.Split(v, ",")
        } else {
            array = strings.Split(s.config.HTTPSAddr, ",")
        }
        for _, v := range array {
            if len(v) == 0 {
                continue
            }
            fd    := 0
            addr  := v
            array := strings.Split(v, "#")
            if len(array) > 1 {
                addr = array[0]
                // windows系统不支持文件描述符传递socket通信平滑交接，因此只能完整重启
                if runtime.GOOS != "windows" {
                    fd = gconv.Int(array[1])
                }
            }
            if fd > 0 {
                s.servers = append(s.servers, s.newGracefulServer(addr, fd))
            } else {
                s.servers = append(s.servers, s.newGracefulServer(addr))
            }
            s.servers[len(s.servers) - 1].isHttps = true
        }
    }
    // ================
    // HTTP
    // ================
    // 当HTTPS服务未启用时，默认HTTP地址才会生效
    if !httpsEnabled && len(s.config.Addr) == 0 {
        s.config.Addr = gDEFAULT_HTTP_ADDR
    }
    var array []string
    if v, ok := fdMap["http"]; ok && len(v) > 0 {
        array = strings.Split(v, ",")
    } else {
        array = strings.Split(s.config.Addr, ",")
    }
    for _, v := range array {
        if len(v) == 0 {
            continue
        }
        fd    := 0
        addr  := v
        array := strings.Split(v, "#")
        if len(array) > 1 {
            addr = array[0]
            // windows系统不支持文件描述符传递socket通信平滑交接，因此只能完整重启
            if runtime.GOOS != "windows" {
                fd = gconv.Int(array[1])
            }
        }
        if fd > 0 {
            s.servers = append(s.servers, s.newGracefulServer(addr, fd))
        } else {
            s.servers = append(s.servers, s.newGracefulServer(addr))
        }
    }
    // 开始执行异步监听
    for _, v := range s.servers {
        go func(server *gracefulServer) {
            var err error
            if server.isHttps {
                err = server.ListenAndServeTLS(s.config.HTTPSCertPath, s.config.HTTPSKeyPath)
            } else {
                err = server.ListenAndServe()
            }
            // 如果非关闭错误，那么提示报错，否则认为是正常的服务关闭操作
            if err != nil && !strings.EqualFold(http.ErrServerClosed.Error(), err.Error()) {
                glog.Error(err)
            }
        }(v)
    }

    s.status = gSERVER_STATUS_RUNNING
}

// 获取当前监听的文件描述符信息，构造成map返回
func (s *Server) getListenerFdMap() map[string]string {
    m := map[string]string {
        "https" : "",
        "http"  : "",
    }
    // s.servers是从HTTPS到HTTP优先级遍历，解析的时候也应当按照这个顺序读取fd
    for _, v := range s.servers {
        str := v.addr + "#" + gconv.String(v.Fd()) + ","
        if v.isHttps {
            m["https"] += str
        } else {
            m["http"]  += str
        }
    }
    // 去掉末尾的","号
    if len(m["https"]) > 0 {
        m["https"] = m["https"][0 : len(m["https"]) - 1]
    }
    if len(m["http"]) > 0 {
        m["http"] = m["http"][0 : len(m["http"]) - 1]
    }

    return m
}
<|endoftext|>