<commit_before>/*
 * Copyright 2017-2019 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package skywalking

import (
	"fmt"
	"github.com/gocolly/colly"
	"regexp"
	"resources/check"
	"resources/in"
	"resources/internal"
)

const root = "https://skywalking.apache.org/downloads"

var checkPattern = internal.Pattern{Regexp: regexp.MustCompile("([\\d]+)\\.([\\d]+)\\.([\\d]+)")}

type SkyWalking struct {
	Version internal.Version `json:"version"`
}

func (s SkyWalking) Check() (check.Result, error) {
	result := check.Result{Since: s.Version}

	c := colly.NewCollector()

	c.OnHTML("table tbody tr td:nth-child(2)", func(e *colly.HTMLElement) {

		_ = checkPattern.IfMatches(e.Text, func(g []string) error {
			result.Add(internal.Version{Ref: fmt.Sprintf("%s.%s.%s", g[1], g[2], g[3])})
			return nil
		})
	})

	err := c.Visit(root)
	return result, err
}

func (s SkyWalking) In(destination string) (in.Result, error) {
	name := s.name()
	uri := s.uri(name)

	sha256, err := in.Artifact{
		Name:        name,
		Version:     s.Version,
		URI:         uri,
		Destination: destination,
	}.Download()
	if err != nil {
		return in.Result{}, err
	}

	return in.Result{
		Version: s.Version,
		Metadata: []in.Metadata{
			{"uri", uri},
			{"sha256", sha256},
		},
	}, nil
}

func (s SkyWalking) name() string {
	return fmt.Sprintf("apache-skywalking-apm-%s.tar.gz", s.Version.Ref)
}

func (s SkyWalking) uri(name string) string {
	return fmt.Sprintf("https://mirrors.ocf.berkeley.edu/apache/skywalking/%s/%s", s.Version.Ref, name)
}
<commit_msg>Dynamic Skywalking URI<commit_after>/*
 * Copyright 2017-2019 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package skywalking

import (
	"fmt"
	"github.com/gocolly/colly"
	"regexp"
	"resources/check"
	"resources/in"
	"resources/internal"
	"strings"
)

const root = "https://skywalking.apache.org/downloads"

var checkPattern = internal.Pattern{Regexp: regexp.MustCompile("([\\d]+)\\.([\\d]+)\\.([\\d]+)")}

type SkyWalking struct {
	Version internal.Version `json:"version"`
}

func (s SkyWalking) Check() (check.Result, error) {
	result := check.Result{Since: s.Version}

	c := colly.NewCollector()

	c.OnHTML("table tbody tr td:nth-child(2)", func(e *colly.HTMLElement) {

		_ = checkPattern.IfMatches(e.Text, func(g []string) error {
			result.Add(internal.Version{Ref: fmt.Sprintf("%s.%s.%s", g[1], g[2], g[3])})
			return nil
		})
	})

	err := c.Visit(root)
	return result, err
}

func (s SkyWalking) In(destination string) (in.Result, error) {
	uri, err := s.uri()
	if err != nil {
		return in.Result{}, err
	}

	sha256, err := in.Artifact{
		Name:        s.name(),
		Version:     s.Version,
		URI:         uri,
		Destination: destination,
	}.Download()
	if err != nil {
		return in.Result{}, err
	}

	return in.Result{
		Version: s.Version,
		Metadata: []in.Metadata{
			{"uri", uri},
			{"sha256", sha256},
		},
	}, nil
}

func (s SkyWalking) name() string {
	return fmt.Sprintf("apache-skywalking-apm-%s.tar.gz", s.Version.Ref)
}

func (s SkyWalking) uri() (string, error) {
	c := colly.NewCollector()

	var u string
	c.OnHTML("div.container p a strong", func(e *colly.HTMLElement) {
		u = strings.TrimSpace(e.Text)
	})

	if err := c.Visit(fmt.Sprintf("https://www.apache.org/dyn/closer.cgi/skywalking/%[1]s/apache-skywalking-apm-%[1]s.tar.gz", s.Version.Ref)); err != nil {
		return "", err
	}

	return u, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/HewlettPackard/oneview-golang/utils"
	"os"
	"strconv"
	"strings"
)

func main() {
	var (
		ClientOV            *ov.OVClient
		logical_enclosure   = "TestLE"
		logical_enclosure_1 = "TestLE"
		logical_enclosure_2 = "log_enclosure88"
		scope_name          = "testing"
		li_name		    = "<logical_interconnect_name>"
	)
	apiversion, _ := strconv.Atoi(os.Getenv("ONEVIEW_APIVERSION"))

	ovc := ClientOV.NewOVClient(
		os.Getenv("ONEVIEW_OV_USER"),
		os.Getenv("ONEVIEW_OV_PASSWORD"),
		os.Getenv("ONEVIEW_OV_DOMAIN"),
		os.Getenv("ONEVIEW_OV_ENDPOINT"),
		false,
		apiversion,
		"*")

	fmt.Println("#................... Create Logical Enclosure ...............#")
	enclosureUris := new([]utils.Nstring)
	*enclosureUris = append(*enclosureUris, utils.NewNstring("/rest/enclosures/0000000000A66101"))
	*enclosureUris = append(*enclosureUris, utils.NewNstring("/rest/enclosures/0000000000A66102"))
	*enclosureUris = append(*enclosureUris, utils.NewNstring("/rest/enclosures/0000000000A66103"))

	enc_grp, err := ovc.GetEnclosureGroupByName("EG-Synergy-Local")

	logicalEnclosure := ov.LogicalEnclosure{Name: logical_enclosure_1,
		EnclosureUris:     *enclosureUris,
		EnclosureGroupUri: enc_grp.URI}

	er := ovc.CreateLogicalEnclosure(logicalEnclosure)
	if er != nil {
		fmt.Println("............... Logical Enclosure Creation Failed:", er)
	} else {
		fmt.Println(".... Logical Enclosure Created Success")
	}

	logicalInterconnect, _ := ovc.GetLogicalInterconnects("", "", "")
	li := ov.LogicalInterconnect{}
	for i := 0; i < len(logicalInterconnect.Members); i++{
		if logicalInterconnect.Members[i].Name == li_name{
			li = logicalInterconnect.Members[i]
		}
	}

	fmt.Println("#................... Create Logical Enclosure Support Dumps ...............#")

	supportdmp := ov.SupportDumps{ErrorCode: "MyDump16",
		ExcludeApplianceDump:    false,
                LogicalInterconnectUris: []utils.Nstring{ li.URI }}

	li_id := strings.Replace(string(li.URI), "/rest/logical-interconnects/", "", 1)

	data, er := ovc.CreateSupportDump(supportdmp, li_id)

	if er != nil {
		fmt.Println("............... Logical Enclosure Support Dump Creation Failed:", er)
	} else {
		fmt.Println(".... Logical Enclosure Support Dump Created Successfully", data)
		fmt.Println(data["URI"])
		id := strings.Trim(data["URI"], "/rest/tasks/")
		task, err := ovc.GetTasksById("", "", "", "", id)
		if err != nil {
			fmt.Println("Error getting the task details ", err)
		}
		fmt.Println(task)
	}

	log_enc, _ := ovc.GetLogicalEnclosureByName(logical_enclosure_1)

	fmt.Println("#................... Logical Enclosure by Name ...............#")
	log_en, err := ovc.GetLogicalEnclosureByName(logical_enclosure)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println(log_en)
	}

	// Update From Group

	err = ovc.UpdateFromGroupLogicalEnclosure(log_en)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("#............. Update From Group Logical Enclosure Successfully .....#")
	}

	scope1, err := ovc.GetScopeByName(scope_name)
	scope_uri := scope1.URI
	scope_Uris := new([]string)
	*scope_Uris = append(*scope_Uris, scope_uri.String())

	sort := "name:desc"
	log_en_list, err := ovc.GetLogicalEnclosures("", "", "", *scope_Uris, sort)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("# ................... Logical Enclosures List .................#")
		for i := 0; i < len(log_en_list.Members); i++ {
			fmt.Println(log_en_list.Members[i].Name)
		}
	}

	log_enc.Name = logical_enclosure_2
	err = ovc.UpdateLogicalEnclosure(log_enc)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("#.................... Logical Enclosure after Updating ...........#")
		log_en_after_update, err := ovc.GetLogicalEnclosures("", "", "", *scope_Uris, sort)
		if err != nil {
			fmt.Println(err)
		} else {
			for i := 0; i < len(log_en_after_update.Members); i++ {
				fmt.Println(log_en_after_update.Members[i].Name)
			}
		}
	}

	err = ovc.DeleteLogicalEnclosure(logical_enclosure_2)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("#...................... Deleted Logical Enclosure Successfully .....#")
	}

}
<commit_msg>fmt issues<commit_after>package main

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/HewlettPackard/oneview-golang/utils"
	"os"
	"strconv"
	"strings"
)

func main() {
	var (
		ClientOV            *ov.OVClient
		logical_enclosure   = "TestLE"
		logical_enclosure_1 = "TestLE"
		logical_enclosure_2 = "log_enclosure88"
		scope_name          = "testing"
		li_name             = "<logical_interconnect_name>"
	)
	apiversion, _ := strconv.Atoi(os.Getenv("ONEVIEW_APIVERSION"))

	ovc := ClientOV.NewOVClient(
		os.Getenv("ONEVIEW_OV_USER"),
		os.Getenv("ONEVIEW_OV_PASSWORD"),
		os.Getenv("ONEVIEW_OV_DOMAIN"),
		os.Getenv("ONEVIEW_OV_ENDPOINT"),
		false,
		apiversion,
		"*")

	fmt.Println("#................... Create Logical Enclosure ...............#")
	enclosureUris := new([]utils.Nstring)
	*enclosureUris = append(*enclosureUris, utils.NewNstring("/rest/enclosures/0000000000A66101"))
	*enclosureUris = append(*enclosureUris, utils.NewNstring("/rest/enclosures/0000000000A66102"))
	*enclosureUris = append(*enclosureUris, utils.NewNstring("/rest/enclosures/0000000000A66103"))

	enc_grp, err := ovc.GetEnclosureGroupByName("EG-Synergy-Local")

	logicalEnclosure := ov.LogicalEnclosure{Name: logical_enclosure_1,
		EnclosureUris:     *enclosureUris,
		EnclosureGroupUri: enc_grp.URI}

	er := ovc.CreateLogicalEnclosure(logicalEnclosure)
	if er != nil {
		fmt.Println("............... Logical Enclosure Creation Failed:", er)
	} else {
		fmt.Println(".... Logical Enclosure Created Success")
	}

	logicalInterconnect, _ := ovc.GetLogicalInterconnects("", "", "")
	li := ov.LogicalInterconnect{}
	for i := 0; i < len(logicalInterconnect.Members); i++ {
		if logicalInterconnect.Members[i].Name == li_name {
			li = logicalInterconnect.Members[i]
		}
	}

	fmt.Println("#................... Create Logical Enclosure Support Dumps ...............#")

	supportdmp := ov.SupportDumps{ErrorCode: "MyDump16",
		ExcludeApplianceDump:    false,
		LogicalInterconnectUris: []utils.Nstring{li.URI}}

	li_id := strings.Replace(string(li.URI), "/rest/logical-interconnects/", "", 1)

	data, er := ovc.CreateSupportDump(supportdmp, li_id)

	if er != nil {
		fmt.Println("............... Logical Enclosure Support Dump Creation Failed:", er)
	} else {
		fmt.Println(".... Logical Enclosure Support Dump Created Successfully", data)
		fmt.Println(data["URI"])
		id := strings.Trim(data["URI"], "/rest/tasks/")
		task, err := ovc.GetTasksById("", "", "", "", id)
		if err != nil {
			fmt.Println("Error getting the task details ", err)
		}
		fmt.Println(task)
	}

	log_enc, _ := ovc.GetLogicalEnclosureByName(logical_enclosure_1)

	fmt.Println("#................... Logical Enclosure by Name ...............#")
	log_en, err := ovc.GetLogicalEnclosureByName(logical_enclosure)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println(log_en)
	}

	// Update From Group

	err = ovc.UpdateFromGroupLogicalEnclosure(log_en)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("#............. Update From Group Logical Enclosure Successfully .....#")
	}

	scope1, err := ovc.GetScopeByName(scope_name)
	scope_uri := scope1.URI
	scope_Uris := new([]string)
	*scope_Uris = append(*scope_Uris, scope_uri.String())

	sort := "name:desc"
	log_en_list, err := ovc.GetLogicalEnclosures("", "", "", *scope_Uris, sort)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("# ................... Logical Enclosures List .................#")
		for i := 0; i < len(log_en_list.Members); i++ {
			fmt.Println(log_en_list.Members[i].Name)
		}
	}

	log_enc.Name = logical_enclosure_2
	err = ovc.UpdateLogicalEnclosure(log_enc)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("#.................... Logical Enclosure after Updating ...........#")
		log_en_after_update, err := ovc.GetLogicalEnclosures("", "", "", *scope_Uris, sort)
		if err != nil {
			fmt.Println(err)
		} else {
			for i := 0; i < len(log_en_after_update.Members); i++ {
				fmt.Println(log_en_after_update.Members[i].Name)
			}
		}
	}

	err = ovc.DeleteLogicalEnclosure(logical_enclosure_2)
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("#...................... Deleted Logical Enclosure Successfully .....#")
	}

}
<|endoftext|>


<commit_before>package static

import (
	"fmt"
	"net/http"
	"strings"
)

func (s *Static) ListenAndServe(addr string, eventHandler EventHandler) error {
	fileServer := http.FileServer(http.Dir(s.BuildDir))
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path

		filePath := path
		if strings.HasSuffix(path, "/") {
			filePath = fmt.Sprintf("%sindex.html", path)
		}

		err := s.handleRequest(w, filePath, true)

		if err == errNotFound {
			fileServer.ServeHTTP(w, r)
			eventHandler(Event{Action: "file", Path: path})
		} else {
			eventHandler(Event{Action: "build", Path: filePath, Error: err})
		}
	})
	return http.ListenAndServe(addr, mux)
}
<commit_msg>Minor refactor of static_serve.go functions.<commit_after>package static

import (
	"fmt"
	"net/http"
	"strings"
)

func (s *Static) ListenAndServe(addr string, ev EventHandler) error {
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		s.serverHandleRequest(w, r, ev)
	})
	return http.ListenAndServe(addr, mux)
}

func (s *Static) serverHandleRequest(w http.ResponseWriter, r *http.Request, ev EventHandler) {
	path := r.URL.Path

	filePath := path
	if strings.HasSuffix(path, "/") {
		filePath = fmt.Sprintf("%sindex.html", path)
	}

	err := s.handleRequest(w, filePath, true)

	if err == errNotFound {
		fileServer := http.FileServer(http.Dir(s.BuildDir))
		fileServer.ServeHTTP(w, r)
		ev(Event{Action: "file", Path: path})
	} else {
		ev(Event{Action: "build", Path: filePath, Error: err})
	}
}
<|endoftext|>


<commit_before>package stats

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"
)

type Server struct {
	base

	history map[string][]*baseSnapshot
}

const (
	serverSnapshotIntervl = 5 * time.Second
	serverHistorySize     = 360 // 30 minutes of 5 second snapshots
)

func NewServer() *Server {
	s := &Server{}
	s.init()
	s.history = make(map[string][]*baseSnapshot)
	go s.takeSnapshots()

	return s
}

func (s *Server) History(rw http.ResponseWriter, _ *http.Request) {
	encoded, err := json.Marshal(s.history)
	if err != nil {
		http.Error(rw, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}

	rw.Header().Add("Access-Control-Allow-Origin", "*")
	rw.Write(encoded)
}

func (s *Server) takeSnapshots() {
	for range time.NewTicker(serverSnapshotIntervl).C {
		s.Lock()
		for name, stat := range s.stats {
			if len(s.history[name]) >= serverHistorySize {
				s.history[name] = s.history[name][1:]
			}
			s.history[name] = append(s.history[name], stat.snapshot())
		}
		s.Unlock()
	}
}
<commit_msg>Reset stats after taking a snapshot<commit_after>package stats

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"
)

type Server struct {
	base

	history map[string][]*baseSnapshot
}

const (
	serverSnapshotIntervl = 5 * time.Second
	serverHistorySize     = 360 // 30 minutes of 5 second snapshots
)

func NewServer() *Server {
	s := &Server{}
	s.init()
	s.history = make(map[string][]*baseSnapshot)
	go s.takeSnapshots()

	return s
}

func (s *Server) History(rw http.ResponseWriter, _ *http.Request) {
	encoded, err := json.Marshal(s.history)
	if err != nil {
		http.Error(rw, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}

	rw.Header().Add("Access-Control-Allow-Origin", "*")
	rw.Write(encoded)
}

func (s *Server) takeSnapshots() {
	for range time.NewTicker(serverSnapshotIntervl).C {
		s.Lock()
		for name, stat := range s.stats {
			if len(s.history[name]) >= serverHistorySize {
				s.history[name] = s.history[name][1:]
			}
			s.history[name] = append(s.history[name], stat.snapshot())
		}
		s.Reset()
		s.Unlock()
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/go/glog"
	"github.com/chrislusf/seaweedfs/go/operation"
	"github.com/chrislusf/seaweedfs/go/security"
	"github.com/chrislusf/seaweedfs/go/util"
	"github.com/golang/protobuf/proto"
)

const (
	MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes
)

type DiskLocation struct {
	Directory      string
	MaxVolumeCount int
	volumes        map[VolumeId]*Volume
}

func (mn *DiskLocation) reset() {
}

type MasterNodes struct {
	nodes    []string
	lastNode int
}

func (mn *MasterNodes) String() string {
	return fmt.Sprintf("nodes:%v, lastNode:%d", mn.nodes, mn.lastNode)
}

func NewMasterNodes(bootstrapNode string) (mn *MasterNodes) {
	mn = &MasterNodes{nodes: []string{bootstrapNode}, lastNode: -1}
	return
}
func (mn *MasterNodes) reset() {
	glog.V(4).Infof("Resetting master nodes: %v", mn)
	if len(mn.nodes) > 1 && mn.lastNode > 0 {
		glog.V(0).Infof("Reset master %s from: %v", mn.nodes[mn.lastNode], mn.nodes)
		mn.lastNode = -mn.lastNode
	}
}
func (mn *MasterNodes) findMaster() (string, error) {
	if len(mn.nodes) == 0 {
		return "", errors.New("No master node found!")
	}
	if mn.lastNode < 0 {
		for _, m := range mn.nodes {
			glog.V(4).Infof("Listing masters on %s", m)
			if masters, e := operation.ListMasters(m); e == nil {
				if len(masters) == 0 {
					continue
				}
				mn.nodes = append(masters, m)
				mn.lastNode = rand.Intn(len(mn.nodes))
				glog.V(2).Infof("current master nodes is %v", mn)
				break
			} else {
				glog.V(4).Infof("Failed listing masters on %s: %v", m, e)
			}
		}
	}
	if mn.lastNode < 0 {
		return "", errors.New("No master node available!")
	}
	return mn.nodes[mn.lastNode], nil
}

/*
 * A VolumeServer contains one Store
 */
type Store struct {
	Ip              string
	Port            int
	PublicUrl       string
	Locations       []*DiskLocation
	dataCenter      string //optional informaton, overwriting master setting if exists
	rack            string //optional information, overwriting master setting if exists
	connected       bool
	volumeSizeLimit uint64 //read from the master
	masterNodes     *MasterNodes
}

func (s *Store) String() (str string) {
	str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d, masterNodes:%s", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.volumeSizeLimit, s.masterNodes)
	return
}

func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, needleMapKind NeedleMapType) (s *Store) {
	s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl}
	s.Locations = make([]*DiskLocation, 0)
	for i := 0; i < len(dirnames); i++ {
		location := &DiskLocation{Directory: dirnames[i], MaxVolumeCount: maxVolumeCounts[i]}
		location.volumes = make(map[VolumeId]*Volume)
		location.loadExistingVolumes(needleMapKind)
		s.Locations = append(s.Locations, location)
	}
	return
}
func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string) error {
	rt, e := NewReplicaPlacementFromString(replicaPlacement)
	if e != nil {
		return e
	}
	ttl, e := ReadTTL(ttlString)
	if e != nil {
		return e
	}
	for _, range_string := range strings.Split(volumeListString, ",") {
		if strings.Index(range_string, "-") < 0 {
			id_string := range_string
			id, err := NewVolumeId(id_string)
			if err != nil {
				return fmt.Errorf("Volume Id %s is not a valid unsigned integer!", id_string)
			}
			e = s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl)
		} else {
			pair := strings.Split(range_string, "-")
			start, start_err := strconv.ParseUint(pair[0], 10, 64)
			if start_err != nil {
				return fmt.Errorf("Volume Start Id %s is not a valid unsigned integer!", pair[0])
			}
			end, end_err := strconv.ParseUint(pair[1], 10, 64)
			if end_err != nil {
				return fmt.Errorf("Volume End Id %s is not a valid unsigned integer!", pair[1])
			}
			for id := start; id <= end; id++ {
				if err := s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl); err != nil {
					e = err
				}
			}
		}
	}
	return e
}
func (s *Store) DeleteCollection(collection string) (e error) {
	for _, location := range s.Locations {
		for k, v := range location.volumes {
			if v.Collection == collection {
				e = v.Destroy()
				if e != nil {
					return
				}
				delete(location.volumes, k)
			}
		}
	}
	return
}
func (s *Store) DeleteVolume(volumes map[VolumeId]*Volume, v *Volume) (e error) {
	e = v.Destroy()
	if e != nil {
		return
	}
	delete(volumes, v.Id)
	return
}
func (s *Store) findVolume(vid VolumeId) *Volume {
	for _, location := range s.Locations {
		if v, found := location.volumes[vid]; found {
			return v
		}
	}
	return nil
}
func (s *Store) findFreeLocation() (ret *DiskLocation) {
	max := 0
	for _, location := range s.Locations {
		currentFreeCount := location.MaxVolumeCount - len(location.volumes)
		if currentFreeCount > max {
			max = currentFreeCount
			ret = location
		}
	}
	return ret
}
func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) error {
	if s.findVolume(vid) != nil {
		return fmt.Errorf("Volume Id %d already exists!", vid)
	}
	if location := s.findFreeLocation(); location != nil {
		glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
			location.Directory, vid, collection, replicaPlacement, ttl)
		if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl); err == nil {
			location.volumes[vid] = volume
			return nil
		} else {
			return err
		}
	}
	return fmt.Errorf("No more free space left")
}

func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
	if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
		for _, dir := range dirs {
			name := dir.Name()
			if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
				collection := ""
				base := name[:len(name)-len(".dat")]
				i := strings.Index(base, "_")
				if i > 0 {
					collection, base = base[0:i], base[i+1:]
				}
				if vid, err := NewVolumeId(base); err == nil {
					if l.volumes[vid] == nil {
						if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil); e == nil {
							l.volumes[vid] = v
							glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s", l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), v.Size(), v.Ttl.String())
						}
					}
				}
			}
		}
	}
	glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
}
func (s *Store) Status() []*VolumeInfo {
	var stats []*VolumeInfo
	for _, location := range s.Locations {
		for k, v := range location.volumes {
			s := &VolumeInfo{
				Id:               VolumeId(k),
				Size:             v.ContentSize(),
				Collection:       v.Collection,
				ReplicaPlacement: v.ReplicaPlacement,
				Version:          v.Version(),
				FileCount:        v.nm.FileCount(),
				DeleteCount:      v.nm.DeletedCount(),
				DeletedByteCount: v.nm.DeletedSize(),
				ReadOnly:         v.readOnly,
				Ttl:              v.Ttl}
			stats = append(stats, s)
		}
	}
	sortVolumeInfos(stats)
	return stats
}

func (s *Store) SetDataCenter(dataCenter string) {
	s.dataCenter = dataCenter
}
func (s *Store) SetRack(rack string) {
	s.rack = rack
}

func (s *Store) SetBootstrapMaster(bootstrapMaster string) {
	s.masterNodes = NewMasterNodes(bootstrapMaster)
}
func (s *Store) SendHeartbeatToMaster() (masterNode string, secretKey security.Secret, e error) {
	masterNode, e = s.masterNodes.findMaster()
	if e != nil {
		return
	}
	var volumeMessages []*operation.VolumeInformationMessage
	maxVolumeCount := 0
	var maxFileKey uint64
	for _, location := range s.Locations {
		maxVolumeCount = maxVolumeCount + location.MaxVolumeCount
		for k, v := range location.volumes {
			if maxFileKey < v.nm.MaxFileKey() {
				maxFileKey = v.nm.MaxFileKey()
			}
			if !v.expired(s.volumeSizeLimit) {
				volumeMessage := &operation.VolumeInformationMessage{
					Id:               proto.Uint32(uint32(k)),
					Size:             proto.Uint64(uint64(v.Size())),
					Collection:       proto.String(v.Collection),
					FileCount:        proto.Uint64(uint64(v.nm.FileCount())),
					DeleteCount:      proto.Uint64(uint64(v.nm.DeletedCount())),
					DeletedByteCount: proto.Uint64(v.nm.DeletedSize()),
					ReadOnly:         proto.Bool(v.readOnly),
					ReplicaPlacement: proto.Uint32(uint32(v.ReplicaPlacement.Byte())),
					Version:          proto.Uint32(uint32(v.Version())),
					Ttl:              proto.Uint32(v.Ttl.ToUint32()),
				}
				volumeMessages = append(volumeMessages, volumeMessage)
			} else {
				if v.exiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) {
					s.DeleteVolume(location.volumes, v)
					glog.V(0).Infoln("volume", v.Id, "is deleted.")
				} else {
					glog.V(0).Infoln("volume", v.Id, "is expired.")
				}
			}
		}
	}

	joinMessage := &operation.JoinMessage{
		IsInit:         proto.Bool(!s.connected),
		Ip:             proto.String(s.Ip),
		Port:           proto.Uint32(uint32(s.Port)),
		PublicUrl:      proto.String(s.PublicUrl),
		MaxVolumeCount: proto.Uint32(uint32(maxVolumeCount)),
		MaxFileKey:     proto.Uint64(maxFileKey),
		DataCenter:     proto.String(s.dataCenter),
		Rack:           proto.String(s.rack),
		Volumes:        volumeMessages,
	}

	data, err := proto.Marshal(joinMessage)
	if err != nil {
		return "", "", err
	}

	joinUrl := "http://" + masterNode + "/dir/join"
	glog.V(4).Infof("Connecting to %s ...", joinUrl)

	jsonBlob, err := util.PostBytes(joinUrl, data)
	if err != nil {
		s.masterNodes.reset()
		return "", "", err
	}
	var ret operation.JoinResult
	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
		glog.V(0).Infof("Failed to join %s with response: %s", joinUrl, string(jsonBlob))
		s.masterNodes.reset()
		return masterNode, "", err
	}
	if ret.Error != "" {
		s.masterNodes.reset()
		return masterNode, "", errors.New(ret.Error)
	}
	s.volumeSizeLimit = ret.VolumeSizeLimit
	secretKey = security.Secret(ret.SecretKey)
	s.connected = true
	return
}
func (s *Store) Close() {
	for _, location := range s.Locations {
		for _, v := range location.volumes {
			v.Close()
		}
	}
}
func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
	if v := s.findVolume(i); v != nil {
		if v.readOnly {
			err = fmt.Errorf("Volume %d is read only", i)
			return
		}
		if MaxPossibleVolumeSize >= v.ContentSize()+uint64(size) {
			size, err = v.write(n)
		} else {
			err = fmt.Errorf("Volume Size Limit %d Exceeded! Current size is %d", s.volumeSizeLimit, v.ContentSize())
		}
		if s.volumeSizeLimit < v.ContentSize()+3*uint64(size) {
			glog.V(0).Infoln("volume", i, "size", v.ContentSize(), "will exceed limit", s.volumeSizeLimit)
			if _, _, e := s.SendHeartbeatToMaster(); e != nil {
				glog.V(0).Infoln("error when reporting size:", e)
			}
		}
		return
	}
	glog.V(0).Infoln("volume", i, "not found!")
	err = fmt.Errorf("Volume %d not found!", i)
	return
}
func (s *Store) Delete(i VolumeId, n *Needle) (uint32, error) {
	if v := s.findVolume(i); v != nil && !v.readOnly {
		return v.delete(n)
	}
	return 0, nil
}
func (s *Store) ReadVolumeNeedle(i VolumeId, n *Needle) (int, error) {
	if v := s.findVolume(i); v != nil {
		return v.readNeedle(n)
	}
	return 0, fmt.Errorf("Volume %v not found!", i)
}
func (s *Store) GetVolume(i VolumeId) *Volume {
	return s.findVolume(i)
}

func (s *Store) HasVolume(i VolumeId) bool {
	v := s.findVolume(i)
	return v != nil
}
<commit_msg>In case lastNode==0, need to set it to -1<commit_after>package storage

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"strconv"
	"strings"

	"github.com/chrislusf/seaweedfs/go/glog"
	"github.com/chrislusf/seaweedfs/go/operation"
	"github.com/chrislusf/seaweedfs/go/security"
	"github.com/chrislusf/seaweedfs/go/util"
	"github.com/golang/protobuf/proto"
)

const (
	MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes
)

type DiskLocation struct {
	Directory      string
	MaxVolumeCount int
	volumes        map[VolumeId]*Volume
}

func (mn *DiskLocation) reset() {
}

type MasterNodes struct {
	nodes    []string
	lastNode int
}

func (mn *MasterNodes) String() string {
	return fmt.Sprintf("nodes:%v, lastNode:%d", mn.nodes, mn.lastNode)
}

func NewMasterNodes(bootstrapNode string) (mn *MasterNodes) {
	mn = &MasterNodes{nodes: []string{bootstrapNode}, lastNode: -1}
	return
}
func (mn *MasterNodes) reset() {
	glog.V(4).Infof("Resetting master nodes: %v", mn)
	if len(mn.nodes) > 1 && mn.lastNode >= 0 {
		glog.V(0).Infof("Reset master %s from: %v", mn.nodes[mn.lastNode], mn.nodes)
		mn.lastNode = -mn.lastNode - 1
	}
}
func (mn *MasterNodes) findMaster() (string, error) {
	if len(mn.nodes) == 0 {
		return "", errors.New("No master node found!")
	}
	if mn.lastNode < 0 {
		for _, m := range mn.nodes {
			glog.V(4).Infof("Listing masters on %s", m)
			if masters, e := operation.ListMasters(m); e == nil {
				if len(masters) == 0 {
					continue
				}
				mn.nodes = append(masters, m)
				mn.lastNode = rand.Intn(len(mn.nodes))
				glog.V(2).Infof("current master nodes is %v", mn)
				break
			} else {
				glog.V(4).Infof("Failed listing masters on %s: %v", m, e)
			}
		}
	}
	if mn.lastNode < 0 {
		return "", errors.New("No master node available!")
	}
	return mn.nodes[mn.lastNode], nil
}

/*
 * A VolumeServer contains one Store
 */
type Store struct {
	Ip              string
	Port            int
	PublicUrl       string
	Locations       []*DiskLocation
	dataCenter      string //optional informaton, overwriting master setting if exists
	rack            string //optional information, overwriting master setting if exists
	connected       bool
	volumeSizeLimit uint64 //read from the master
	masterNodes     *MasterNodes
}

func (s *Store) String() (str string) {
	str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d, masterNodes:%s", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.volumeSizeLimit, s.masterNodes)
	return
}

func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, needleMapKind NeedleMapType) (s *Store) {
	s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl}
	s.Locations = make([]*DiskLocation, 0)
	for i := 0; i < len(dirnames); i++ {
		location := &DiskLocation{Directory: dirnames[i], MaxVolumeCount: maxVolumeCounts[i]}
		location.volumes = make(map[VolumeId]*Volume)
		location.loadExistingVolumes(needleMapKind)
		s.Locations = append(s.Locations, location)
	}
	return
}
func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string) error {
	rt, e := NewReplicaPlacementFromString(replicaPlacement)
	if e != nil {
		return e
	}
	ttl, e := ReadTTL(ttlString)
	if e != nil {
		return e
	}
	for _, range_string := range strings.Split(volumeListString, ",") {
		if strings.Index(range_string, "-") < 0 {
			id_string := range_string
			id, err := NewVolumeId(id_string)
			if err != nil {
				return fmt.Errorf("Volume Id %s is not a valid unsigned integer!", id_string)
			}
			e = s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl)
		} else {
			pair := strings.Split(range_string, "-")
			start, start_err := strconv.ParseUint(pair[0], 10, 64)
			if start_err != nil {
				return fmt.Errorf("Volume Start Id %s is not a valid unsigned integer!", pair[0])
			}
			end, end_err := strconv.ParseUint(pair[1], 10, 64)
			if end_err != nil {
				return fmt.Errorf("Volume End Id %s is not a valid unsigned integer!", pair[1])
			}
			for id := start; id <= end; id++ {
				if err := s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl); err != nil {
					e = err
				}
			}
		}
	}
	return e
}
func (s *Store) DeleteCollection(collection string) (e error) {
	for _, location := range s.Locations {
		for k, v := range location.volumes {
			if v.Collection == collection {
				e = v.Destroy()
				if e != nil {
					return
				}
				delete(location.volumes, k)
			}
		}
	}
	return
}
func (s *Store) DeleteVolume(volumes map[VolumeId]*Volume, v *Volume) (e error) {
	e = v.Destroy()
	if e != nil {
		return
	}
	delete(volumes, v.Id)
	return
}
func (s *Store) findVolume(vid VolumeId) *Volume {
	for _, location := range s.Locations {
		if v, found := location.volumes[vid]; found {
			return v
		}
	}
	return nil
}
func (s *Store) findFreeLocation() (ret *DiskLocation) {
	max := 0
	for _, location := range s.Locations {
		currentFreeCount := location.MaxVolumeCount - len(location.volumes)
		if currentFreeCount > max {
			max = currentFreeCount
			ret = location
		}
	}
	return ret
}
func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) error {
	if s.findVolume(vid) != nil {
		return fmt.Errorf("Volume Id %d already exists!", vid)
	}
	if location := s.findFreeLocation(); location != nil {
		glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
			location.Directory, vid, collection, replicaPlacement, ttl)
		if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl); err == nil {
			location.volumes[vid] = volume
			return nil
		} else {
			return err
		}
	}
	return fmt.Errorf("No more free space left")
}

func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
	if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
		for _, dir := range dirs {
			name := dir.Name()
			if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
				collection := ""
				base := name[:len(name)-len(".dat")]
				i := strings.Index(base, "_")
				if i > 0 {
					collection, base = base[0:i], base[i+1:]
				}
				if vid, err := NewVolumeId(base); err == nil {
					if l.volumes[vid] == nil {
						if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil); e == nil {
							l.volumes[vid] = v
							glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s", l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), v.Size(), v.Ttl.String())
						}
					}
				}
			}
		}
	}
	glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
}
func (s *Store) Status() []*VolumeInfo {
	var stats []*VolumeInfo
	for _, location := range s.Locations {
		for k, v := range location.volumes {
			s := &VolumeInfo{
				Id:               VolumeId(k),
				Size:             v.ContentSize(),
				Collection:       v.Collection,
				ReplicaPlacement: v.ReplicaPlacement,
				Version:          v.Version(),
				FileCount:        v.nm.FileCount(),
				DeleteCount:      v.nm.DeletedCount(),
				DeletedByteCount: v.nm.DeletedSize(),
				ReadOnly:         v.readOnly,
				Ttl:              v.Ttl}
			stats = append(stats, s)
		}
	}
	sortVolumeInfos(stats)
	return stats
}

func (s *Store) SetDataCenter(dataCenter string) {
	s.dataCenter = dataCenter
}
func (s *Store) SetRack(rack string) {
	s.rack = rack
}

func (s *Store) SetBootstrapMaster(bootstrapMaster string) {
	s.masterNodes = NewMasterNodes(bootstrapMaster)
}
func (s *Store) SendHeartbeatToMaster() (masterNode string, secretKey security.Secret, e error) {
	masterNode, e = s.masterNodes.findMaster()
	if e != nil {
		return
	}
	var volumeMessages []*operation.VolumeInformationMessage
	maxVolumeCount := 0
	var maxFileKey uint64
	for _, location := range s.Locations {
		maxVolumeCount = maxVolumeCount + location.MaxVolumeCount
		for k, v := range location.volumes {
			if maxFileKey < v.nm.MaxFileKey() {
				maxFileKey = v.nm.MaxFileKey()
			}
			if !v.expired(s.volumeSizeLimit) {
				volumeMessage := &operation.VolumeInformationMessage{
					Id:               proto.Uint32(uint32(k)),
					Size:             proto.Uint64(uint64(v.Size())),
					Collection:       proto.String(v.Collection),
					FileCount:        proto.Uint64(uint64(v.nm.FileCount())),
					DeleteCount:      proto.Uint64(uint64(v.nm.DeletedCount())),
					DeletedByteCount: proto.Uint64(v.nm.DeletedSize()),
					ReadOnly:         proto.Bool(v.readOnly),
					ReplicaPlacement: proto.Uint32(uint32(v.ReplicaPlacement.Byte())),
					Version:          proto.Uint32(uint32(v.Version())),
					Ttl:              proto.Uint32(v.Ttl.ToUint32()),
				}
				volumeMessages = append(volumeMessages, volumeMessage)
			} else {
				if v.exiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) {
					s.DeleteVolume(location.volumes, v)
					glog.V(0).Infoln("volume", v.Id, "is deleted.")
				} else {
					glog.V(0).Infoln("volume", v.Id, "is expired.")
				}
			}
		}
	}

	joinMessage := &operation.JoinMessage{
		IsInit:         proto.Bool(!s.connected),
		Ip:             proto.String(s.Ip),
		Port:           proto.Uint32(uint32(s.Port)),
		PublicUrl:      proto.String(s.PublicUrl),
		MaxVolumeCount: proto.Uint32(uint32(maxVolumeCount)),
		MaxFileKey:     proto.Uint64(maxFileKey),
		DataCenter:     proto.String(s.dataCenter),
		Rack:           proto.String(s.rack),
		Volumes:        volumeMessages,
	}

	data, err := proto.Marshal(joinMessage)
	if err != nil {
		return "", "", err
	}

	joinUrl := "http://" + masterNode + "/dir/join"
	glog.V(4).Infof("Connecting to %s ...", joinUrl)

	jsonBlob, err := util.PostBytes(joinUrl, data)
	if err != nil {
		s.masterNodes.reset()
		return "", "", err
	}
	var ret operation.JoinResult
	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
		glog.V(0).Infof("Failed to join %s with response: %s", joinUrl, string(jsonBlob))
		s.masterNodes.reset()
		return masterNode, "", err
	}
	if ret.Error != "" {
		s.masterNodes.reset()
		return masterNode, "", errors.New(ret.Error)
	}
	s.volumeSizeLimit = ret.VolumeSizeLimit
	secretKey = security.Secret(ret.SecretKey)
	s.connected = true
	return
}
func (s *Store) Close() {
	for _, location := range s.Locations {
		for _, v := range location.volumes {
			v.Close()
		}
	}
}
func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
	if v := s.findVolume(i); v != nil {
		if v.readOnly {
			err = fmt.Errorf("Volume %d is read only", i)
			return
		}
		if MaxPossibleVolumeSize >= v.ContentSize()+uint64(size) {
			size, err = v.write(n)
		} else {
			err = fmt.Errorf("Volume Size Limit %d Exceeded! Current size is %d", s.volumeSizeLimit, v.ContentSize())
		}
		if s.volumeSizeLimit < v.ContentSize()+3*uint64(size) {
			glog.V(0).Infoln("volume", i, "size", v.ContentSize(), "will exceed limit", s.volumeSizeLimit)
			if _, _, e := s.SendHeartbeatToMaster(); e != nil {
				glog.V(0).Infoln("error when reporting size:", e)
			}
		}
		return
	}
	glog.V(0).Infoln("volume", i, "not found!")
	err = fmt.Errorf("Volume %d not found!", i)
	return
}
func (s *Store) Delete(i VolumeId, n *Needle) (uint32, error) {
	if v := s.findVolume(i); v != nil && !v.readOnly {
		return v.delete(n)
	}
	return 0, nil
}
func (s *Store) ReadVolumeNeedle(i VolumeId, n *Needle) (int, error) {
	if v := s.findVolume(i); v != nil {
		return v.readNeedle(n)
	}
	return 0, fmt.Errorf("Volume %v not found!", i)
}
func (s *Store) GetVolume(i VolumeId) *Volume {
	return s.findVolume(i)
}

func (s *Store) HasVolume(i VolumeId) bool {
	v := s.findVolume(i)
	return v != nil
}
<|endoftext|>


<commit_before>package teams

import (
	"fmt"
	"log"
	"sync"

	lru "github.com/hashicorp/golang-lru"
	context "golang.org/x/net/context"

	"github.com/keybase/client/go/encrypteddb"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

// Store TeamData's on memory and disk. Threadsafe.
type Storage struct {
	libkb.Contextified
	sync.Mutex
	mem  *MemoryStorage
	disk *DiskStorage
}

func NewStorage(g *libkb.GlobalContext) *Storage {
	return &Storage{
		Contextified: libkb.NewContextified(g),
		mem:          NewMemoryStorage(g),
		disk:         NewDiskStorage(g),
	}
}

func (s *Storage) Put(ctx context.Context, state *keybase1.TeamData) {
	s.Lock()
	defer s.Unlock()

	s.mem.Put(ctx, state)

	err := s.disk.Put(ctx, state)
	if err != nil {
		s.G().Log.CWarningf(ctx, "teams.Storage.Put err: %v", err)
	}
}

// Can return nil.
func (s *Storage) Get(ctx context.Context, teamID keybase1.TeamID, public bool) *keybase1.TeamData {
	s.Lock()
	defer s.Unlock()

	item := s.mem.Get(ctx, teamID, public)
	if item != nil {
		// Mem hit
		return item
	}

	res, found, err := s.disk.Get(ctx, teamID, public)
	if found && err == nil {
		// Disk hit
		s.mem.Put(ctx, res)
		return res
	}
	if err != nil {
		s.G().Log.Debug("teams.Storage#Get disk err: %v", err)
	}

	return nil
}

func (s *Storage) Delete(ctx context.Context, teamID keybase1.TeamID, public bool) error {
	s.Lock()
	defer s.Unlock()

	s.mem.Delete(ctx, teamID, public)
	return s.disk.Delete(ctx, teamID, public)
}

func (s *Storage) onLogout() {
	s.mem.onLogout()
}

// --------------------------------------------------

// Store TeamData's on disk. Threadsafe.
type DiskStorage struct {
	libkb.Contextified
	sync.Mutex
	encryptedDB *encrypteddb.EncryptedDB
}

// Increment to invalidate the disk cache.
const diskStorageVersion = 5

type DiskStorageItem struct {
	Version int                `codec:"V"`
	State   *keybase1.TeamData `codec:"S"`
}

func NewDiskStorage(g *libkb.GlobalContext) *DiskStorage {
	keyFn := func(ctx context.Context) ([32]byte, error) {
		return getLocalStorageSecretBoxKey(ctx, g)
	}
	dbFn := func(g *libkb.GlobalContext) *libkb.JSONLocalDb {
		return g.LocalDb
	}
	return &DiskStorage{
		Contextified: libkb.NewContextified(g),
		encryptedDB:  encrypteddb.New(g, dbFn, keyFn),
	}
}

func (s *DiskStorage) Put(ctx context.Context, state *keybase1.TeamData) error {
	s.Lock()
	defer s.Unlock()

	key := s.dbKey(ctx, state.Chain.Id, state.Chain.Public)
	item := DiskStorageItem{
		Version: diskStorageVersion,
		State:   state,
	}

	err := s.encryptedDB.Put(ctx, key, item)
	if err != nil {
		return err
	}
	return nil
}

// Res is valid if (found && err == nil)
func (s *DiskStorage) Get(ctx context.Context, teamID keybase1.TeamID, public bool) (res *keybase1.TeamData, found bool, err error) {
	s.Lock()
	defer s.Unlock()

	key := s.dbKey(ctx, teamID, public)
	var item DiskStorageItem
	found, err = s.encryptedDB.Get(ctx, key, &item)
	if (err != nil) || !found {
		return res, found, err
	}

	if item.Version != diskStorageVersion {
		// Pretend it wasn't found.
		return res, false, nil
	}

	// Sanity check
	if len(item.State.Chain.Id) == 0 {
		return res, false, fmt.Errorf("decode from disk had empty team id")
	}
	if !item.State.Chain.Id.Eq(teamID) {
		return res, false, fmt.Errorf("decode from disk had wrong team id %v != %v", item.State.Chain.Id, teamID)
	}
	if item.State.Chain.Public != public {
		return res, false, fmt.Errorf("decode from disk had wrong publicness %v != %v (%v)", item.State.Chain.Public, public, teamID)
	}

	return item.State, true, nil
}

func (s *DiskStorage) Delete(ctx context.Context, teamID keybase1.TeamID, public bool) error {
	s.Lock()
	defer s.Unlock()

	key := s.dbKey(ctx, teamID, public)
	return s.encryptedDB.Delete(ctx, key)
}

func (s *DiskStorage) dbKey(ctx context.Context, teamID keybase1.TeamID, public bool) libkb.DbKey {
	key := fmt.Sprintf("tid:%s", teamID)
	if public {
		key = fmt.Sprintf("tid:%s|pub", teamID)
	}
	return libkb.DbKey{
		Typ: libkb.DBChatInbox,
		Key: key,
	}
}

// --------------------------------------------------

const MemCacheLRUSize = 200

// Store some TeamSigChainState's in memory. Threadsafe.
type MemoryStorage struct {
	libkb.Contextified
	lru *lru.Cache
}

func NewMemoryStorage(g *libkb.GlobalContext) *MemoryStorage {
	nlru, err := lru.New(MemCacheLRUSize)
	if err != nil {
		// lru.New only panics if size <= 0
		log.Panicf("Could not create lru cache: %v", err)
	}
	return &MemoryStorage{
		Contextified: libkb.NewContextified(g),
		lru:          nlru,
	}
}

func (s *MemoryStorage) Put(ctx context.Context, state *keybase1.TeamData) {
	s.lru.Add(s.key(state.Chain.Id, state.Chain.Public), state)
}

// Can return nil.
func (s *MemoryStorage) Get(ctx context.Context, teamID keybase1.TeamID, public bool) *keybase1.TeamData {
	untyped, ok := s.lru.Get(s.key(teamID, public))
	if !ok {
		return nil
	}
	state, ok := untyped.(*keybase1.TeamData)
	if !ok {
		s.G().Log.Warning("Team MemoryStorage got bad type from lru: %T", untyped)
		return nil
	}
	return state
}

func (s *MemoryStorage) Delete(ctx context.Context, teamID keybase1.TeamID, public bool) {
	s.lru.Remove(s.key(teamID, public))
}

func (s *MemoryStorage) onLogout() {
	s.lru.Purge()
}

func (s *MemoryStorage) key(teamID keybase1.TeamID, public bool) (key string) {
	key = fmt.Sprintf("tid:%s", teamID)
	if public {
		key = fmt.Sprintf("tid:%s|pub", teamID)
	}
	return key
}

// --------------------------------------------------

func getLocalStorageSecretBoxKey(ctx context.Context, g *libkb.GlobalContext) (fkey [32]byte, err error) {
	// Get secret device key
	encKey, err := engine.GetMySecretKey(ctx, g, getLameSecretUI, libkb.DeviceEncryptionKeyType,
		"encrypt teams storage")
	if err != nil {
		return fkey, err
	}
	kp, ok := encKey.(libkb.NaclDHKeyPair)
	if !ok || kp.Private == nil {
		return fkey, libkb.KeyCannotDecryptError{}
	}

	// Derive symmetric key from device key
	skey, err := encKey.SecretSymmetricKey(libkb.EncryptionReasonTeamsLocalStorage)
	if err != nil {
		return fkey, err
	}

	copy(fkey[:], skey[:])
	return fkey, nil
}

type LameSecretUI struct{}

func (d LameSecretUI) GetPassphrase(pinentry keybase1.GUIEntryArg, terminal *keybase1.SecretEntryArg) (keybase1.GetPassphraseRes, error) {
	return keybase1.GetPassphraseRes{}, fmt.Errorf("no secret UI available")
}

var getLameSecretUI = func() libkb.SecretUI { return LameSecretUI{} }
<commit_msg>bust team cache<commit_after>package teams

import (
	"fmt"
	"log"
	"sync"

	lru "github.com/hashicorp/golang-lru"
	context "golang.org/x/net/context"

	"github.com/keybase/client/go/encrypteddb"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

// Store TeamData's on memory and disk. Threadsafe.
type Storage struct {
	libkb.Contextified
	sync.Mutex
	mem  *MemoryStorage
	disk *DiskStorage
}

func NewStorage(g *libkb.GlobalContext) *Storage {
	return &Storage{
		Contextified: libkb.NewContextified(g),
		mem:          NewMemoryStorage(g),
		disk:         NewDiskStorage(g),
	}
}

func (s *Storage) Put(ctx context.Context, state *keybase1.TeamData) {
	s.Lock()
	defer s.Unlock()

	s.mem.Put(ctx, state)

	err := s.disk.Put(ctx, state)
	if err != nil {
		s.G().Log.CWarningf(ctx, "teams.Storage.Put err: %v", err)
	}
}

// Can return nil.
func (s *Storage) Get(ctx context.Context, teamID keybase1.TeamID, public bool) *keybase1.TeamData {
	s.Lock()
	defer s.Unlock()

	item := s.mem.Get(ctx, teamID, public)
	if item != nil {
		// Mem hit
		return item
	}

	res, found, err := s.disk.Get(ctx, teamID, public)
	if found && err == nil {
		// Disk hit
		s.mem.Put(ctx, res)
		return res
	}
	if err != nil {
		s.G().Log.Debug("teams.Storage#Get disk err: %v", err)
	}

	return nil
}

func (s *Storage) Delete(ctx context.Context, teamID keybase1.TeamID, public bool) error {
	s.Lock()
	defer s.Unlock()

	s.mem.Delete(ctx, teamID, public)
	return s.disk.Delete(ctx, teamID, public)
}

func (s *Storage) onLogout() {
	s.mem.onLogout()
}

// --------------------------------------------------

// Store TeamData's on disk. Threadsafe.
type DiskStorage struct {
	libkb.Contextified
	sync.Mutex
	encryptedDB *encrypteddb.EncryptedDB
}

// Increment to invalidate the disk cache.
const diskStorageVersion = 6

type DiskStorageItem struct {
	Version int                `codec:"V"`
	State   *keybase1.TeamData `codec:"S"`
}

func NewDiskStorage(g *libkb.GlobalContext) *DiskStorage {
	keyFn := func(ctx context.Context) ([32]byte, error) {
		return getLocalStorageSecretBoxKey(ctx, g)
	}
	dbFn := func(g *libkb.GlobalContext) *libkb.JSONLocalDb {
		return g.LocalDb
	}
	return &DiskStorage{
		Contextified: libkb.NewContextified(g),
		encryptedDB:  encrypteddb.New(g, dbFn, keyFn),
	}
}

func (s *DiskStorage) Put(ctx context.Context, state *keybase1.TeamData) error {
	s.Lock()
	defer s.Unlock()

	key := s.dbKey(ctx, state.Chain.Id, state.Chain.Public)
	item := DiskStorageItem{
		Version: diskStorageVersion,
		State:   state,
	}

	err := s.encryptedDB.Put(ctx, key, item)
	if err != nil {
		return err
	}
	return nil
}

// Res is valid if (found && err == nil)
func (s *DiskStorage) Get(ctx context.Context, teamID keybase1.TeamID, public bool) (res *keybase1.TeamData, found bool, err error) {
	s.Lock()
	defer s.Unlock()

	key := s.dbKey(ctx, teamID, public)
	var item DiskStorageItem
	found, err = s.encryptedDB.Get(ctx, key, &item)
	if (err != nil) || !found {
		return res, found, err
	}

	if item.Version != diskStorageVersion {
		// Pretend it wasn't found.
		return res, false, nil
	}

	// Sanity check
	if len(item.State.Chain.Id) == 0 {
		return res, false, fmt.Errorf("decode from disk had empty team id")
	}
	if !item.State.Chain.Id.Eq(teamID) {
		return res, false, fmt.Errorf("decode from disk had wrong team id %v != %v", item.State.Chain.Id, teamID)
	}
	if item.State.Chain.Public != public {
		return res, false, fmt.Errorf("decode from disk had wrong publicness %v != %v (%v)", item.State.Chain.Public, public, teamID)
	}

	return item.State, true, nil
}

func (s *DiskStorage) Delete(ctx context.Context, teamID keybase1.TeamID, public bool) error {
	s.Lock()
	defer s.Unlock()

	key := s.dbKey(ctx, teamID, public)
	return s.encryptedDB.Delete(ctx, key)
}

func (s *DiskStorage) dbKey(ctx context.Context, teamID keybase1.TeamID, public bool) libkb.DbKey {
	key := fmt.Sprintf("tid:%s", teamID)
	if public {
		key = fmt.Sprintf("tid:%s|pub", teamID)
	}
	return libkb.DbKey{
		Typ: libkb.DBChatInbox,
		Key: key,
	}
}

// --------------------------------------------------

const MemCacheLRUSize = 200

// Store some TeamSigChainState's in memory. Threadsafe.
type MemoryStorage struct {
	libkb.Contextified
	lru *lru.Cache
}

func NewMemoryStorage(g *libkb.GlobalContext) *MemoryStorage {
	nlru, err := lru.New(MemCacheLRUSize)
	if err != nil {
		// lru.New only panics if size <= 0
		log.Panicf("Could not create lru cache: %v", err)
	}
	return &MemoryStorage{
		Contextified: libkb.NewContextified(g),
		lru:          nlru,
	}
}

func (s *MemoryStorage) Put(ctx context.Context, state *keybase1.TeamData) {
	s.lru.Add(s.key(state.Chain.Id, state.Chain.Public), state)
}

// Can return nil.
func (s *MemoryStorage) Get(ctx context.Context, teamID keybase1.TeamID, public bool) *keybase1.TeamData {
	untyped, ok := s.lru.Get(s.key(teamID, public))
	if !ok {
		return nil
	}
	state, ok := untyped.(*keybase1.TeamData)
	if !ok {
		s.G().Log.Warning("Team MemoryStorage got bad type from lru: %T", untyped)
		return nil
	}
	return state
}

func (s *MemoryStorage) Delete(ctx context.Context, teamID keybase1.TeamID, public bool) {
	s.lru.Remove(s.key(teamID, public))
}

func (s *MemoryStorage) onLogout() {
	s.lru.Purge()
}

func (s *MemoryStorage) key(teamID keybase1.TeamID, public bool) (key string) {
	key = fmt.Sprintf("tid:%s", teamID)
	if public {
		key = fmt.Sprintf("tid:%s|pub", teamID)
	}
	return key
}

// --------------------------------------------------

func getLocalStorageSecretBoxKey(ctx context.Context, g *libkb.GlobalContext) (fkey [32]byte, err error) {
	// Get secret device key
	encKey, err := engine.GetMySecretKey(ctx, g, getLameSecretUI, libkb.DeviceEncryptionKeyType,
		"encrypt teams storage")
	if err != nil {
		return fkey, err
	}
	kp, ok := encKey.(libkb.NaclDHKeyPair)
	if !ok || kp.Private == nil {
		return fkey, libkb.KeyCannotDecryptError{}
	}

	// Derive symmetric key from device key
	skey, err := encKey.SecretSymmetricKey(libkb.EncryptionReasonTeamsLocalStorage)
	if err != nil {
		return fkey, err
	}

	copy(fkey[:], skey[:])
	return fkey, nil
}

type LameSecretUI struct{}

func (d LameSecretUI) GetPassphrase(pinentry keybase1.GUIEntryArg, terminal *keybase1.SecretEntryArg) (keybase1.GetPassphraseRes, error) {
	return keybase1.GetPassphraseRes{}, fmt.Errorf("no secret UI available")
}

var getLameSecretUI = func() libkb.SecretUI { return LameSecretUI{} }
<|endoftext|>


<commit_before>package repo

import (
	"database/sql/driver"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"
)

// URI identifies a repository.
type URI string

// IsGitHubRepository returns true iff this repository is hosted on GitHub.
func (u URI) IsGitHubRepository() bool {
	return strings.HasPrefix(strings.ToLower(string(u)), "github.com/")
}

// GitHubURL returns the https://github.com/USER/REPO URL for this repository.
func (u URI) GitHubURL() string {
	return "https://" + string(u)
}

// MakeURI converts a repository clone URL, such as
// "git://github.com/user/repo.git", to a normalized URI string, such as
// "github.com/user/repo".
func MakeURI(cloneURL string) URI {
	if cloneURL == "" {
		panic("MakeURI: empty clone URL")
	}

	url, err := url.Parse(cloneURL)
	if err != nil {
		panic(fmt.Sprintf("MakeURI(%q): %s", cloneURL, err))
	}

	path := strings.TrimSuffix(url.Path, ".git")
	path = filepath.Clean(path)
	path = strings.TrimSuffix(path, "/")
	return URI(strings.ToLower(url.Host) + path)
}

// URIEqual returns true if a and b are equal, based on a case insensitive
// comparison.
func URIEqual(a, b URI) bool {
	return strings.ToLower(string(a)) == strings.ToLower(string(b))
}

// Scan implements database/sql.Scanner.
func (u *URI) Scan(v interface{}) error {
	if v == nil {
		*u = ""
		return nil
	} else if data, ok := v.([]byte); ok {
		*u = URI(data)
		return nil
	}
	return fmt.Errorf("%T.Scan failed: %v", u, v)
}

// Value implements database/sql/driver.Valuer
func (u URI) Value() (driver.Value, error) {
	return string(u), nil
}

// URIs is a wrapper type for a slice of URIs.
type URIs []URI

// Strings returns the URIs as strings.
func (us URIs) Strings() []string {
	s := make([]string, len(us))
	for i, u := range us {
		s[i] = string(u)
	}
	return s
}
<commit_msg>IsGoogleCodeRepository helper fn<commit_after>package repo

import (
	"database/sql/driver"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"
)

// URI identifies a repository.
type URI string

// IsGitHubRepository returns true iff this repository is hosted on GitHub.
func (u URI) IsGitHubRepository() bool {
	return strings.HasPrefix(strings.ToLower(string(u)), "github.com/")
}

// IsGoogleCodeRepository returns true iff this repository is hosted on Google
// Code (code.google.com).
func (u URI) IsGoogleCodeRepository() bool {
	return strings.HasPrefix(strings.ToLower(string(u)), "code.google.com/p/")
}

// GitHubURL returns the https://github.com/USER/REPO URL for this repository.
func (u URI) GitHubURL() string {
	return "https://" + string(u)
}

// MakeURI converts a repository clone URL, such as
// "git://github.com/user/repo.git", to a normalized URI string, such as
// "github.com/user/repo".
func MakeURI(cloneURL string) URI {
	if cloneURL == "" {
		panic("MakeURI: empty clone URL")
	}

	url, err := url.Parse(cloneURL)
	if err != nil {
		panic(fmt.Sprintf("MakeURI(%q): %s", cloneURL, err))
	}

	path := strings.TrimSuffix(url.Path, ".git")
	path = filepath.Clean(path)
	path = strings.TrimSuffix(path, "/")
	return URI(strings.ToLower(url.Host) + path)
}

// URIEqual returns true if a and b are equal, based on a case insensitive
// comparison.
func URIEqual(a, b URI) bool {
	return strings.ToLower(string(a)) == strings.ToLower(string(b))
}

// Scan implements database/sql.Scanner.
func (u *URI) Scan(v interface{}) error {
	if v == nil {
		*u = ""
		return nil
	} else if data, ok := v.([]byte); ok {
		*u = URI(data)
		return nil
	}
	return fmt.Errorf("%T.Scan failed: %v", u, v)
}

// Value implements database/sql/driver.Valuer
func (u URI) Value() (driver.Value, error) {
	return string(u), nil
}

// URIs is a wrapper type for a slice of URIs.
type URIs []URI

// Strings returns the URIs as strings.
func (us URIs) Strings() []string {
	s := make([]string, len(us))
	for i, u := range us {
		s[i] = string(u)
	}
	return s
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"net/http"
	"path"
	"time"

	"strings"

	"net/url"

	"io/ioutil"

	"net"

	"fmt"

	"os"

	"github.com/asaskevich/govalidator"
	"github.com/ellcrys/cocoon/core/config"
	"github.com/ellcrys/cocoon/core/connector/server/proto_connector"
	"github.com/ellcrys/util"
	"github.com/gorilla/mux"
	logging "github.com/op/go-logging"
	"github.com/pkg/errors"
	context "golang.org/x/net/context"
)

var httpLog *logging.Logger

// MaxBytesRead limits the number of bytes to read per request
const MaxBytesRead = 10000000

// InvokeRequest describes the body of a structured invoke request
type InvokeRequest struct {
	ID       string   `json:"id"`
	Function string   `json:"function"`
	Params   []string `json:"params"`
}

// InvokeError describes an invoke error
type InvokeError struct {
	Code  string `json:"code"`
	Msg   string `json:"msg"`
	Error bool   `json:"error"`
}

// Success describes a successful response from a cocoon code
type Success struct {
	Body []byte `json:"body"`
}

// headerToMap converts http.Header to a map[string]string.
// Header keys with multiple values are joined to as a single string with
// a comma `,` as a delimeter.
func headerToMap(header http.Header) map[string]string {
	var newMap = make(map[string]string)
	for h, v := range header {
		newMap[h] = strings.Join(v, ",")
	}
	return newMap
}

// headerToMap converts url.Values to a map[string]string.
// Values keys with multiple values are joined to as a single string with
// a comma `,` as a delimeter.
func valuesToMap(values url.Values) map[string]string {
	var newMap = make(map[string]string)
	for h, v := range values {
		newMap[h] = strings.Join(v, ",")
	}
	return newMap
}

// prepareInvokeHeader collects and sets values to be included in the header
// that will be sent to the cocoon code.
func prepareInvokeHeader(w http.ResponseWriter, r *http.Request) http.Header {
	header := r.Header
	header.Set("Method", r.Method)
	header.Set("Host", r.Host)
	header.Set("Remote-Addr", r.RemoteAddr)
	if len(r.Cookies()) > 0 {
		cookiesToJSON, _ := util.ToJSON(r.Cookies())
		header.Set("Cookies", string(cookiesToJSON))
	}
	header.Set("Referer", r.Referer())
	header.Set("URL", r.URL.String())
	header.Set("Request-Uri", r.RequestURI)
	if len(r.URL.Query()) > 0 {
		queryToJSON, _ := util.ToJSON(valuesToMap(r.URL.Query()))
		header.Set("Query", string(queryToJSON))
	}

	r.ParseForm()
	r.ParseMultipartForm(MaxBytesRead)
	if len(r.PostForm) > 0 {
		formValToJSON, _ := util.ToJSON(valuesToMap(r.PostForm))
		header.Set("Form", string(formValToJSON))
	}
	return header
}

// HTTP defines a structure for an HTTP server
// that provides REST API services.
type HTTP struct {
	rpc *RPC
}

// NewHTTP creates an new http server instance
func NewHTTP(rpc *RPC) *HTTP {
	httpLog = config.MakeLogger("connector.http")
	return &HTTP{rpc}
}

// getRouter returns the router
func (s *HTTP) getRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/", s.index)
	r.Handle("/static/", http.FileServer(http.Dir(path.Join(os.Getenv("SHARED_DIR"), "static"))))

	// v1 API routes
	v1 := r.PathPrefix("/v1").Subrouter()
	v1.HandleFunc("/invoke", s.invokeCocoonCode)

	return r
}

// Start starts the http server. Passes true to the startedCh channel
// when started
func (s *HTTP) Start(addr string, startedCh chan bool) error {
	_, port, err := net.SplitHostPort(addr)
	if err != nil {
		log.Errorf("%+v", errors.Wrap(err, "failed to parse address"))
		startedCh <- false
		return err
	}

	time.AfterFunc(2*time.Second, func() {
		httpLog.Infof("Started HTTP API server @ :%s", port)
		startedCh <- true
	})

	return http.ListenAndServe(addr, s.getRouter())
}

// index page
func (s *HTTP) index(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")

	fmt.Fprint(w, "Hello!")
}

// invokeCocoonCode handles cocoon code invocation
func (s *HTTP) invokeCocoonCode(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization")

	// return 200 for options request
	if r.Method == http.MethodOptions {
		return
	}

	var err error
	var resp *proto_connector.Response
	var invokeRequest InvokeRequest
	var body []byte
	var ctx context.Context
	var cc context.CancelFunc

	defer r.Body.Close()

	// prepare header
	r.Body = http.MaxBytesReader(w, r.Body, MaxBytesRead)
	preparedHeader := prepareInvokeHeader(w, r)

	// set response content type
	w.Header().Set("Content-Type", "application/json")

	// get request are considered as unstructured
	if r.Method == "GET" {
		goto UNSTRUCTURED
	}

	// get body
	if r.Method == "POST" {
		body, err = ioutil.ReadAll(r.Body)
		if err != nil {
			w.WriteHeader(500)
			json.NewEncoder(w).Encode(InvokeError{
				Error: true,
				Code:  "server_error",
				Msg:   "Invalid invoke body structure",
			})
			return
		}
	}

	// non-json content type are considered as unstructured request
	if r.Header.Get("Content-Type") != "application/json" {
		goto UNSTRUCTURED
	}

	// attempt to decode json body to an invoke request structure
	err = util.FromJSON(body, &invokeRequest)
	if err != nil {
		w.WriteHeader(400)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "1",
			Msg:   "Invalid invoke body structure",
		})
		return
	}

	// expect ID, if set to be a UUIDv4 string
	if len(invokeRequest.ID) > 0 && !govalidator.IsUUIDv4(invokeRequest.ID) {
		w.WriteHeader(400)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "2",
			Msg:   "invalid ID. ID must be a UUIDv4 value",
		})
		return
	}

	// return error if request function is not set
	if len(strings.TrimSpace(invokeRequest.Function)) == 0 {
		w.WriteHeader(400)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "3",
			Msg:   "function is required",
		})
		return
	}

	// set ID of not set
	if len(invokeRequest.ID) == 0 {
		invokeRequest.ID = util.UUID4()
	}

	preparedHeader.Set("Transaction-Id", invokeRequest.ID)
	preparedHeader.Set("Structured", "yes")

	ctx, cc = context.WithTimeout(context.Background(), 2*time.Minute)
	defer cc()
	resp, err = s.rpc.cocoonCodeOps.Handle(ctx, &proto_connector.CocoonCodeOperation{
		ID:       invokeRequest.ID,
		Function: invokeRequest.Function,
		Params:   invokeRequest.Params,
		Header:   headerToMap(preparedHeader),
	})
	if err != nil {
		w.WriteHeader(500)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "code_error",
			Msg:   err.Error(),
		})
		return
	}

	if r.Method == "OPTIONS" {
		return
	}

	json.NewEncoder(w).Encode(Success{
		Body: resp.GetBody(),
	})
	return

UNSTRUCTURED:

	preparedHeader.Set("Structured", "no")

	if r.Method != "GET" {
		preparedHeader.Set("Raw-Body", string(body))
	}

	ctx, cc = context.WithTimeout(context.Background(), 2*time.Minute)
	defer cc()
	resp, err = s.rpc.cocoonCodeOps.Handle(ctx, &proto_connector.CocoonCodeOperation{
		Header: headerToMap(preparedHeader),
	})
	if err != nil {
		w.WriteHeader(500)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "code_error",
			Msg:   err.Error(),
		})
		return
	}

	w.WriteHeader(200)
	json.NewEncoder(w).Encode(Success{
		Body: resp.GetBody(),
	})
}
<commit_msg>strip prefix<commit_after>package server

import (
	"encoding/json"
	"net/http"
	"path"
	"time"

	"strings"

	"net/url"

	"io/ioutil"

	"net"

	"fmt"

	"os"

	"github.com/asaskevich/govalidator"
	"github.com/ellcrys/cocoon/core/config"
	"github.com/ellcrys/cocoon/core/connector/server/proto_connector"
	"github.com/ellcrys/util"
	"github.com/gorilla/mux"
	logging "github.com/op/go-logging"
	"github.com/pkg/errors"
	context "golang.org/x/net/context"
)

var httpLog *logging.Logger

// MaxBytesRead limits the number of bytes to read per request
const MaxBytesRead = 10000000

// InvokeRequest describes the body of a structured invoke request
type InvokeRequest struct {
	ID       string   `json:"id"`
	Function string   `json:"function"`
	Params   []string `json:"params"`
}

// InvokeError describes an invoke error
type InvokeError struct {
	Code  string `json:"code"`
	Msg   string `json:"msg"`
	Error bool   `json:"error"`
}

// Success describes a successful response from a cocoon code
type Success struct {
	Body []byte `json:"body"`
}

// headerToMap converts http.Header to a map[string]string.
// Header keys with multiple values are joined to as a single string with
// a comma `,` as a delimeter.
func headerToMap(header http.Header) map[string]string {
	var newMap = make(map[string]string)
	for h, v := range header {
		newMap[h] = strings.Join(v, ",")
	}
	return newMap
}

// headerToMap converts url.Values to a map[string]string.
// Values keys with multiple values are joined to as a single string with
// a comma `,` as a delimeter.
func valuesToMap(values url.Values) map[string]string {
	var newMap = make(map[string]string)
	for h, v := range values {
		newMap[h] = strings.Join(v, ",")
	}
	return newMap
}

// prepareInvokeHeader collects and sets values to be included in the header
// that will be sent to the cocoon code.
func prepareInvokeHeader(w http.ResponseWriter, r *http.Request) http.Header {
	header := r.Header
	header.Set("Method", r.Method)
	header.Set("Host", r.Host)
	header.Set("Remote-Addr", r.RemoteAddr)
	if len(r.Cookies()) > 0 {
		cookiesToJSON, _ := util.ToJSON(r.Cookies())
		header.Set("Cookies", string(cookiesToJSON))
	}
	header.Set("Referer", r.Referer())
	header.Set("URL", r.URL.String())
	header.Set("Request-Uri", r.RequestURI)
	if len(r.URL.Query()) > 0 {
		queryToJSON, _ := util.ToJSON(valuesToMap(r.URL.Query()))
		header.Set("Query", string(queryToJSON))
	}

	r.ParseForm()
	r.ParseMultipartForm(MaxBytesRead)
	if len(r.PostForm) > 0 {
		formValToJSON, _ := util.ToJSON(valuesToMap(r.PostForm))
		header.Set("Form", string(formValToJSON))
	}
	return header
}

// HTTP defines a structure for an HTTP server
// that provides REST API services.
type HTTP struct {
	rpc *RPC
}

// NewHTTP creates an new http server instance
func NewHTTP(rpc *RPC) *HTTP {
	httpLog = config.MakeLogger("connector.http")
	return &HTTP{rpc}
}

// getRouter returns the router
func (s *HTTP) getRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/", s.index)
	r.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(path.Join(os.Getenv("SHARED_DIR"), "static")))))

	// v1 API routes
	v1 := r.PathPrefix("/v1").Subrouter()
	v1.HandleFunc("/invoke", s.invokeCocoonCode)

	return r
}

// Start starts the http server. Passes true to the startedCh channel
// when started
func (s *HTTP) Start(addr string, startedCh chan bool) error {
	_, port, err := net.SplitHostPort(addr)
	if err != nil {
		log.Errorf("%+v", errors.Wrap(err, "failed to parse address"))
		startedCh <- false
		return err
	}

	time.AfterFunc(2*time.Second, func() {
		httpLog.Infof("Started HTTP API server @ :%s", port)
		startedCh <- true
	})

	return http.ListenAndServe(addr, s.getRouter())
}

// index page
func (s *HTTP) index(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")

	fmt.Fprint(w, "Hello!")
}

// invokeCocoonCode handles cocoon code invocation
func (s *HTTP) invokeCocoonCode(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization")

	// return 200 for options request
	if r.Method == http.MethodOptions {
		return
	}

	var err error
	var resp *proto_connector.Response
	var invokeRequest InvokeRequest
	var body []byte
	var ctx context.Context
	var cc context.CancelFunc

	defer r.Body.Close()

	// prepare header
	r.Body = http.MaxBytesReader(w, r.Body, MaxBytesRead)
	preparedHeader := prepareInvokeHeader(w, r)

	// set response content type
	w.Header().Set("Content-Type", "application/json")

	// get request are considered as unstructured
	if r.Method == "GET" {
		goto UNSTRUCTURED
	}

	// get body
	if r.Method == "POST" {
		body, err = ioutil.ReadAll(r.Body)
		if err != nil {
			w.WriteHeader(500)
			json.NewEncoder(w).Encode(InvokeError{
				Error: true,
				Code:  "server_error",
				Msg:   "Invalid invoke body structure",
			})
			return
		}
	}

	// non-json content type are considered as unstructured request
	if r.Header.Get("Content-Type") != "application/json" {
		goto UNSTRUCTURED
	}

	// attempt to decode json body to an invoke request structure
	err = util.FromJSON(body, &invokeRequest)
	if err != nil {
		w.WriteHeader(400)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "1",
			Msg:   "Invalid invoke body structure",
		})
		return
	}

	// expect ID, if set to be a UUIDv4 string
	if len(invokeRequest.ID) > 0 && !govalidator.IsUUIDv4(invokeRequest.ID) {
		w.WriteHeader(400)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "2",
			Msg:   "invalid ID. ID must be a UUIDv4 value",
		})
		return
	}

	// return error if request function is not set
	if len(strings.TrimSpace(invokeRequest.Function)) == 0 {
		w.WriteHeader(400)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "3",
			Msg:   "function is required",
		})
		return
	}

	// set ID of not set
	if len(invokeRequest.ID) == 0 {
		invokeRequest.ID = util.UUID4()
	}

	preparedHeader.Set("Transaction-Id", invokeRequest.ID)
	preparedHeader.Set("Structured", "yes")

	ctx, cc = context.WithTimeout(context.Background(), 2*time.Minute)
	defer cc()
	resp, err = s.rpc.cocoonCodeOps.Handle(ctx, &proto_connector.CocoonCodeOperation{
		ID:       invokeRequest.ID,
		Function: invokeRequest.Function,
		Params:   invokeRequest.Params,
		Header:   headerToMap(preparedHeader),
	})
	if err != nil {
		w.WriteHeader(500)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "code_error",
			Msg:   err.Error(),
		})
		return
	}

	if r.Method == "OPTIONS" {
		return
	}

	json.NewEncoder(w).Encode(Success{
		Body: resp.GetBody(),
	})
	return

UNSTRUCTURED:

	preparedHeader.Set("Structured", "no")

	if r.Method != "GET" {
		preparedHeader.Set("Raw-Body", string(body))
	}

	ctx, cc = context.WithTimeout(context.Background(), 2*time.Minute)
	defer cc()
	resp, err = s.rpc.cocoonCodeOps.Handle(ctx, &proto_connector.CocoonCodeOperation{
		Header: headerToMap(preparedHeader),
	})
	if err != nil {
		w.WriteHeader(500)
		json.NewEncoder(w).Encode(InvokeError{
			Error: true,
			Code:  "code_error",
			Msg:   err.Error(),
		})
		return
	}

	w.WriteHeader(200)
	json.NewEncoder(w).Encode(Success{
		Body: resp.GetBody(),
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Matt Jibson (mjibson@cockroachlabs.com)

package sql_test

import (
	gosql "database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/internal/rsg"
	"github.com/cockroachdb/cockroach/sql/parser"
	"github.com/cockroachdb/cockroach/testutils/serverutils"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/timeutil"
)

var (
	flagRSGTime       = flag.Duration("rsg", 0, "random syntax generator test duration")
	flagRSGGoRoutines = flag.Int("rsg-routines", 1, "number of Go routines executing random statements in each RSG test")
)

func TestRandomSyntaxGeneration(t *testing.T) {
	defer leaktest.AfterTest(t)()

	const rootStmt = "target_list"

	testRandomSyntax(t, func(db *gosql.DB, r *rsg.RSG) {
		s := r.Generate(rootStmt, 20)
		if strings.HasPrefix(s, "REVOKE") || strings.HasPrefix(s, "GRANT") {
			return
		}
		_, _ = db.Exec(`ROLLBACK`)
		_, _ = db.Exec(`CREATE DATABASE IF NOT EXISTS name; SET DATABASE name;`)
		_, _ = db.Exec(s)
	})
}

func TestRandomSyntaxSelect(t *testing.T) {
	defer leaktest.AfterTest(t)()

	const rootStmt = "stmt"

	testRandomSyntax(t, func(db *gosql.DB, r *rsg.RSG) {
		if _, err := db.Exec(`CREATE DATABASE IF NOT EXISTS ident; CREATE TABLE IF NOT EXISTS ident.ident (ident decimal);`); err != nil {
			panic(err)
		}

		targets := r.Generate(rootStmt, 30)
		var where, from string
		// Only generate complex clauses half the time.
		if rand.Intn(2) == 0 {
			where = r.Generate("where_clause", 30)
			from = r.Generate("from_clause", 30)
		} else {
			from = "FROM ident"
		}
		s := fmt.Sprintf("SELECT %s %s %s", targets, from, where)
		_, _ = db.Exec(`ROLLBACK`)
		_, _ = db.Exec(`SET DATABASE = ident`)
		_, _ = db.Exec(s)
	})
}

func TestRandomSyntaxFunctions(t *testing.T) {
	defer leaktest.AfterTest(t)()

	var names []string
	for b := range parser.Builtins {
		names = append(names, b)
	}

	testRandomSyntax(t, func(db *gosql.DB, r *rsg.RSG) {
		name := names[r.Intn(len(names))]
		variations := parser.Builtins[name]
		fn := variations[r.Intn(len(variations))]
		var args []string
		switch ft := fn.Types.(type) {
		case parser.ArgTypes:
			for _, typ := range ft {
				var v interface{}
				switch typ.(type) {
				case *parser.DInt:
					i := r.Intn(math.MaxInt64)
					i -= r.Intn(math.MaxInt64)
					v = i
				case *parser.DFloat, *parser.DDecimal:
					v = r.Float64()
				case *parser.DString:
					v = `'string'`
				case *parser.DBytes:
					v = `b'bytes'`
				case *parser.DTimestamp:
					t := time.Unix(0, int64(r.Intn(math.MaxInt64)))
					v = fmt.Sprintf(`'%s'`, t.Format(time.RFC3339Nano))
				default:
					panic(fmt.Errorf("unknown arg type: %T", typ))
				}
				args = append(args, fmt.Sprint(v))
			}
		default:
			return
		}
		s := fmt.Sprintf("SELECT %s(%s)", name, strings.Join(args, ", "))
		_, _ = db.Exec("ROLLBACK")
		funcdone := make(chan bool, 1)
		go func() {
			_, _ = db.Exec(s)
			funcdone <- true
		}()
		select {
		case <-funcdone:
		case <-time.After(time.Second * 5):
			panic(fmt.Errorf("func exec timeout: %s", s))
		}
	})
}

func TestRandomSyntaxFuncCommon(t *testing.T) {
	defer leaktest.AfterTest(t)()

	const rootStmt = "func_expr_common_subexpr"

	testRandomSyntax(t, func(db *gosql.DB, r *rsg.RSG) {
		expr := r.Generate(rootStmt, 30)
		s := fmt.Sprintf("SELECT %s", expr)
		_, _ = db.Exec(`ROLLBACK`)
		_, _ = db.Exec(s)
	})
}

func testRandomSyntax(t *testing.T, f func(db *gosql.DB, r *rsg.RSG)) {
	if *flagRSGTime == 0 {
		t.Skip("enable with '-rsg <duration>'")
	}

	params, _ := createTestServerParams()
	s, db, _ := serverutils.StartServer(t, params)
	defer s.Stopper().Stop()

	y, err := ioutil.ReadFile(filepath.Join("parser", "sql.y"))
	if err != nil {
		t.Fatal(err)
	}
	r, err := rsg.NewRSG(timeutil.Now().UnixNano(), string(y))
	if err != nil {
		t.Fatal(err)
	}
	// Broadcast channel for all workers.
	done := make(chan bool)
	var wg sync.WaitGroup
	worker := func() {
		defer wg.Done()
		for {
			select {
			case <-done:
				return
			default:
			}
			f(db, r)
		}
	}
	for i := 0; i < *flagRSGGoRoutines; i++ {
		go worker()
		wg.Add(1)
	}
	time.Sleep(*flagRSGTime)
	close(done)
	wg.Wait()
}
<commit_msg>sql: fix random tests to actually work<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Matt Jibson (mjibson@cockroachlabs.com)

package sql_test

import (
	gosql "database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/internal/rsg"
	"github.com/cockroachdb/cockroach/sql/parser"
	"github.com/cockroachdb/cockroach/testutils/serverutils"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/syncutil"
	"github.com/cockroachdb/cockroach/util/timeutil"
)

var (
	flagRSGTime       = flag.Duration("rsg", 0, "random syntax generator test duration")
	flagRSGGoRoutines = flag.Int("rsg-routines", 1, "number of Go routines executing random statements in each RSG test")
)

func TestRandomSyntaxGeneration(t *testing.T) {
	defer leaktest.AfterTest(t)()

	const rootStmt = "stmt"

	testRandomSyntax(t, nil, func(db *gosql.DB, r *rsg.RSG) bool {
		s := r.Generate(rootStmt, 20)
		// TODO(mjibson): figure out why these run slowly. May be a bug.
		if strings.HasPrefix(s, "REVOKE") || strings.HasPrefix(s, "GRANT") {
			return false
		}
		// The rollback may fail; it's here in case the stmt started a transaction.
		_, _ = db.Exec(`ROLLBACK`)
		// But the create should always succeed.
		_, err := db.Exec(`CREATE DATABASE IF NOT EXISTS ident; SET DATABASE = ident;`)
		if err != nil {
			panic(err)
		}
		_, err = db.Exec(s)
		return err == nil
	})
}

func TestRandomSyntaxSelect(t *testing.T) {
	defer leaktest.AfterTest(t)()

	const rootStmt = "target_list"

	testRandomSyntax(t, func(db *gosql.DB) error {
		_, err := db.Exec(`CREATE DATABASE IF NOT EXISTS ident; CREATE TABLE IF NOT EXISTS ident.ident (ident decimal);`)
		return err
	}, func(db *gosql.DB, r *rsg.RSG) bool {
		targets := r.Generate(rootStmt, 30)
		var where, from string
		// Only generate complex clauses half the time.
		if rand.Intn(2) == 0 {
			where = r.Generate("where_clause", 30)
			from = r.Generate("from_clause", 30)
		} else {
			from = "FROM ident"
		}
		s := fmt.Sprintf("SELECT %s %s %s", targets, from, where)
		_, err := db.Exec(`SET DATABASE = ident`)
		if err != nil {
			panic(err)
		}
		_, err = db.Exec(s)
		return err == nil
	})
}

func TestRandomSyntaxFunctions(t *testing.T) {
	defer leaktest.AfterTest(t)()

	var names []string
	for b := range parser.Builtins {
		names = append(names, b)
	}

	testRandomSyntax(t, nil, func(db *gosql.DB, r *rsg.RSG) bool {
		name := names[r.Intn(len(names))]
		variations := parser.Builtins[name]
		fn := variations[r.Intn(len(variations))]
		var args []string
		switch ft := fn.Types.(type) {
		case parser.ArgTypes:
			for _, typ := range ft {
				var v interface{}
				switch typ.(type) {
				case *parser.DInt:
					i := r.Intn(math.MaxInt64)
					i -= r.Intn(math.MaxInt64)
					v = i
				case *parser.DFloat, *parser.DDecimal:
					v = r.Float64()
				case *parser.DString:
					v = `'string'`
				case *parser.DBytes:
					v = `b'bytes'`
				case *parser.DTimestamp:
					t := time.Unix(0, int64(r.Intn(math.MaxInt64)))
					v = fmt.Sprintf(`'%s'`, t.Format(time.RFC3339Nano))
				default:
					panic(fmt.Errorf("unknown arg type: %T", typ))
				}
				args = append(args, fmt.Sprint(v))
			}
		default:
			return false
		}
		s := fmt.Sprintf("SELECT %s(%s)", name, strings.Join(args, ", "))
		funcdone := make(chan bool, 1)
		go func() {
			_, err := db.Exec(s)
			funcdone <- err == nil
		}()
		select {
		case success := <-funcdone:
			return success
		case <-time.After(time.Second * 5):
			panic(fmt.Errorf("func exec timeout: %s", s))
		}
	})
}

func TestRandomSyntaxFuncCommon(t *testing.T) {
	defer leaktest.AfterTest(t)()

	const rootStmt = "func_expr_common_subexpr"

	testRandomSyntax(t, nil, func(db *gosql.DB, r *rsg.RSG) bool {
		expr := r.Generate(rootStmt, 30)
		s := fmt.Sprintf("SELECT %s", expr)
		_, err := db.Exec(s)
		return err == nil
	})
}

// testRandomSyntax performs all of the RSG setup and teardown for common random syntax testing operations. It takes f, a closure where the random expression should be generated and executed. It returns a boolean indicating if the statement executed successfully. This is used to verify that at least 1 success occurs (otherwise it is likely a bad test).
func testRandomSyntax(t *testing.T, setup func(db *gosql.DB) error, f func(db *gosql.DB, r *rsg.RSG) (success bool)) {
	if *flagRSGTime == 0 {
		t.Skip("enable with '-rsg <duration>'")
	}

	params, _ := createTestServerParams()
	s, db, _ := serverutils.StartServer(t, params)
	defer s.Stopper().Stop()

	if setup != nil {
		err := setup(db)
		if err != nil {
			t.Fatal(err)
		}
	}

	y, err := ioutil.ReadFile(filepath.Join("parser", "sql.y"))
	if err != nil {
		t.Fatal(err)
	}
	r, err := rsg.NewRSG(timeutil.Now().UnixNano(), string(y))
	if err != nil {
		t.Fatal(err)
	}
	// Broadcast channel for all workers.
	done := make(chan bool)
	var wg sync.WaitGroup
	var lock syncutil.Mutex
	var total, success int
	worker := func() {
		defer wg.Done()
		for {
			select {
			case <-done:
				return
			default:
			}
			s := f(db, r)
			lock.Lock()
			total++
			if s {
				success++
			}
			lock.Unlock()
		}
	}
	for i := 0; i < *flagRSGGoRoutines; i++ {
		go worker()
		wg.Add(1)
	}
	time.Sleep(*flagRSGTime)
	close(done)
	wg.Wait()
	t.Logf("%d executions, %d successful", total, success)
	if success == 0 {
		t.Fatal("0 successful executions")
	}
}
<|endoftext|>