<commit_before>package waveguide

import (
	"html/template"
	"net/http"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
)

func init() {
	http.HandleFunc("/", handler)
	http.HandleFunc("/worker", worker)
}

type Counter struct {
	Name  string
	Count int
}

func handler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if name := r.FormValue("name"); name != "" {
		t := taskqueue.NewPOSTTask("/worker", map[string][]string{"name": {name}})
		if _, err := taskqueue.Add(ctx, t, ""); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	q := datastore.NewQuery("Counter")
	var counters []Counter
	if _, err := q.GetAll(ctx, &counters); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if err := handlerTemplate.Execute(w, counters); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	// OK
}

func worker(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	name := r.FormValue("name")
	key := datastore.NewKey(ctx, "Counter", name, 0, nil)
	var counter Counter
	if err := datastore.Get(ctx, key, &counter); err == datastore.ErrNoSuchEntity {
		counter.Name = name
	} else if err != nil {
		log.Errorf(ctx, "%v", err)
		return
	}
	counter.Count++
	if _, err := datastore.Put(ctx, key, &counter); err != nil {
		log.Errorf(ctx, "%v", err)
	}
}

var handlerTemplate = template.Must(template.New("handler").Parse(handlerHTML))

const handlerHTML = `
{{range .}}

{{.Name}}: {{.Count}}

{{end}}

Start a new counter:

<form action="/" method="POST">
<input type="text" name="name">
<input type="submit" value="Add">
</form>
`<commit_msg>Add /update_all and /update_one handlers.<commit_after>package waveguide

import (
	"net/http"
	"net/url"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"fmt"
	"io/ioutil"
	"regexp"
	"html"
	"strings"
	"time"
	"google.golang.org/appengine/urlfetch"
)

func init() {
	http.HandleFunc("/", handle(root))
	http.HandleFunc("/update_all", handle(updateAll))
	http.HandleFunc("/update_one", handle(updateOne))
}

func handle(f func(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := appengine.NewContext(r)
		status, err := f(ctx, w, r)
		if err != nil {
			log.Errorf(ctx, "%v", err)
			w.WriteHeader(status)
			fmt.Fprintf(w, "%v", err)
		}
	}
}

// root shows the main page.
func root(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	fmt.Fprintf(w, "Waveguide")
	return http.StatusOK, nil
}

func updateAll(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	// Download magicseaweed.com/site-map.php
	client := urlfetch.Client(ctx)
	body, err := get(client,"http://magicseaweed.com/site-map.php")
	if err != nil {
		return http.StatusInternalServerError, err
	}
	reportPaths := reportRx.FindAll(body, -1)

	// For each report url:
	for _, rp := range reportPaths {
		// TODO: See about using PutMulti if this needs to be more
		// efficient.
		err = saveNewSurfReportPath(ctx, rp)
		if err != nil {
			return http.StatusInternalServerError, err
		}

		// Add a task to the queue to download the report and store its stars and wave
		// height to the db.
		srp := string(rp)
		t := taskqueue.NewPOSTTask("/update_one", map[string][]string{"path": {srp}})
		if _, err := taskqueue.Add(ctx, t, ""); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return http.StatusInternalServerError, err
		}
	}

	fmt.Fprintf(w,"Processed %d paths.", len(reportPaths))

	return http.StatusOK, nil
}

// updateOne updates the conditions for a single surfing spot.
func updateOne(ctx context.Context, w http.ResponseWriter, r *http.Request) (int, error) {
	path := r.FormValue("path")
	log.Infof(ctx, "updateOne: path=%q", path)
	key := datastore.NewKey(ctx, "Spot", path, 0, nil)
	var s Spot
	if err := datastore.Get(ctx, key, &s); err == datastore.ErrNoSuchEntity {
		url := "http://magicseaweed.com" + s.MswPath
		q, err := fetchQuality(ctx, url)
		if err != nil {
			return http.StatusInternalServerError, err
		}
		s.Qual = append(s.Qual, *q)
		_, err = datastore.Put(ctx, key, &s)
		if err != nil {
			return http.StatusInternalServerError, err
		}
	}
	return http.StatusOK, nil
}

func get(client *http.Client, url string) ([]byte, error) {
	res, err := client.Get(url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to read body of %s", url)
	}
	return body, nil
}

var starSectionRx = regexp.MustCompile(`<ul class="rating rating-large clearfix">.*?</ul>`)
var starRx = regexp.MustCompile(`<li class="active"> *<i class="glyphicon glyphicon-star"></i> *</li>`)
var heightRx = regexp.MustCompile(`(\d+(?:-\d+)?)<small>ft`)
var reportRx = regexp.MustCompile(`/[^"/]+-Surf-Report/\d+/`)
var srpTailRx = regexp.MustCompile(`-Surf-Report/\d+/`)

type Spot struct {
	Name 		 string
	MswPath		 string
	// Surf conditions at this spot, most recent last
	Qual		 []Quality
}

type Quality struct {
	// How many stars out of five
	Rating int
	WaveHeight string
	Time time.Time
}

func surfReportPathToName(srp string) (string, error) {
	s := srpTailRx.ReplaceAllString(srp, "")
	// TODO: Use PathUnescape once GAE supports Go 1.8
	s, err := url.QueryUnescape(s)
	if err != nil {
		return "", fmt.Errorf("Failed to unescape path in surfReportPathToName(%q). %v", srp, err)
	}
	s = html.UnescapeString(s)
	s = s[1:] // Remove leading /
	s = strings.Replace(s, "-", " ", -1)
	return s, nil
}

func saveNewSurfReportPath(ctx context.Context, path []byte) error {
	sp := string(path)
	name, err := surfReportPathToName(sp)
	if err != nil {
		return fmt.Errorf("saveNewSurfReportPaths: %v", err)
	}
	loc := Spot{Name: name, MswPath: sp}
	key := datastore.NewKey(ctx, "Spot", string(sp), 0, nil)
	// TODO: First check whether there is already something at this key.
	_, err = datastore.Put(ctx, key, &loc)
	if err != nil {
		return fmt.Errorf("saveNewSurfReportPaths: %v", err)
	}
	return nil
}

// fetchQuality scrapes the surf report at the given url and returns a Quality struct
// summarizing the conditions.
func fetchQuality(ctx context.Context, url string) (*Quality, error) {
	log.Infof(ctx, "Fetching %s", url)
	client := urlfetch.Client(ctx)
	body, err := get(client, url)
	if err != nil {
		return nil, err
	}
	rating := countTopStars(body)
	hMatch := heightRx.FindSubmatch(body)
	if len(hMatch) != 2 {
		return nil, fmt.Errorf("Wave height regex failed.")
	}
	height := fmt.Sprintf("%s ft", hMatch[1])
	q := &Quality {
		Rating: rating,
		WaveHeight: height,
		Time: time.Now(),
	}
	return q, nil
}

// countTopStars returns the number of stars in the first rating section on the page.
func countTopStars(body []byte) int {
	starSection := starSectionRx.Find(body)
	foundStars := starRx.FindAll(starSection, -1)
	return len(foundStars)
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"net"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/validate"
)

// physical represents a LXD physical network.
type physical struct {
	common
}

// Type returns the network type.
func (n *physical) Type() string {
	return "physical"
}

// DBType returns the network type DB ID.
func (n *physical) DBType() db.NetworkType {
	return db.NetworkTypePhysical
}

// Validate network config.
func (n *physical) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"parent":                      validate.Required(validate.IsNotEmpty, validInterfaceName),
		"mtu":                         validate.Optional(validate.IsNetworkMTU),
		"vlan":                        validate.Optional(validate.IsNetworkVLAN),
		"maas.subnet.ipv4":            validate.IsAny,
		"maas.subnet.ipv6":            validate.IsAny,
		"ipv4.gateway":                validate.Optional(validate.IsNetworkAddressCIDRV4),
		"ipv6.gateway":                validate.Optional(validate.IsNetworkAddressCIDRV6),
		"ipv4.ovn.ranges":             validate.Optional(validate.IsNetworkRangeV4List),
		"ipv6.ovn.ranges":             validate.Optional(validate.IsNetworkRangeV6List),
		"ipv4.routes":                 validate.Optional(validate.IsNetworkV4List),
		"ipv6.routes":                 validate.Optional(validate.IsNetworkV6List),
		"dns.nameservers":             validate.Optional(validate.IsNetworkAddressList),
		"volatile.last_state.created": validate.Optional(validate.IsBool),
	}

	err := n.validate(config, rules)
	if err != nil {
		return err
	}

	return nil
}

// checkParentUse checks if parent is already in use by another network or instance device.
func (n *physical) checkParentUse(ourConfig map[string]string) (bool, error) {
	// Get all managed networks across all projects.
	var err error
	var projectNetworks map[string]map[int64]api.Network

	err = n.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		projectNetworks, err = tx.GetNonPendingNetworks()
		return err
	})
	if err != nil {
		return false, errors.Wrapf(err, "Failed to load all networks")
	}

	for projectName, networks := range projectNetworks {
		if projectName != project.Default {
			continue // Only default project networks can possibly reference a physical interface.
		}

		for _, network := range networks {
			if network.Name == n.name {
				continue // Ignore our own DB record.
			}

			// Check if another network is using our parent.
			if network.Config["parent"] == ourConfig["parent"] {
				// If either network doesn't specify a vlan, or both specify same vlan,
				// then we can't use this parent.
				if (network.Config["vlan"] == "" || ourConfig["vlan"] == "") || network.Config["vlan"] == ourConfig["vlan"] {
					return true, nil
				}
			}
		}
	}

	return false, nil
}

// Create checks whether the referenced parent interface is used by other networks or instance devices, as we
// need to have exclusive access to the interface.
func (n *physical) Create(clientType request.ClientType) error {
	n.logger.Debug("Create", log.Ctx{"clientType": clientType, "config": n.config})

	// We only need to check in the database once, not on every clustered node.
	if clientType == request.ClientTypeNormal {
		inUse, err := n.checkParentUse(n.config)
		if err != nil {
			return err
		}
		if inUse {
			return fmt.Errorf("Parent interface %q in use by another network", n.config["parent"])
		}
	}

	return n.common.create(clientType)
}

// Delete deletes a network.
func (n *physical) Delete(clientType request.ClientType) error {
	n.logger.Debug("Delete", log.Ctx{"clientType": clientType})

	if n.LocalStatus() == api.NetworkStatusCreated || n.LocalStatus() == api.NetworkStatusUnknown {
		err := n.Stop()
		if err != nil {
			return err
		}
	}

	return n.common.delete(clientType)
}

// Rename renames a network.
func (n *physical) Rename(newName string) error {
	n.logger.Debug("Rename", log.Ctx{"newName": newName})

	// Rename common steps.
	err := n.common.rename(newName)
	if err != nil {
		return err
	}

	return nil
}

// Start starts is a no-op.
func (n *physical) Start() error {
	n.logger.Debug("Start")

	revert := revert.New()
	defer revert.Fail()

	hostName := GetHostDevice(n.config["parent"], n.config["vlan"])
	created, err := VLANInterfaceCreate(n.config["parent"], hostName, n.config["vlan"])
	if err != nil {
		return err
	}
	if created {
		revert.Add(func() { InterfaceRemove(hostName) })
	}

	// Set the MTU.
	if n.config["mtu"] != "" {
		err = InterfaceSetMTU(hostName, n.config["mtu"])
		if err != nil {
			return err
		}
	}

	// Record if we created this device or not (if we have not already recorded that we created it previously),
	// so it can be removed on stop. This way we won't overwrite the setting on LXD restart.
	if !shared.IsTrue(n.config["volatile.last_state.created"]) {
		n.config["volatile.last_state.created"] = fmt.Sprintf("%t", created)
		err = n.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
			return tx.UpdateNetwork(n.id, n.description, n.config)
		})
		if err != nil {
			return errors.Wrapf(err, "Failed saving volatile config")
		}
	}

	revert.Success()
	return nil
}

// Stop stops is a no-op.
func (n *physical) Stop() error {
	n.logger.Debug("Stop")

	hostName := GetHostDevice(n.config["parent"], n.config["vlan"])

	// Only try and remove created VLAN interfaces.
	if n.config["vlan"] != "" && shared.IsTrue(n.config["volatile.last_state.created"]) && InterfaceExists(hostName) {
		err := InterfaceRemove(hostName)
		if err != nil {
			return err
		}
	}

	// Reset MTU back to 1500 if overridden in config.
	if n.config["mtu"] != "" && InterfaceExists(hostName) {
		err := InterfaceSetMTU(hostName, "1500")
		if err != nil {
			return err
		}
	}

	// Remove last state config.
	delete(n.config, "volatile.last_state.created")
	err := n.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		return tx.UpdateNetwork(n.id, n.description, n.config)
	})
	if err != nil {
		return errors.Wrapf(err, "Failed removing volatile config")
	}

	return nil
}

// Update updates the network. Accepts notification boolean indicating if this update request is coming from a
// cluster notification, in which case do not update the database, just apply local changes needed.
func (n *physical) Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error {
	n.logger.Debug("Update", log.Ctx{"clientType": clientType, "newNetwork": newNetwork})

	dbUpdateNeeeded, changedKeys, oldNetwork, err := n.common.configChanged(newNetwork)
	if err != nil {
		return err
	}

	if !dbUpdateNeeeded {
		return nil // Nothing changed.
	}

	if n.LocalStatus() == api.NetworkStatusPending {
		// Apply DB change to local node only.
		return n.common.update(newNetwork, targetNode, clientType)
	}

	revert := revert.New()
	defer revert.Fail()

	hostNameChanged := shared.StringInSlice("vlan", changedKeys) || shared.StringInSlice("parent", changedKeys)

	// We only need to check in the database once, not on every clustered node.
	if clientType == request.ClientTypeNormal {
		if hostNameChanged {
			isUsed, err := n.IsUsed()
			if isUsed || err != nil {
				return fmt.Errorf("Cannot update network host name when in use")
			}

			inUse, err := n.checkParentUse(newNetwork.Config)
			if err != nil {
				return err
			}
			if inUse {
				return fmt.Errorf("Parent interface %q in use by another network", newNetwork.Config["parent"])
			}
		}
	}

	if hostNameChanged {
		err = n.Stop()
		if err != nil {
			return err
		}

		// Remove the volatile last state from submitted new config if present.
		delete(newNetwork.Config, "volatile.last_state.created")
	}

	// Define a function which reverts everything.
	revert.Add(func() {
		// Reset changes to all nodes and database.
		n.common.update(oldNetwork, targetNode, clientType)
	})

	// Apply changes to all nodes and databse.
	err = n.common.update(newNetwork, targetNode, clientType)
	if err != nil {
		return err
	}

	err = n.Start()
	if err != nil {
		return err
	}

	revert.Success()
	return nil
}

// DHCPv4Subnet returns the DHCPv4 subnet (if DHCP is enabled on network).
func (n *physical) DHCPv4Subnet() *net.IPNet {
	_, subnet, err := net.ParseCIDR(n.config["ipv4.gateway"])
	if err != nil {
		return nil
	}

	return subnet
}

// DHCPv6Subnet returns the DHCPv6 subnet (if DHCP or SLAAC is enabled on network).
func (n *physical) DHCPv6Subnet() *net.IPNet {
	_, subnet, err := net.ParseCIDR(n.config["ipv6.gateway"])
	if err != nil {
		return nil
	}

	return subnet
}
<commit_msg>lxd/network/driver/physical: Adds ovn.ingress_mode config key<commit_after>package network

import (
	"fmt"
	"net"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/validate"
)

// physical represents a LXD physical network.
type physical struct {
	common
}

// Type returns the network type.
func (n *physical) Type() string {
	return "physical"
}

// DBType returns the network type DB ID.
func (n *physical) DBType() db.NetworkType {
	return db.NetworkTypePhysical
}

// Validate network config.
func (n *physical) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"parent":           validate.Required(validate.IsNotEmpty, validInterfaceName),
		"mtu":              validate.Optional(validate.IsNetworkMTU),
		"vlan":             validate.Optional(validate.IsNetworkVLAN),
		"maas.subnet.ipv4": validate.IsAny,
		"maas.subnet.ipv6": validate.IsAny,
		"ipv4.gateway":     validate.Optional(validate.IsNetworkAddressCIDRV4),
		"ipv6.gateway":     validate.Optional(validate.IsNetworkAddressCIDRV6),
		"ipv4.ovn.ranges":  validate.Optional(validate.IsNetworkRangeV4List),
		"ipv6.ovn.ranges":  validate.Optional(validate.IsNetworkRangeV6List),
		"ipv4.routes":      validate.Optional(validate.IsNetworkV4List),
		"ipv6.routes":      validate.Optional(validate.IsNetworkV6List),
		"dns.nameservers":  validate.Optional(validate.IsNetworkAddressList),
		"ovn.ingress_mode": validate.Optional(func(value string) error {
			return validate.IsOneOf(value, []string{"l2proxy", "routed"})
		}),
		"volatile.last_state.created": validate.Optional(validate.IsBool),
	}

	err := n.validate(config, rules)
	if err != nil {
		return err
	}

	return nil
}

// checkParentUse checks if parent is already in use by another network or instance device.
func (n *physical) checkParentUse(ourConfig map[string]string) (bool, error) {
	// Get all managed networks across all projects.
	var err error
	var projectNetworks map[string]map[int64]api.Network

	err = n.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		projectNetworks, err = tx.GetNonPendingNetworks()
		return err
	})
	if err != nil {
		return false, errors.Wrapf(err, "Failed to load all networks")
	}

	for projectName, networks := range projectNetworks {
		if projectName != project.Default {
			continue // Only default project networks can possibly reference a physical interface.
		}

		for _, network := range networks {
			if network.Name == n.name {
				continue // Ignore our own DB record.
			}

			// Check if another network is using our parent.
			if network.Config["parent"] == ourConfig["parent"] {
				// If either network doesn't specify a vlan, or both specify same vlan,
				// then we can't use this parent.
				if (network.Config["vlan"] == "" || ourConfig["vlan"] == "") || network.Config["vlan"] == ourConfig["vlan"] {
					return true, nil
				}
			}
		}
	}

	return false, nil
}

// Create checks whether the referenced parent interface is used by other networks or instance devices, as we
// need to have exclusive access to the interface.
func (n *physical) Create(clientType request.ClientType) error {
	n.logger.Debug("Create", log.Ctx{"clientType": clientType, "config": n.config})

	// We only need to check in the database once, not on every clustered node.
	if clientType == request.ClientTypeNormal {
		inUse, err := n.checkParentUse(n.config)
		if err != nil {
			return err
		}
		if inUse {
			return fmt.Errorf("Parent interface %q in use by another network", n.config["parent"])
		}
	}

	return n.common.create(clientType)
}

// Delete deletes a network.
func (n *physical) Delete(clientType request.ClientType) error {
	n.logger.Debug("Delete", log.Ctx{"clientType": clientType})

	if n.LocalStatus() == api.NetworkStatusCreated || n.LocalStatus() == api.NetworkStatusUnknown {
		err := n.Stop()
		if err != nil {
			return err
		}
	}

	return n.common.delete(clientType)
}

// Rename renames a network.
func (n *physical) Rename(newName string) error {
	n.logger.Debug("Rename", log.Ctx{"newName": newName})

	// Rename common steps.
	err := n.common.rename(newName)
	if err != nil {
		return err
	}

	return nil
}

// Start starts is a no-op.
func (n *physical) Start() error {
	n.logger.Debug("Start")

	revert := revert.New()
	defer revert.Fail()

	hostName := GetHostDevice(n.config["parent"], n.config["vlan"])
	created, err := VLANInterfaceCreate(n.config["parent"], hostName, n.config["vlan"])
	if err != nil {
		return err
	}
	if created {
		revert.Add(func() { InterfaceRemove(hostName) })
	}

	// Set the MTU.
	if n.config["mtu"] != "" {
		err = InterfaceSetMTU(hostName, n.config["mtu"])
		if err != nil {
			return err
		}
	}

	// Record if we created this device or not (if we have not already recorded that we created it previously),
	// so it can be removed on stop. This way we won't overwrite the setting on LXD restart.
	if !shared.IsTrue(n.config["volatile.last_state.created"]) {
		n.config["volatile.last_state.created"] = fmt.Sprintf("%t", created)
		err = n.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
			return tx.UpdateNetwork(n.id, n.description, n.config)
		})
		if err != nil {
			return errors.Wrapf(err, "Failed saving volatile config")
		}
	}

	revert.Success()
	return nil
}

// Stop stops is a no-op.
func (n *physical) Stop() error {
	n.logger.Debug("Stop")

	hostName := GetHostDevice(n.config["parent"], n.config["vlan"])

	// Only try and remove created VLAN interfaces.
	if n.config["vlan"] != "" && shared.IsTrue(n.config["volatile.last_state.created"]) && InterfaceExists(hostName) {
		err := InterfaceRemove(hostName)
		if err != nil {
			return err
		}
	}

	// Reset MTU back to 1500 if overridden in config.
	if n.config["mtu"] != "" && InterfaceExists(hostName) {
		err := InterfaceSetMTU(hostName, "1500")
		if err != nil {
			return err
		}
	}

	// Remove last state config.
	delete(n.config, "volatile.last_state.created")
	err := n.state.Cluster.Transaction(func(tx *db.ClusterTx) error {
		return tx.UpdateNetwork(n.id, n.description, n.config)
	})
	if err != nil {
		return errors.Wrapf(err, "Failed removing volatile config")
	}

	return nil
}

// Update updates the network. Accepts notification boolean indicating if this update request is coming from a
// cluster notification, in which case do not update the database, just apply local changes needed.
func (n *physical) Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error {
	n.logger.Debug("Update", log.Ctx{"clientType": clientType, "newNetwork": newNetwork})

	dbUpdateNeeeded, changedKeys, oldNetwork, err := n.common.configChanged(newNetwork)
	if err != nil {
		return err
	}

	if !dbUpdateNeeeded {
		return nil // Nothing changed.
	}

	if n.LocalStatus() == api.NetworkStatusPending {
		// Apply DB change to local node only.
		return n.common.update(newNetwork, targetNode, clientType)
	}

	revert := revert.New()
	defer revert.Fail()

	hostNameChanged := shared.StringInSlice("vlan", changedKeys) || shared.StringInSlice("parent", changedKeys)

	// We only need to check in the database once, not on every clustered node.
	if clientType == request.ClientTypeNormal {
		if hostNameChanged {
			isUsed, err := n.IsUsed()
			if isUsed || err != nil {
				return fmt.Errorf("Cannot update network host name when in use")
			}

			inUse, err := n.checkParentUse(newNetwork.Config)
			if err != nil {
				return err
			}
			if inUse {
				return fmt.Errorf("Parent interface %q in use by another network", newNetwork.Config["parent"])
			}
		}
	}

	if hostNameChanged {
		err = n.Stop()
		if err != nil {
			return err
		}

		// Remove the volatile last state from submitted new config if present.
		delete(newNetwork.Config, "volatile.last_state.created")
	}

	// Define a function which reverts everything.
	revert.Add(func() {
		// Reset changes to all nodes and database.
		n.common.update(oldNetwork, targetNode, clientType)
	})

	// Apply changes to all nodes and databse.
	err = n.common.update(newNetwork, targetNode, clientType)
	if err != nil {
		return err
	}

	err = n.Start()
	if err != nil {
		return err
	}

	revert.Success()
	return nil
}

// DHCPv4Subnet returns the DHCPv4 subnet (if DHCP is enabled on network).
func (n *physical) DHCPv4Subnet() *net.IPNet {
	_, subnet, err := net.ParseCIDR(n.config["ipv4.gateway"])
	if err != nil {
		return nil
	}

	return subnet
}

// DHCPv6Subnet returns the DHCPv6 subnet (if DHCP or SLAAC is enabled on network).
func (n *physical) DHCPv6Subnet() *net.IPNet {
	_, subnet, err := net.ParseCIDR(n.config["ipv6.gateway"])
	if err != nil {
		return nil
	}

	return subnet
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package header

import (
	"encoding/binary"

	"github.com/google/netstack/tcpip"
)

const (
	versIHL  = 0
	tos      = 1
	totalLen = 2
	id       = 4
	flagsFO  = 6
	ttl      = 8
	protocol = 9
	checksum = 10
	srcAddr  = 12
	dstAddr  = 16
)

// IPv4Fields contains the fields of an IPv4 packet. It is used to describe the
// fields of a packet that needs to be encoded.
type IPv4Fields struct {
	// IHL is the "internet header length" field of an IPv4 packet.
	IHL uint8

	// TOS is the "type of service" field of an IPv4 packet.
	TOS uint8

	// TotalLength is the "total length" field of an IPv4 packet.
	TotalLength uint16

	// ID is the "identification" field of an IPv4 packet.
	ID uint16

	// Flags is the "flags" field of an IPv4 packet.
	Flags uint8

	// FragmentOffset is the "fragment offset" field of an IPv4 packet.
	FragmentOffset uint16

	// TTL is the "time to live" field of an IPv4 packet.
	TTL uint8

	// Protocol is the "protocol" field of an IPv4 packet.
	Protocol uint8

	// Checksum is the "checksum" field of an IPv4 packet.
	Checksum uint16

	// SrcAddr is the "source ip address" of an IPv4 packet.
	SrcAddr tcpip.Address

	// DstAddr is the "destination ip address" of an IPv4 packet.
	DstAddr tcpip.Address
}

// IPv4 represents an ipv4 header stored in a byte array.
// Most of the methods of IPv4 access to the underlying slice without
// checking the boundaries and could panic because of 'index out of range'.
// Always call IsValid() to validate an instance of IPv4 before using other methods.
type IPv4 []byte

const (
	// IPv4MinimumSize is the minimum size of a valid IPv4 packet.
	IPv4MinimumSize = 20

	// IPv4MaximumHeaderSize is the maximum size of an IPv4 header. Given
	// that there are only 4 bits to represents the header length in 32-bit
	// units, the header cannot exceed 15*4 = 60 bytes.
	IPv4MaximumHeaderSize = 60

	// IPv4AddressSize is the size, in bytes, of an IPv4 address.
	IPv4AddressSize = 4

	// IPv4ProtocolNumber is IPv4's network protocol number.
	IPv4ProtocolNumber tcpip.NetworkProtocolNumber = 0x0800

	// IPv4Version is the version of the ipv4 protocol.
	IPv4Version = 4

	// IPv4Broadcast is the broadcast address of the IPv4 procotol.
	IPv4Broadcast tcpip.Address = "\xff\xff\xff\xff"

	// IPv4Any is the non-routable IPv4 "any" meta address.
	IPv4Any tcpip.Address = "\x00\x00\x00\x00"
)

// Flags that may be set in an IPv4 packet.
const (
	IPv4FlagMoreFragments = 1 << iota
	IPv4FlagDontFragment
)

// IPVersion returns the version of IP used in the given packet. It returns -1
// if the packet is not large enough to contain the version field.
func IPVersion(b []byte) int {
	// Length must be at least offset+length of version field.
	if len(b) < versIHL+1 {
		return -1
	}
	return int(b[versIHL] >> 4)
}

// HeaderLength returns the value of the "header length" field of the ipv4
// header.
func (b IPv4) HeaderLength() uint8 {
	return (b[versIHL] & 0xf) * 4
}

// ID returns the value of the identifier field of the ipv4 header.
func (b IPv4) ID() uint16 {
	return binary.BigEndian.Uint16(b[id:])
}

// Protocol returns the value of the protocol field of the ipv4 header.
func (b IPv4) Protocol() uint8 {
	return b[protocol]
}

// Flags returns the "flags" field of the ipv4 header.
func (b IPv4) Flags() uint8 {
	return uint8(binary.BigEndian.Uint16(b[flagsFO:]) >> 13)
}

// TTL returns the "TTL" field of the ipv4 header.
func (b IPv4) TTL() uint8 {
	return b[ttl]
}

// FragmentOffset returns the "fragment offset" field of the ipv4 header.
func (b IPv4) FragmentOffset() uint16 {
	return binary.BigEndian.Uint16(b[flagsFO:]) << 3
}

// TotalLength returns the "total length" field of the ipv4 header.
func (b IPv4) TotalLength() uint16 {
	return binary.BigEndian.Uint16(b[totalLen:])
}

// Checksum returns the checksum field of the ipv4 header.
func (b IPv4) Checksum() uint16 {
	return binary.BigEndian.Uint16(b[checksum:])
}

// SourceAddress returns the "source address" field of the ipv4 header.
func (b IPv4) SourceAddress() tcpip.Address {
	return tcpip.Address(b[srcAddr : srcAddr+IPv4AddressSize])
}

// DestinationAddress returns the "destination address" field of the ipv4
// header.
func (b IPv4) DestinationAddress() tcpip.Address {
	return tcpip.Address(b[dstAddr : dstAddr+IPv4AddressSize])
}

// TransportProtocol implements Network.TransportProtocol.
func (b IPv4) TransportProtocol() tcpip.TransportProtocolNumber {
	return tcpip.TransportProtocolNumber(b.Protocol())
}

// Payload implements Network.Payload.
func (b IPv4) Payload() []byte {
	return b[b.HeaderLength():][:b.PayloadLength()]
}

// PayloadLength returns the length of the payload portion of the ipv4 packet.
func (b IPv4) PayloadLength() uint16 {
	return b.TotalLength() - uint16(b.HeaderLength())
}

// TOS returns the "type of service" field of the ipv4 header.
func (b IPv4) TOS() (uint8, uint32) {
	return b[tos], 0
}

// SetTOS sets the "type of service" field of the ipv4 header.
func (b IPv4) SetTOS(v uint8, _ uint32) {
	b[tos] = v
}

// SetTotalLength sets the "total length" field of the ipv4 header.
func (b IPv4) SetTotalLength(totalLength uint16) {
	binary.BigEndian.PutUint16(b[totalLen:], totalLength)
}

// SetChecksum sets the checksum field of the ipv4 header.
func (b IPv4) SetChecksum(v uint16) {
	binary.BigEndian.PutUint16(b[checksum:], v)
}

// SetFlagsFragmentOffset sets the "flags" and "fragment offset" fields of the
// ipv4 header.
func (b IPv4) SetFlagsFragmentOffset(flags uint8, offset uint16) {
	v := (uint16(flags) << 13) | (offset >> 3)
	binary.BigEndian.PutUint16(b[flagsFO:], v)
}

// SetSourceAddress sets the "source address" field of the ipv4 header.
func (b IPv4) SetSourceAddress(addr tcpip.Address) {
	copy(b[srcAddr:srcAddr+IPv4AddressSize], addr)
}

// SetDestinationAddress sets the "destination address" field of the ipv4
// header.
func (b IPv4) SetDestinationAddress(addr tcpip.Address) {
	copy(b[dstAddr:dstAddr+IPv4AddressSize], addr)
}

// CalculateChecksum calculates the checksum of the ipv4 header.
func (b IPv4) CalculateChecksum() uint16 {
	return Checksum(b[:b.HeaderLength()], 0)
}

// Encode encodes all the fields of the ipv4 header.
func (b IPv4) Encode(i *IPv4Fields) {
	b[versIHL] = (4 << 4) | ((i.IHL / 4) & 0xf)
	b[tos] = i.TOS
	b.SetTotalLength(i.TotalLength)
	binary.BigEndian.PutUint16(b[id:], i.ID)
	b.SetFlagsFragmentOffset(i.Flags, i.FragmentOffset)
	b[ttl] = i.TTL
	b[protocol] = i.Protocol
	b.SetChecksum(i.Checksum)
	copy(b[srcAddr:srcAddr+IPv4AddressSize], i.SrcAddr)
	copy(b[dstAddr:dstAddr+IPv4AddressSize], i.DstAddr)
}

// EncodePartial updates the total length and checksum fields of ipv4 header,
// taking in the partial checksum, which is the checksum of the header without
// the total length and checksum fields. It is useful in cases when similar
// packets are produced.
func (b IPv4) EncodePartial(partialChecksum, totalLength uint16) {
	b.SetTotalLength(totalLength)
	checksum := Checksum(b[totalLen:totalLen+2], partialChecksum)
	b.SetChecksum(^checksum)
}

// IsValid performs basic validation on the packet.
func (b IPv4) IsValid(pktSize int) bool {
	if len(b) < IPv4MinimumSize {
		return false
	}

	hlen := int(b.HeaderLength())
	tlen := int(b.TotalLength())
	if hlen > tlen || tlen > pktSize {
		return false
	}

	return true
}

// IsV4MulticastAddress determines if the provided address is an IPv4 multicast
// address (range 224.0.0.0 to 239.255.255.255). The four most significant bits
// will be 1110 = 0xe0.
func IsV4MulticastAddress(addr tcpip.Address) bool {
	if len(addr) != IPv4AddressSize {
		return false
	}
	return (addr[0] & 0xf0) == 0xe0
}
<commit_msg>Internal change.<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package header

import (
	"encoding/binary"

	"github.com/google/netstack/tcpip"
)

const (
	versIHL  = 0
	tos      = 1
	totalLen = 2
	id       = 4
	flagsFO  = 6
	ttl      = 8
	protocol = 9
	checksum = 10
	srcAddr  = 12
	dstAddr  = 16
)

// IPv4Fields contains the fields of an IPv4 packet. It is used to describe the
// fields of a packet that needs to be encoded.
type IPv4Fields struct {
	// IHL is the "internet header length" field of an IPv4 packet.
	IHL uint8

	// TOS is the "type of service" field of an IPv4 packet.
	TOS uint8

	// TotalLength is the "total length" field of an IPv4 packet.
	TotalLength uint16

	// ID is the "identification" field of an IPv4 packet.
	ID uint16

	// Flags is the "flags" field of an IPv4 packet.
	Flags uint8

	// FragmentOffset is the "fragment offset" field of an IPv4 packet.
	FragmentOffset uint16

	// TTL is the "time to live" field of an IPv4 packet.
	TTL uint8

	// Protocol is the "protocol" field of an IPv4 packet.
	Protocol uint8

	// Checksum is the "checksum" field of an IPv4 packet.
	Checksum uint16

	// SrcAddr is the "source ip address" of an IPv4 packet.
	SrcAddr tcpip.Address

	// DstAddr is the "destination ip address" of an IPv4 packet.
	DstAddr tcpip.Address
}

// IPv4 represents an ipv4 header stored in a byte array.
// Most of the methods of IPv4 access to the underlying slice without
// checking the boundaries and could panic because of 'index out of range'.
// Always call IsValid() to validate an instance of IPv4 before using other methods.
type IPv4 []byte

const (
	// IPv4MinimumSize is the minimum size of a valid IPv4 packet.
	IPv4MinimumSize = 20

	// IPv4MaximumHeaderSize is the maximum size of an IPv4 header. Given
	// that there are only 4 bits to represents the header length in 32-bit
	// units, the header cannot exceed 15*4 = 60 bytes.
	IPv4MaximumHeaderSize = 60

	// IPv4AddressSize is the size, in bytes, of an IPv4 address.
	IPv4AddressSize = 4

	// IPv4ProtocolNumber is IPv4's network protocol number.
	IPv4ProtocolNumber tcpip.NetworkProtocolNumber = 0x0800

	// IPv4Version is the version of the ipv4 protocol.
	IPv4Version = 4

	// IPv4Broadcast is the broadcast address of the IPv4 procotol.
	IPv4Broadcast tcpip.Address = "\xff\xff\xff\xff"

	// IPv4Any is the non-routable IPv4 "any" meta address.
	IPv4Any tcpip.Address = "\x00\x00\x00\x00"
)

// Flags that may be set in an IPv4 packet.
const (
	IPv4FlagMoreFragments = 1 << iota
	IPv4FlagDontFragment
)

// IPVersion returns the version of IP used in the given packet. It returns -1
// if the packet is not large enough to contain the version field.
func IPVersion(b []byte) int {
	// Length must be at least offset+length of version field.
	if len(b) < versIHL+1 {
		return -1
	}
	return int(b[versIHL] >> 4)
}

// HeaderLength returns the value of the "header length" field of the ipv4
// header.
func (b IPv4) HeaderLength() uint8 {
	return (b[versIHL] & 0xf) * 4
}

// ID returns the value of the identifier field of the ipv4 header.
func (b IPv4) ID() uint16 {
	return binary.BigEndian.Uint16(b[id:])
}

// Protocol returns the value of the protocol field of the ipv4 header.
func (b IPv4) Protocol() uint8 {
	return b[protocol]
}

// Flags returns the "flags" field of the ipv4 header.
func (b IPv4) Flags() uint8 {
	return uint8(binary.BigEndian.Uint16(b[flagsFO:]) >> 13)
}

// TTL returns the "TTL" field of the ipv4 header.
func (b IPv4) TTL() uint8 {
	return b[ttl]
}

// FragmentOffset returns the "fragment offset" field of the ipv4 header.
func (b IPv4) FragmentOffset() uint16 {
	return binary.BigEndian.Uint16(b[flagsFO:]) << 3
}

// TotalLength returns the "total length" field of the ipv4 header.
func (b IPv4) TotalLength() uint16 {
	return binary.BigEndian.Uint16(b[totalLen:])
}

// Checksum returns the checksum field of the ipv4 header.
func (b IPv4) Checksum() uint16 {
	return binary.BigEndian.Uint16(b[checksum:])
}

// SourceAddress returns the "source address" field of the ipv4 header.
func (b IPv4) SourceAddress() tcpip.Address {
	return tcpip.Address(b[srcAddr : srcAddr+IPv4AddressSize])
}

// DestinationAddress returns the "destination address" field of the ipv4
// header.
func (b IPv4) DestinationAddress() tcpip.Address {
	return tcpip.Address(b[dstAddr : dstAddr+IPv4AddressSize])
}

// TransportProtocol implements Network.TransportProtocol.
func (b IPv4) TransportProtocol() tcpip.TransportProtocolNumber {
	return tcpip.TransportProtocolNumber(b.Protocol())
}

// Payload implements Network.Payload.
func (b IPv4) Payload() []byte {
	return b[b.HeaderLength():][:b.PayloadLength()]
}

// PayloadLength returns the length of the payload portion of the ipv4 packet.
func (b IPv4) PayloadLength() uint16 {
	return b.TotalLength() - uint16(b.HeaderLength())
}

// TOS returns the "type of service" field of the ipv4 header.
func (b IPv4) TOS() (uint8, uint32) {
	return b[tos], 0
}

// SetTOS sets the "type of service" field of the ipv4 header.
func (b IPv4) SetTOS(v uint8, _ uint32) {
	b[tos] = v
}

// SetTotalLength sets the "total length" field of the ipv4 header.
func (b IPv4) SetTotalLength(totalLength uint16) {
	binary.BigEndian.PutUint16(b[totalLen:], totalLength)
}

// SetChecksum sets the checksum field of the ipv4 header.
func (b IPv4) SetChecksum(v uint16) {
	binary.BigEndian.PutUint16(b[checksum:], v)
}

// SetFlagsFragmentOffset sets the "flags" and "fragment offset" fields of the
// ipv4 header.
func (b IPv4) SetFlagsFragmentOffset(flags uint8, offset uint16) {
	v := (uint16(flags) << 13) | (offset >> 3)
	binary.BigEndian.PutUint16(b[flagsFO:], v)
}

// SetID sets the identification field.
func (b IPv4) SetID(v uint16) {
	binary.BigEndian.PutUint16(b[id:], v)
}

// SetSourceAddress sets the "source address" field of the ipv4 header.
func (b IPv4) SetSourceAddress(addr tcpip.Address) {
	copy(b[srcAddr:srcAddr+IPv4AddressSize], addr)
}

// SetDestinationAddress sets the "destination address" field of the ipv4
// header.
func (b IPv4) SetDestinationAddress(addr tcpip.Address) {
	copy(b[dstAddr:dstAddr+IPv4AddressSize], addr)
}

// CalculateChecksum calculates the checksum of the ipv4 header.
func (b IPv4) CalculateChecksum() uint16 {
	return Checksum(b[:b.HeaderLength()], 0)
}

// Encode encodes all the fields of the ipv4 header.
func (b IPv4) Encode(i *IPv4Fields) {
	b[versIHL] = (4 << 4) | ((i.IHL / 4) & 0xf)
	b[tos] = i.TOS
	b.SetTotalLength(i.TotalLength)
	binary.BigEndian.PutUint16(b[id:], i.ID)
	b.SetFlagsFragmentOffset(i.Flags, i.FragmentOffset)
	b[ttl] = i.TTL
	b[protocol] = i.Protocol
	b.SetChecksum(i.Checksum)
	copy(b[srcAddr:srcAddr+IPv4AddressSize], i.SrcAddr)
	copy(b[dstAddr:dstAddr+IPv4AddressSize], i.DstAddr)
}

// EncodePartial updates the total length and checksum fields of ipv4 header,
// taking in the partial checksum, which is the checksum of the header without
// the total length and checksum fields. It is useful in cases when similar
// packets are produced.
func (b IPv4) EncodePartial(partialChecksum, totalLength uint16) {
	b.SetTotalLength(totalLength)
	checksum := Checksum(b[totalLen:totalLen+2], partialChecksum)
	b.SetChecksum(^checksum)
}

// IsValid performs basic validation on the packet.
func (b IPv4) IsValid(pktSize int) bool {
	if len(b) < IPv4MinimumSize {
		return false
	}

	hlen := int(b.HeaderLength())
	tlen := int(b.TotalLength())
	if hlen > tlen || tlen > pktSize {
		return false
	}

	return true
}

// IsV4MulticastAddress determines if the provided address is an IPv4 multicast
// address (range 224.0.0.0 to 239.255.255.255). The four most significant bits
// will be 1110 = 0xe0.
func IsV4MulticastAddress(addr tcpip.Address) bool {
	if len(addr) != IPv4AddressSize {
		return false
	}
	return (addr[0] & 0xf0) == 0xe0
}
<|endoftext|>


<commit_before>package quiclatest

import (
	"sync"

	"github.com/ami-GS/gQUIC/latest/qtype"
	"github.com/ami-GS/gQUIC/latest/utils"
)

type signedChannel struct {
	ch     chan struct{}
	closed bool
}

type StreamManager struct {
	streamMap map[qtype.StreamID]Stream
	// would be RcvStream or SendRcvStream
	finishedStreams    *utils.RingBuffer
	maxStreamIDUni     qtype.StreamID
	nxtSendStreamIDUni qtype.StreamID
	maxStreamIDBidi    qtype.StreamID
	nxtStreamIDBidi    qtype.StreamID
	sess               *Session
	// TODO: name should be considered
	blockedIDs      map[qtype.StreamID]*signedChannel
	blockedIDsMutex *sync.Mutex

	waitReadingChs *utils.RingBuffer
}

func NewStreamManager(sess *Session) *StreamManager {
	var nxtUniID, nxtBidiID qtype.StreamID
	if sess.isClient {
		nxtUniID = 2
		nxtBidiID = 0
	} else {
		nxtUniID = 3
		nxtBidiID = 1
	}

	return &StreamManager{
		streamMap:       make(map[qtype.StreamID]Stream),
		sess:            sess,
		finishedStreams: utils.NewRingBuffer(20),
		// may be set after handshake, or MAX_STREAM_ID frame
		maxStreamIDUni:     1,
		nxtSendStreamIDUni: nxtUniID,
		maxStreamIDBidi:    100,
		nxtStreamIDBidi:    nxtBidiID,
		blockedIDs:         make(map[qtype.StreamID]*signedChannel),
		blockedIDsMutex:    new(sync.Mutex),
		// TODO: should be big enough and be able to configurable
		waitReadingChs: utils.NewRingBuffer(30),
	}
}

func (s *StreamManager) IsValidID(streamID qtype.StreamID) error {
	if streamID&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		// unidirectional
		if streamID > s.maxStreamIDUni {
			return qtype.StreamIDError
		}
	} else {
		// bidirectional
		if streamID > s.maxStreamIDBidi {
			return qtype.StreamIDError
		}
	}
	return nil
}

func (s *StreamManager) StartNewSendStream() (Stream, error) {
	targetID := s.nxtSendStreamIDUni
	// TODO: atmic increment?
	s.nxtSendStreamIDUni.Increment()

	if targetID > s.maxStreamIDUni {
		blockedChan := &signedChannel{make(chan struct{}), false}
		s.blockedIDsMutex.Lock()
		s.blockedIDs[targetID] = blockedChan
		s.blockedIDsMutex.Unlock()
		s.sess.sendFrameChan <- NewStreamIDBlockedFrame(targetID)

		<-blockedChan.ch
		delete(s.blockedIDs, targetID)
		close(blockedChan.ch)
	}

	stream, _, err := s.GetOrNewStream(targetID, true)
	if err != nil {
		return nil, err
	}
	return stream, nil
}

func (s *StreamManager) GetOrNewStream(streamID qtype.StreamID, send bool) (st Stream, isNew bool, err error) {
	if streamID == 0 {
		return s.getOrNewBidiStream(streamID, s.sess)
	}

	if streamID&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		st, isNew, err = s.getOrNewUniStream(streamID, s.sess, send)
	} else {
		st, isNew, err = s.getOrNewBidiStream(streamID, s.sess)
	}

	if err != nil {
		delete(s.streamMap, streamID)
		return nil, false, err
	}

	if isNew {
		// check whether ID is larger than MAX_STREAM_ID
		err = s.IsValidID(streamID)
		if err != nil {
			delete(s.streamMap, streamID)
			return nil, false, err
		}
	}

	return st, isNew, err
}

func (s *StreamManager) getOrNewUniStream(streamID qtype.StreamID, sess *Session, send bool) (Stream, bool, error) {
	stream, ok := s.streamMap[streamID]
	// get
	if ok {
		if send {
			st, ok := stream.(*SendStream)
			if !ok {
				// TODO: what error?
				return nil, false, nil
			}
			return st, false, nil
		}
		st, ok := stream.(*RecvStream)
		if !ok {
			// TODO: what error?
			return nil, false, nil
		}
		return st, false, nil

	}
	// new
	if send {
		st := newSendStream(streamID, sess)
		s.streamMap[streamID] = st
		return st, true, nil
	}
	st := newRecvStream(streamID, sess)
	s.streamMap[streamID] = st
	return st, true, nil
}

func (s *StreamManager) getOrNewBidiStream(streamID qtype.StreamID, sess *Session) (*SendRecvStream, bool, error) {
	sidVal := streamID
	stream, ok := s.streamMap[streamID]
	if ok {
		st, ok := stream.(*SendRecvStream)
		if !ok {
			// TODO: what error?
			return nil, false, nil
		}
		return st, false, nil
	}
	st := newSendRecvStream(streamID, sess)
	s.streamMap[sidVal] = st
	return st, true, nil
}

// called from session.QueueFrame(f)
func (s *StreamManager) QueueFrame(f StreamLevelFrame) error {
	sid := f.GetStreamID()
	var stream Stream
	var err error
	//var isNew bool
	switch f.(type) {
	case *StreamFrame, *RstStreamFrame, *StreamBlockedFrame:
		stream, _, err = s.GetOrNewStream(sid, true)
	case *MaxStreamDataFrame, *StopSendingFrame:
		stream, _, err = s.GetOrNewStream(sid, false)
	case *MaxStreamIDFrame:
		// this is special, affect only stream_manager
		//s.sess.sendFrameChan <- f
		return nil
	case *StreamIDBlockedFrame:
		// ??
		//s.sess.sendFrameChan <- f
		return nil
	default:
		// error, but impossible to reach here
		return nil
	}
	if err != nil {
		return err
	}
	err = stream.QueueFrame(f)
	if err != nil {
		return err
	}
	if stream.IsTerminated() {
		stream.UpdateConnectionByteSent()
	}
	return nil
}

func (s *StreamManager) handleFrame(f StreamLevelFrame) error {
	sid := f.GetStreamID()
	protocolViolationFunc := func() error {
		// An endpoint that receives a MAX_STREAM_DATA frame for a send-only
		// stream it has not opened MUST terminate the connection with error
		// PROTOCOL_VIOLATION.
		delete(s.streamMap, sid)
		return qtype.ProtocolViolation
	}

	var stream Stream
	var err error
	var isNew bool
	switch frame := f.(type) {
	case *MaxStreamIDFrame:
		return s.handleMaxStreamIDFrame(frame)
	case *StreamIDBlockedFrame:
		return s.handleStreamIDBlockedFrame(frame)
	case *StreamFrame:
		stream, isNew, err = s.GetOrNewStream(sid, false)
		if err != nil {
			return err
		}
		err = stream.handleStreamFrame(frame)
	case *RstStreamFrame:
		stream, _, err = s.GetOrNewStream(sid, false)
		if err != nil {
			return err
		}
		err = stream.handleRstStreamFrame(frame)
	case *StreamBlockedFrame:
		stream, isNew, err = s.GetOrNewStream(sid, false)
		if err != nil {
			return err
		}
		err = stream.handleStreamBlockedFrame(frame)
	case *MaxStreamDataFrame:
		stream, isNew, err = s.GetOrNewStream(sid, true)
		if err != nil {
			return err
		}
		if isNew {
			return protocolViolationFunc()
		}
		err = stream.handleMaxStreamDataFrame(frame)
		return err
	case *StopSendingFrame:
		stream, isNew, err = s.GetOrNewStream(sid, true)
		if err != nil {
			return err
		}
		if isNew {
			return protocolViolationFunc()
		}
		err = stream.handleStopSendingFrame(frame)
	default:
		// error, but impossible to reach here
		return nil
	}

	if err != nil {
		return err
	}

	if stream.IsTerminated() {
		stream.UpdateConnectionByteReceived()
	}
	return err
}

func (s *StreamManager) handleStreamIDBlockedFrame(frame *StreamIDBlockedFrame) error {
	// should be from sender stream which needs new ID, but could not open due to limit

	if frame.StreamID&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		s.maxStreamIDUni = frame.StreamID
	} else {
		s.maxStreamIDBidi = frame.StreamID
	}

	s.sess.sendFrameChan <- NewMaxStreamIDFrame(frame.StreamID)
	return nil
}
func (s *StreamManager) handleMaxStreamIDFrame(frame *MaxStreamIDFrame) error {
	sid := frame.StreamID

	if sid&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		// unidirectional
		if sid < s.maxStreamIDUni {
			// ignored
			return nil
		}
		s.maxStreamIDUni = frame.StreamID
	} else {
		//bidirectional
		if sid < s.maxStreamIDBidi {
			// ignored
			return nil
		}
		s.maxStreamIDBidi = frame.StreamID
	}
	s.blockedIDsMutex.Lock()
	for blockedID, val := range s.blockedIDs {
		if !val.closed && sid >= blockedID {
			val.closed = true
			val.ch <- struct{}{}
		}
	}
	s.blockedIDsMutex.Unlock()
	return nil
}

func (s *StreamManager) resendBlockedFrames(blockedFrames *utils.RingBuffer) error {

	var stream Stream
	var isNew bool
	var err error
	sID := qtype.StreamID(0)
	size := blockedFrames.Size()
	for i := 0; i < size; i++ {
		frame := blockedFrames.Dequeue().(*StreamFrame)
		if sID != frame.GetStreamID() {
			sID = frame.GetStreamID()
			stream, isNew, err = s.GetOrNewStream(sID, true)
			if err != nil {
				return err
			}
			if isNew {
				delete(s.streamMap, sID)
				return nil
			}
		}
		// TODO: stream might be nil
		err := stream.(*SendStream).QueueFrame(frame)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *StreamManager) Read() ([]byte, error) {
	if s.finishedStreams.Empty() {
		waiting := make(chan struct{})
		s.waitReadingChs.Enqueue(&waiting)
		<-waiting
		close(waiting)
	}
	stream, ok := s.finishedStreams.Dequeue().(*RecvStream)
	if !ok || stream == nil {
		return nil, nil
	}
	data, isReset := stream.ReadData()
	// TODO: use isReset to notify the stream was reset
	if isReset {
	}
	return data, nil
}

func (s *StreamManager) CloseAllStream() error {
	// implicitly close all stream
	wg := &sync.WaitGroup{}
	for _, stream := range s.streamMap {
		wg.Add(1)
		go func(st Stream) {
			st.Close()
			wg.Done()
		}(stream)
	}
	wg.Wait()
	return nil
}
<commit_msg>fix MaxStreamID update multithreading issue<commit_after>package quiclatest

import (
	"sync"

	"github.com/ami-GS/gQUIC/latest/qtype"
	"github.com/ami-GS/gQUIC/latest/utils"
)

type signedChannel struct {
	ch     chan struct{}
	closed bool
}

type StreamManager struct {
	streamMap map[qtype.StreamID]Stream
	// would be RcvStream or SendRcvStream
	finishedStreams    *utils.RingBuffer
	maxStreamIDUni     qtype.StreamID
	nxtSendStreamIDUni qtype.StreamID
	maxStreamIDBidi    qtype.StreamID
	nxtStreamIDBidi    qtype.StreamID
	sess               *Session
	// TODO: name should be considered
	blockedIDs      map[qtype.StreamID]*signedChannel
	blockedIDsMutex *sync.Mutex
	handleMaxStreamIDMutex *sync.Mutex

	waitReadingChs *utils.RingBuffer
}

func NewStreamManager(sess *Session) *StreamManager {
	var nxtUniID, nxtBidiID qtype.StreamID
	if sess.isClient {
		nxtUniID = 2
		nxtBidiID = 0
	} else {
		nxtUniID = 3
		nxtBidiID = 1
	}

	return &StreamManager{
		streamMap:       make(map[qtype.StreamID]Stream),
		sess:            sess,
		finishedStreams: utils.NewRingBuffer(20),
		// may be set after handshake, or MAX_STREAM_ID frame
		maxStreamIDUni:     1,
		nxtSendStreamIDUni: nxtUniID,
		maxStreamIDBidi:    100,
		nxtStreamIDBidi:    nxtBidiID,
		blockedIDs:         make(map[qtype.StreamID]*signedChannel),
		blockedIDsMutex:    new(sync.Mutex),
		handleMaxStreamIDMutex:  new(sync.Mutex),
		// TODO: should be big enough and be able to configurable
		waitReadingChs: utils.NewRingBuffer(30),
	}
}

func (s *StreamManager) IsValidID(streamID qtype.StreamID) error {
	if streamID&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		// unidirectional
		if streamID > s.maxStreamIDUni {
			return qtype.StreamIDError
		}
	} else {
		// bidirectional
		if streamID > s.maxStreamIDBidi {
			return qtype.StreamIDError
		}
	}
	return nil
}

func (s *StreamManager) StartNewSendStream() (Stream, error) {
	targetID := s.nxtSendStreamIDUni
	// TODO: atmic increment?
	s.nxtSendStreamIDUni.Increment()

	if targetID > s.maxStreamIDUni {
		blockedChan := &signedChannel{make(chan struct{}), false}
		s.blockedIDsMutex.Lock()
		s.blockedIDs[targetID] = blockedChan
		s.blockedIDsMutex.Unlock()
		s.sess.sendFrameChan <- NewStreamIDBlockedFrame(targetID)

		<-blockedChan.ch
		delete(s.blockedIDs, targetID)
		close(blockedChan.ch)
	}

	stream, _, err := s.GetOrNewStream(targetID, true)
	if err != nil {
		return nil, err
	}
	return stream, nil
}

func (s *StreamManager) GetOrNewStream(streamID qtype.StreamID, send bool) (st Stream, isNew bool, err error) {
	if streamID == 0 {
		return s.getOrNewBidiStream(streamID, s.sess)
	}

	if streamID&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		st, isNew, err = s.getOrNewUniStream(streamID, s.sess, send)
	} else {
		st, isNew, err = s.getOrNewBidiStream(streamID, s.sess)
	}

	if err != nil {
		delete(s.streamMap, streamID)
		return nil, false, err
	}

	if isNew {
		// check whether ID is larger than MAX_STREAM_ID
		err = s.IsValidID(streamID)
		if err != nil {
			delete(s.streamMap, streamID)
			return nil, false, err
		}
	}

	return st, isNew, err
}

func (s *StreamManager) getOrNewUniStream(streamID qtype.StreamID, sess *Session, send bool) (Stream, bool, error) {
	stream, ok := s.streamMap[streamID]
	// get
	if ok {
		if send {
			st, ok := stream.(*SendStream)
			if !ok {
				// TODO: what error?
				return nil, false, nil
			}
			return st, false, nil
		}
		st, ok := stream.(*RecvStream)
		if !ok {
			// TODO: what error?
			return nil, false, nil
		}
		return st, false, nil

	}
	// new
	if send {
		st := newSendStream(streamID, sess)
		s.streamMap[streamID] = st
		return st, true, nil
	}
	st := newRecvStream(streamID, sess)
	s.streamMap[streamID] = st
	return st, true, nil
}

func (s *StreamManager) getOrNewBidiStream(streamID qtype.StreamID, sess *Session) (*SendRecvStream, bool, error) {
	sidVal := streamID
	stream, ok := s.streamMap[streamID]
	if ok {
		st, ok := stream.(*SendRecvStream)
		if !ok {
			// TODO: what error?
			return nil, false, nil
		}
		return st, false, nil
	}
	st := newSendRecvStream(streamID, sess)
	s.streamMap[sidVal] = st
	return st, true, nil
}

// called from session.QueueFrame(f)
func (s *StreamManager) QueueFrame(f StreamLevelFrame) error {
	sid := f.GetStreamID()
	var stream Stream
	var err error
	//var isNew bool
	switch f.(type) {
	case *StreamFrame, *RstStreamFrame, *StreamBlockedFrame:
		stream, _, err = s.GetOrNewStream(sid, true)
	case *MaxStreamDataFrame, *StopSendingFrame:
		stream, _, err = s.GetOrNewStream(sid, false)
	case *MaxStreamIDFrame:
		// this is special, affect only stream_manager
		//s.sess.sendFrameChan <- f
		return nil
	case *StreamIDBlockedFrame:
		// ??
		//s.sess.sendFrameChan <- f
		return nil
	default:
		// error, but impossible to reach here
		return nil
	}
	if err != nil {
		return err
	}
	err = stream.QueueFrame(f)
	if err != nil {
		return err
	}
	if stream.IsTerminated() {
		stream.UpdateConnectionByteSent()
	}
	return nil
}

func (s *StreamManager) handleFrame(f StreamLevelFrame) error {
	sid := f.GetStreamID()
	protocolViolationFunc := func() error {
		// An endpoint that receives a MAX_STREAM_DATA frame for a send-only
		// stream it has not opened MUST terminate the connection with error
		// PROTOCOL_VIOLATION.
		delete(s.streamMap, sid)
		return qtype.ProtocolViolation
	}

	var stream Stream
	var err error
	var isNew bool
	switch frame := f.(type) {
	case *MaxStreamIDFrame:
		return s.handleMaxStreamIDFrame(frame)
	case *StreamIDBlockedFrame:
		return s.handleStreamIDBlockedFrame(frame)
	case *StreamFrame:
		stream, isNew, err = s.GetOrNewStream(sid, false)
		if err != nil {
			return err
		}
		err = stream.handleStreamFrame(frame)
	case *RstStreamFrame:
		stream, _, err = s.GetOrNewStream(sid, false)
		if err != nil {
			return err
		}
		err = stream.handleRstStreamFrame(frame)
	case *StreamBlockedFrame:
		stream, isNew, err = s.GetOrNewStream(sid, false)
		if err != nil {
			return err
		}
		err = stream.handleStreamBlockedFrame(frame)
	case *MaxStreamDataFrame:
		stream, isNew, err = s.GetOrNewStream(sid, true)
		if err != nil {
			return err
		}
		if isNew {
			return protocolViolationFunc()
		}
		err = stream.handleMaxStreamDataFrame(frame)
		return err
	case *StopSendingFrame:
		stream, isNew, err = s.GetOrNewStream(sid, true)
		if err != nil {
			return err
		}
		if isNew {
			return protocolViolationFunc()
		}
		err = stream.handleStopSendingFrame(frame)
	default:
		// error, but impossible to reach here
		return nil
	}

	if err != nil {
		return err
	}

	if stream.IsTerminated() {
		stream.UpdateConnectionByteReceived()
	}
	return err
}

func (s *StreamManager) handleStreamIDBlockedFrame(frame *StreamIDBlockedFrame) error {
	// should be from sender stream which needs new ID, but could not open due to limit

	if frame.StreamID&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		s.maxStreamIDUni = frame.StreamID
	} else {
		s.maxStreamIDBidi = frame.StreamID
	}

	s.sess.sendFrameChan <- NewMaxStreamIDFrame(frame.StreamID)
	return nil
}
func (s *StreamManager) handleMaxStreamIDFrame(frame *MaxStreamIDFrame) error {
	sid := frame.StreamID
	s.handleMaxStreamIDMutex.Lock()
	if sid&qtype.UnidirectionalStream == qtype.UnidirectionalStream {
		// unidirectional
		if sid < s.maxStreamIDUni {
			// ignored
			return nil
		}
		s.maxStreamIDUni = frame.StreamID
	} else {
		//bidirectional
		if sid < s.maxStreamIDBidi {
			// ignored
			return nil
		}
		s.maxStreamIDBidi = frame.StreamID
	}
	s.handleMaxStreamIDMutex.Unlock()

	s.blockedIDsMutex.Lock()
	for blockedID, val := range s.blockedIDs {
		if !val.closed && sid >= blockedID {
			val.closed = true
			val.ch <- struct{}{}
		}
	}
	s.blockedIDsMutex.Unlock()
	return nil
}

func (s *StreamManager) resendBlockedFrames(blockedFrames *utils.RingBuffer) error {

	var stream Stream
	var isNew bool
	var err error
	sID := qtype.StreamID(0)
	size := blockedFrames.Size()
	for i := 0; i < size; i++ {
		frame := blockedFrames.Dequeue().(*StreamFrame)
		if sID != frame.GetStreamID() {
			sID = frame.GetStreamID()
			stream, isNew, err = s.GetOrNewStream(sID, true)
			if err != nil {
				return err
			}
			if isNew {
				delete(s.streamMap, sID)
				return nil
			}
		}
		// TODO: stream might be nil
		err := stream.(*SendStream).QueueFrame(frame)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *StreamManager) Read() ([]byte, error) {
	if s.finishedStreams.Empty() {
		waiting := make(chan struct{})
		s.waitReadingChs.Enqueue(&waiting)
		<-waiting
		close(waiting)
	}
	stream, ok := s.finishedStreams.Dequeue().(*RecvStream)
	if !ok || stream == nil {
		return nil, nil
	}
	data, isReset := stream.ReadData()
	// TODO: use isReset to notify the stream was reset
	if isReset {
	}
	return data, nil
}

func (s *StreamManager) CloseAllStream() error {
	// implicitly close all stream
	wg := &sync.WaitGroup{}
	for _, stream := range s.streamMap {
		wg.Add(1)
		go func(st Stream) {
			st.Close()
			wg.Done()
		}(stream)
	}
	wg.Wait()
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vpython

import (
	"context"
	"os"

	"go.chromium.org/luci/vpython/api/vpython"
	"go.chromium.org/luci/vpython/python"
	"go.chromium.org/luci/vpython/spec"
	"go.chromium.org/luci/vpython/venv"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/system/environ"
	"go.chromium.org/luci/common/system/filesystem"
)

// IsUserError is tagged into errors caused by bad user inputs (e.g. modules or
// scripts which don't exist).
var IsUserError = errors.BoolTag{
	Key: errors.NewTagKey("this error occured due to a user input."),
}

// Options is the set of options to use to construct and execute a VirtualEnv
// Python application.
type Options struct {
	// The Python command-line to execute. Must not be nil.
	CommandLine *python.CommandLine

	// EnvConfig is the VirtualEnv configuration to run from.
	EnvConfig venv.Config

	// DefaultSpec is the default specification to use, if no specification was
	// supplied or probed.
	DefaultSpec vpython.Spec

	// BaseWheels is the set of wheels to include in the spec. These will always
	// be merged into the runtime spec and normalized, such that any duplicate
	// wheels will be deduplicated.
	BaseWheels []*vpython.Spec_Package

	// SpecLoader is the spec.Loader to use to load a specification file for a
	// given script.
	//
	// The empty value is a valid default spec.Loader.
	SpecLoader spec.Loader

	// WaitForEnv, if true, means that if another agent holds a lock on the target
	// environment, we will wait until it is available. If false, we will
	// immediately exit Setup with an error.
	WaitForEnv bool

	// WorkDir is the Python working directory. If empty, the current working
	// directory will be used.
	//
	// If EnvRoot is empty, WorkDir will be used as the base environment root.
	WorkDir string

	// Environ is environment to pass to subprocesses.
	Environ environ.Env

	// ClearPythonPath, if true, instructs vpython to clear the PYTHONPATH
	// environment variable prior to launch.
	//
	// TODO(iannucci): Delete this once we're satisfied that PYTHONPATH exports
	// are under control.
	ClearPythonPath bool
}

func (o *Options) resolve(c context.Context) error {
	// Resolve our working directory to an absolute path.
	if o.WorkDir == "" {
		wd, err := os.Getwd()
		if err != nil {
			return errors.Annotate(err, "failed to get working directory").Err()
		}
		o.WorkDir = wd
	}
	if err := filesystem.AbsPath(&o.WorkDir); err != nil {
		return errors.Annotate(err, "failed to resolve absolute path of WorkDir").Err()
	}

	// Resolve our target python script.
	if err := o.ResolveSpec(c); err != nil {
		return errors.Annotate(err, "failed to resolve Python script").Err()
	}
	if len(o.BaseWheels) > 0 {
		o.EnvConfig.Spec = o.EnvConfig.Spec.Clone()
		o.EnvConfig.Spec.Wheel = append(o.EnvConfig.Spec.Wheel, o.BaseWheels...)
	}

	return nil
}

// ResolveSpec resolves the configured environment specification. The resulting
// spec is installed into o's EnvConfig.Spec field.
func (o *Options) ResolveSpec(c context.Context) error {
	if o.CommandLine == nil {
		panic("a CommandLine must be specified")
	}

	// If a spec is explicitly provided, we're done.
	if o.EnvConfig.Spec != nil {
		return nil
	}

	// If we're running a Python script, assert that the target script exists.
	// Additionally, track whether it's a file or a module (directory).
	target := o.CommandLine.Target
	script, isScriptTarget := target.(python.ScriptTarget)
	isModule := false
	if isScriptTarget {
		logging.Debugf(c, "Resolved Python target script: %s", target)

		// Resolve to absolute script path.
		if err := filesystem.AbsPath(&script.Path); err != nil {
			return errors.Annotate(err, "failed to get absolute path of: %s", target).Err()
		}

		// Confirm that the script path actually exists.
		st, err := os.Stat(script.Path)
		if err != nil {
			return IsUserError.Apply(err)
		}

		// If the script is a directory, then we assume that we're doing a module
		// invocation (__main__.py).
		isModule = st.IsDir()
	}

	// If it's a script, try resolving from filesystem first.
	if isScriptTarget {
		spec, err := o.SpecLoader.LoadForScript(c, script.Path, isModule)
		if err != nil {
			return errors.Annotate(err, "failed to load spec for script: %s", target).
				InternalReason("isModule(%v)", isModule).Err()
		}
		if spec != nil {
			o.EnvConfig.Spec = spec
			return nil
		}
	}

	// If standard resolution doesn't yield a spec, fall back on our default spec.
	logging.Infof(c, "Unable to resolve specification path. Using default specification.")
	o.EnvConfig.Spec = &o.DefaultSpec
	return nil
}
<commit_msg>vpython: skip spec probing for scripts passed via stdin.<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vpython

import (
	"context"
	"os"

	"go.chromium.org/luci/vpython/api/vpython"
	"go.chromium.org/luci/vpython/python"
	"go.chromium.org/luci/vpython/spec"
	"go.chromium.org/luci/vpython/venv"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/system/environ"
	"go.chromium.org/luci/common/system/filesystem"
)

// IsUserError is tagged into errors caused by bad user inputs (e.g. modules or
// scripts which don't exist).
var IsUserError = errors.BoolTag{
	Key: errors.NewTagKey("this error occured due to a user input."),
}

// Options is the set of options to use to construct and execute a VirtualEnv
// Python application.
type Options struct {
	// The Python command-line to execute. Must not be nil.
	CommandLine *python.CommandLine

	// EnvConfig is the VirtualEnv configuration to run from.
	EnvConfig venv.Config

	// DefaultSpec is the default specification to use, if no specification was
	// supplied or probed.
	DefaultSpec vpython.Spec

	// BaseWheels is the set of wheels to include in the spec. These will always
	// be merged into the runtime spec and normalized, such that any duplicate
	// wheels will be deduplicated.
	BaseWheels []*vpython.Spec_Package

	// SpecLoader is the spec.Loader to use to load a specification file for a
	// given script.
	//
	// The empty value is a valid default spec.Loader.
	SpecLoader spec.Loader

	// WaitForEnv, if true, means that if another agent holds a lock on the target
	// environment, we will wait until it is available. If false, we will
	// immediately exit Setup with an error.
	WaitForEnv bool

	// WorkDir is the Python working directory. If empty, the current working
	// directory will be used.
	//
	// If EnvRoot is empty, WorkDir will be used as the base environment root.
	WorkDir string

	// Environ is environment to pass to subprocesses.
	Environ environ.Env

	// ClearPythonPath, if true, instructs vpython to clear the PYTHONPATH
	// environment variable prior to launch.
	//
	// TODO(iannucci): Delete this once we're satisfied that PYTHONPATH exports
	// are under control.
	ClearPythonPath bool
}

func (o *Options) resolve(c context.Context) error {
	// Resolve our working directory to an absolute path.
	if o.WorkDir == "" {
		wd, err := os.Getwd()
		if err != nil {
			return errors.Annotate(err, "failed to get working directory").Err()
		}
		o.WorkDir = wd
	}
	if err := filesystem.AbsPath(&o.WorkDir); err != nil {
		return errors.Annotate(err, "failed to resolve absolute path of WorkDir").Err()
	}

	// Resolve our target python script.
	if err := o.ResolveSpec(c); err != nil {
		return errors.Annotate(err, "failed to resolve Python script").Err()
	}
	if len(o.BaseWheels) > 0 {
		o.EnvConfig.Spec = o.EnvConfig.Spec.Clone()
		o.EnvConfig.Spec.Wheel = append(o.EnvConfig.Spec.Wheel, o.BaseWheels...)
	}

	return nil
}

// ResolveSpec resolves the configured environment specification. The resulting
// spec is installed into o's EnvConfig.Spec field.
func (o *Options) ResolveSpec(c context.Context) error {
	if o.CommandLine == nil {
		panic("a CommandLine must be specified")
	}

	// If a spec is explicitly provided, we're done.
	if o.EnvConfig.Spec != nil {
		return nil
	}

	o.EnvConfig.Spec = &o.DefaultSpec

	// If we're running a Python script, assert that the target script exists.
	// Additionally, track whether it's a file or a module (directory).
	target := o.CommandLine.Target
	script, isScriptTarget := target.(python.ScriptTarget)
	if isScriptTarget && script.Path == "-" {
		logging.Infof(c, "Skipping specification probing for script via stdin.")
		return nil
	}

	isModule := false
	if isScriptTarget {
		logging.Debugf(c, "Resolved Python target script: %s", target)

		// Resolve to absolute script path.
		if err := filesystem.AbsPath(&script.Path); err != nil {
			return errors.Annotate(err, "failed to get absolute path of: %s", target).Err()
		}

		// Confirm that the script path actually exists.
		st, err := os.Stat(script.Path)
		if err != nil {
			return IsUserError.Apply(err)
		}

		// If the script is a directory, then we assume that we're doing a module
		// invocation (__main__.py).
		isModule = st.IsDir()
	}

	// If it's a script, try resolving from filesystem first.
	if isScriptTarget {
		spec, err := o.SpecLoader.LoadForScript(c, script.Path, isModule)
		if err != nil {
			return errors.Annotate(err, "failed to load spec for script: %s", target).
				InternalReason("isModule(%v)", isModule).Err()
		}
		if spec != nil {
			o.EnvConfig.Spec = spec
			return nil
		}
	}

	// If standard resolution doesn't yield a spec, fall back on our default spec.
	logging.Infof(c, "Unable to resolve specification path. Using default specification.")
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/imdario/mergo"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

type Probe struct {
	Path             string
	InitialDelay     int `yaml:"initialDelay"`
	PeriodSeconds    int `yaml:"periodSeconds"`
	FailureThreshold int `yaml:"failureThreshold"`
}

type Healthcheck struct {
	Liveness  Probe
	Readiness Probe
}

type ResourceList struct {
	Cpu    string
	Memory string
}

type ResourceRequirements struct {
	Limits   ResourceList
	Requests ResourceList
}

type PrometheusConfig struct {
	Enabled bool
	Port    string
	Path    string
}

type NaisManifest struct {
	Image           string
	Port            int
	Healthcheck     Healthcheck
	PreStopHookPath string `yaml:"preStopHookPath"`
	Prometheus      PrometheusConfig
	Replicas        Replicas
	Ingress         Ingress
	Resources       ResourceRequirements
	FasitResources  FasitResources `yaml:"fasitResources"`
}

type Ingress struct {
	Enabled bool
}

type Replicas struct {
	Min                    int
	Max                    int
	CpuThresholdPercentage int `yaml:"cpuThresholdPercentage"`
}

type FasitResources struct {
	Used    []UsedResource
	Exposed []ExposedResource
}

type UsedResource struct {
	Alias        string
	ResourceType string            `yaml:"resourceType"`
	PropertyMap  map[string]string `yaml:"propertyMap"`
}

type ExposedResource struct {
	Alias          string
	ResourceType   string `yaml:"resourceType"`
	Path           string
	Description    string
	WsdlGroupId    string `yaml:"wsdlGroupId"`
	WsdlArtifactId string `yaml:"wsdlArtifactId"`
	WsdlVersion    string `yaml:"wsdlVersion"`
	SecurityToken  string `yaml:"securityToken"`
	AllZones       bool   `yaml:"allZones"`
}

type ValidationErrors struct {
	Errors []ValidationError
}

type ValidationError struct {
	ErrorMessage string
	Fields       map[string]string
}

type Field struct {
	Name  string
	Value string
}

func GenerateManifest(deploymentRequest NaisDeploymentRequest) (naisManifest NaisManifest, err error) {

	manifest, err := downloadManifest(deploymentRequest)
	if err != nil {
		glog.Errorf("could not download manifest", err)
		return NaisManifest{}, err
	}

	if err := AddDefaultManifestValues(&manifest, deploymentRequest.Application); err != nil {
		glog.Errorf("Could not merge manifest %s", err)
		return NaisManifest{}, err
	}

	validationErrors := ValidateManifest(manifest)
	if len(validationErrors.Errors) != 0 {
		glog.Error("Invalid manifest: ", validationErrors.Error())
		return NaisManifest{}, validationErrors
	}

	return manifest, nil
}

func downloadManifest(deploymentRequest NaisDeploymentRequest) (naisManifest NaisManifest, err error) {
	// manifest url is provided in deployment request
	manifestUrl := getManifestUrl(deploymentRequest)
	if len(manifestUrl) > 0 {
		manifest, err := fetchManifest(manifestUrl)
		if err != nil {
			return NaisManifest{}, err
		} else {
			return manifest, nil
		}
	}

	// not provided, using defaults
	urls := createManifestUrl(deploymentRequest.Application, deploymentRequest.Version)
	for _, url := range urls {
		manifest, err := fetchManifest(url)
		if err == nil {
			return manifest, nil
		}
	}

	return NaisManifest{}, fmt.Errorf("No manifest found on the URLs %s, or the url %s\n", urls, manifestUrl)
}

// Deprecated
func getManifestUrl(req NaisDeploymentRequest) string {
	if len(req.AppConfigUrl) > 0 {
		glog.Info("%s is deploying %s using deprecated appconfigurl", req.Username, req.Application)
		return req.AppConfigUrl
	}
	return req.ManifestUrl
}

func createManifestUrl(application, version string) []string {
	return []string{
		fmt.Sprintf("https://repo.adeo.no/repository/raw/nais/%s/%s/nais.yaml", application, version),
		fmt.Sprintf("http://nexus.adeo.no/nexus/service/local/repositories/m2internal/content/nais/%s/%s/nais.yaml", application, version),
		fmt.Sprintf("http://nexus.adeo.no/nexus/service/local/repositories/m2internal/content/nais/%s/%s/%s.yaml", application, version, application+"-"+version),
	}
}

func AddDefaultManifestValues(manifest *NaisManifest, application string) error {
	return mergo.Merge(manifest, GetDefaultManifest(application))
}
func fetchManifest(url string) (NaisManifest, error) {
	glog.Infof("Fetching manifest from URL %s\n", url)
	response, err := http.Get(url)
	if err != nil {
		glog.Errorf("Could not fetch %s", err)
		return NaisManifest{}, fmt.Errorf("HTTP GET failed for url: %s. %s", url, err.Error())
	}

	defer response.Body.Close()

	if response.StatusCode > 299 {
		return NaisManifest{}, fmt.Errorf("Got HTTP status code %d fetching manifest from URL: %s", response.StatusCode, url)
	}

	if body, err := ioutil.ReadAll(response.Body); err != nil {
		return NaisManifest{}, err
	} else {
		var manifest NaisManifest
		if err := yaml.Unmarshal(body, &manifest); err != nil {
			glog.Errorf("Could not unmarshal yaml %s", err)
			return NaisManifest{}, fmt.Errorf("Unable to unmarshal yaml: %s", err.Error())
		}
		glog.Infof("Got manifest %s", manifest)
		return manifest, err
	}
}

func ValidateManifest(manifest NaisManifest) ValidationErrors {
	validations := []func(NaisManifest) *ValidationError{
		validateImage,
		validateReplicasMax,
		validateReplicasMin,
		validateMinIsSmallerThanMax,
		validateCpuThreshold,
		validateResources,
	}

	var validationErrors ValidationErrors
	for _, valfunc := range validations {
		if valError := valfunc(manifest); valError != nil {
			validationErrors.Errors = append(validationErrors.Errors, *valError)
		}
	}

	return validationErrors
}

func validateResources(manifest NaisManifest) *ValidationError {
	for _, resource := range manifest.FasitResources.Exposed {
		if resource.ResourceType == "" || resource.Alias == "" {
			return &ValidationError{
				"Alias and ResourceType must be specified",
				map[string]string{"Alias": resource.Alias},
			}
		}
	}
	for _, resource := range manifest.FasitResources.Used {
		if resource.ResourceType == "" || resource.Alias == "" {
			return &ValidationError{
				"Alias and ResourceType must be specified",
				map[string]string{"Alias": resource.Alias},
			}
		}
	}
	return nil
}
func validateImage(manifest NaisManifest) *ValidationError {
	if strings.LastIndex(manifest.Image, ":") > strings.LastIndex(manifest.Image, "/") {
		return &ValidationError{
			"Image cannot contain tag",
			map[string]string{"Image": manifest.Image},
		}
	}
	return nil
}

func validateCpuThreshold(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.CpuThresholdPercentage < 10 || manifest.Replicas.CpuThresholdPercentage > 90 {
		error := new(ValidationError)
		error.ErrorMessage = "CpuThreshold must be between 10 and 90."
		error.Fields = make(map[string]string)
		error.Fields["Replicas.CpuThreshold"] = strconv.Itoa(manifest.Replicas.CpuThresholdPercentage)
		return error

	}
	return nil

}
func validateMinIsSmallerThanMax(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.Min > manifest.Replicas.Max {
		validationError := new(ValidationError)
		validationError.ErrorMessage = "Replicas.Min is larger than Replicas.Max."
		validationError.Fields = make(map[string]string)
		validationError.Fields["Replicas.Max"] = strconv.Itoa(manifest.Replicas.Max)
		validationError.Fields["Replicas.Min"] = strconv.Itoa(manifest.Replicas.Min)
		return validationError
	}
	return nil

}
func validateReplicasMin(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.Min == 0 {
		validationError := new(ValidationError)
		validationError.ErrorMessage = "Replicas.Min is not set"
		validationError.Fields = make(map[string]string)
		validationError.Fields["Replicas.Min"] = strconv.Itoa(manifest.Replicas.Min)
		return validationError

	}
	return nil

}
func validateReplicasMax(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.Max == 0 {
		validationError := new(ValidationError)
		validationError.ErrorMessage = "Replicas.Max is not set"
		validationError.Fields = make(map[string]string)
		validationError.Fields["Replicas.Max"] = strconv.Itoa(manifest.Replicas.Max)
		return validationError
	}
	return nil

}

func (errors ValidationErrors) Error() (s string) {
	for _, validationError := range errors.Errors {
		s += validationError.ErrorMessage + "\n"
		for k, v := range validationError.Fields {
			s += " - " + k + ": " + v + ".\n"
		}
	}
	return s
}
<commit_msg>Huge difference between info and infof<commit_after>package api

import (
	"fmt"
	"github.com/golang/glog"
	"github.com/imdario/mergo"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

type Probe struct {
	Path             string
	InitialDelay     int `yaml:"initialDelay"`
	PeriodSeconds    int `yaml:"periodSeconds"`
	FailureThreshold int `yaml:"failureThreshold"`
}

type Healthcheck struct {
	Liveness  Probe
	Readiness Probe
}

type ResourceList struct {
	Cpu    string
	Memory string
}

type ResourceRequirements struct {
	Limits   ResourceList
	Requests ResourceList
}

type PrometheusConfig struct {
	Enabled bool
	Port    string
	Path    string
}

type NaisManifest struct {
	Image           string
	Port            int
	Healthcheck     Healthcheck
	PreStopHookPath string `yaml:"preStopHookPath"`
	Prometheus      PrometheusConfig
	Replicas        Replicas
	Ingress         Ingress
	Resources       ResourceRequirements
	FasitResources  FasitResources `yaml:"fasitResources"`
}

type Ingress struct {
	Enabled bool
}

type Replicas struct {
	Min                    int
	Max                    int
	CpuThresholdPercentage int `yaml:"cpuThresholdPercentage"`
}

type FasitResources struct {
	Used    []UsedResource
	Exposed []ExposedResource
}

type UsedResource struct {
	Alias        string
	ResourceType string            `yaml:"resourceType"`
	PropertyMap  map[string]string `yaml:"propertyMap"`
}

type ExposedResource struct {
	Alias          string
	ResourceType   string `yaml:"resourceType"`
	Path           string
	Description    string
	WsdlGroupId    string `yaml:"wsdlGroupId"`
	WsdlArtifactId string `yaml:"wsdlArtifactId"`
	WsdlVersion    string `yaml:"wsdlVersion"`
	SecurityToken  string `yaml:"securityToken"`
	AllZones       bool   `yaml:"allZones"`
}

type ValidationErrors struct {
	Errors []ValidationError
}

type ValidationError struct {
	ErrorMessage string
	Fields       map[string]string
}

type Field struct {
	Name  string
	Value string
}

func GenerateManifest(deploymentRequest NaisDeploymentRequest) (naisManifest NaisManifest, err error) {

	manifest, err := downloadManifest(deploymentRequest)
	if err != nil {
		glog.Errorf("could not download manifest", err)
		return NaisManifest{}, err
	}

	if err := AddDefaultManifestValues(&manifest, deploymentRequest.Application); err != nil {
		glog.Errorf("Could not merge manifest %s", err)
		return NaisManifest{}, err
	}

	validationErrors := ValidateManifest(manifest)
	if len(validationErrors.Errors) != 0 {
		glog.Error("Invalid manifest: ", validationErrors.Error())
		return NaisManifest{}, validationErrors
	}

	return manifest, nil
}

func downloadManifest(deploymentRequest NaisDeploymentRequest) (naisManifest NaisManifest, err error) {
	// manifest url is provided in deployment request
	manifestUrl := getManifestUrl(deploymentRequest)
	if len(manifestUrl) > 0 {
		manifest, err := fetchManifest(manifestUrl)
		if err != nil {
			return NaisManifest{}, err
		} else {
			return manifest, nil
		}
	}

	// not provided, using defaults
	urls := createManifestUrl(deploymentRequest.Application, deploymentRequest.Version)
	for _, url := range urls {
		manifest, err := fetchManifest(url)
		if err == nil {
			return manifest, nil
		}
	}

	return NaisManifest{}, fmt.Errorf("No manifest found on the URLs %s, or the url %s\n", urls, manifestUrl)
}

// Deprecated
func getManifestUrl(req NaisDeploymentRequest) string {
	if len(req.AppConfigUrl) > 0 {
		glog.Infof("%s is deploying %s using deprecated appconfigurl", req.Username, req.Application)
		return req.AppConfigUrl
	}
	return req.ManifestUrl
}

func createManifestUrl(application, version string) []string {
	return []string{
		fmt.Sprintf("https://repo.adeo.no/repository/raw/nais/%s/%s/nais.yaml", application, version),
		fmt.Sprintf("http://nexus.adeo.no/nexus/service/local/repositories/m2internal/content/nais/%s/%s/nais.yaml", application, version),
		fmt.Sprintf("http://nexus.adeo.no/nexus/service/local/repositories/m2internal/content/nais/%s/%s/%s.yaml", application, version, application+"-"+version),
	}
}

func AddDefaultManifestValues(manifest *NaisManifest, application string) error {
	return mergo.Merge(manifest, GetDefaultManifest(application))
}
func fetchManifest(url string) (NaisManifest, error) {
	glog.Infof("Fetching manifest from URL %s\n", url)
	response, err := http.Get(url)
	if err != nil {
		glog.Errorf("Could not fetch %s", err)
		return NaisManifest{}, fmt.Errorf("HTTP GET failed for url: %s. %s", url, err.Error())
	}

	defer response.Body.Close()

	if response.StatusCode > 299 {
		return NaisManifest{}, fmt.Errorf("Got HTTP status code %d fetching manifest from URL: %s", response.StatusCode, url)
	}

	if body, err := ioutil.ReadAll(response.Body); err != nil {
		return NaisManifest{}, err
	} else {
		var manifest NaisManifest
		if err := yaml.Unmarshal(body, &manifest); err != nil {
			glog.Errorf("Could not unmarshal yaml %s", err)
			return NaisManifest{}, fmt.Errorf("Unable to unmarshal yaml: %s", err.Error())
		}
		glog.Infof("Got manifest %s", manifest)
		return manifest, err
	}
}

func ValidateManifest(manifest NaisManifest) ValidationErrors {
	validations := []func(NaisManifest) *ValidationError{
		validateImage,
		validateReplicasMax,
		validateReplicasMin,
		validateMinIsSmallerThanMax,
		validateCpuThreshold,
		validateResources,
	}

	var validationErrors ValidationErrors
	for _, valfunc := range validations {
		if valError := valfunc(manifest); valError != nil {
			validationErrors.Errors = append(validationErrors.Errors, *valError)
		}
	}

	return validationErrors
}

func validateResources(manifest NaisManifest) *ValidationError {
	for _, resource := range manifest.FasitResources.Exposed {
		if resource.ResourceType == "" || resource.Alias == "" {
			return &ValidationError{
				"Alias and ResourceType must be specified",
				map[string]string{"Alias": resource.Alias},
			}
		}
	}
	for _, resource := range manifest.FasitResources.Used {
		if resource.ResourceType == "" || resource.Alias == "" {
			return &ValidationError{
				"Alias and ResourceType must be specified",
				map[string]string{"Alias": resource.Alias},
			}
		}
	}
	return nil
}
func validateImage(manifest NaisManifest) *ValidationError {
	if strings.LastIndex(manifest.Image, ":") > strings.LastIndex(manifest.Image, "/") {
		return &ValidationError{
			"Image cannot contain tag",
			map[string]string{"Image": manifest.Image},
		}
	}
	return nil
}

func validateCpuThreshold(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.CpuThresholdPercentage < 10 || manifest.Replicas.CpuThresholdPercentage > 90 {
		error := new(ValidationError)
		error.ErrorMessage = "CpuThreshold must be between 10 and 90."
		error.Fields = make(map[string]string)
		error.Fields["Replicas.CpuThreshold"] = strconv.Itoa(manifest.Replicas.CpuThresholdPercentage)
		return error

	}
	return nil

}
func validateMinIsSmallerThanMax(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.Min > manifest.Replicas.Max {
		validationError := new(ValidationError)
		validationError.ErrorMessage = "Replicas.Min is larger than Replicas.Max."
		validationError.Fields = make(map[string]string)
		validationError.Fields["Replicas.Max"] = strconv.Itoa(manifest.Replicas.Max)
		validationError.Fields["Replicas.Min"] = strconv.Itoa(manifest.Replicas.Min)
		return validationError
	}
	return nil

}
func validateReplicasMin(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.Min == 0 {
		validationError := new(ValidationError)
		validationError.ErrorMessage = "Replicas.Min is not set"
		validationError.Fields = make(map[string]string)
		validationError.Fields["Replicas.Min"] = strconv.Itoa(manifest.Replicas.Min)
		return validationError

	}
	return nil

}
func validateReplicasMax(manifest NaisManifest) *ValidationError {
	if manifest.Replicas.Max == 0 {
		validationError := new(ValidationError)
		validationError.ErrorMessage = "Replicas.Max is not set"
		validationError.Fields = make(map[string]string)
		validationError.Fields["Replicas.Max"] = strconv.Itoa(manifest.Replicas.Max)
		return validationError
	}
	return nil

}

func (errors ValidationErrors) Error() (s string) {
	for _, validationError := range errors.Errors {
		s += validationError.ErrorMessage + "\n"
		for k, v := range validationError.Fields {
			s += " - " + k + ": " + v + ".\n"
		}
	}
	return s
}
<|endoftext|>


<commit_before>// config.go
//
// This file implements the configuration part for when you need the API
// key to modify things in the Atlas configuration and manage measurements.

package main

import (
	"os"
	"path/filepath"
	"strings"
)

// Check the parameter for either tag or filename
func checkName(file string) (str string) {
	// Full path, begin by \\ or N: MUST have .toml
	bfile := []byte(file)
	if bfile[1] == ':' || strings.HasPrefix(file, "\\\\"){
		if !strings.HasSuffix(file, ".toml") {
			str = ""
		} else {
			str = file
		}
		return
	}

	// If ending with .toml, take it literally
	if strings.HasSuffix(file, ".toml") {
		str = file
		return
	}

	// Check for tag
	if !strings.HasSuffix(file, ".toml") {
		// file must be a tag so add a "."
		str = filepath.Join(os.Getenv("%LOCALAPPDATA%"),
			"RIPE-Atlas",
			"config.toml")
	}
	return
}
<commit_msg>OS-specific variables.<commit_after>// config.go
//
// This file implements the configuration part for when you need the API
// key to modify things in the Atlas configuration and manage measurements.

package main

import (
	"os"
	"path/filepath"
	"strings"
)

var (
	basedir = filepath.Join(os.Getenv("%LOCALAPPDATA%"),
		"RIPE-ATLAS",
	)

	dbrcFile = filepath.Join(basedir, "dbrc")
)

// Check the parameter for either tag or filename
func checkName(file string) (str string) {
	// Full path, begin by \\ or N: MUST have .toml
	bfile := []byte(file)
	if bfile[1] == ':' || strings.HasPrefix(file, "\\\\"){
		if !strings.HasSuffix(file, ".toml") {
			str = ""
		} else {
			str = file
		}
		return
	}

	// If ending with .toml, take it literally
	if strings.HasSuffix(file, ".toml") {
		str = file
		return
	}

	// Check for tag
	if !strings.HasSuffix(file, ".toml") {
		// file must be a tag so add a "."
		str = filepath.Join(basedir, "config.toml")
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux darwin

package main

import (
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

var proxyGolangOrg *httputil.ReverseProxy // initialized just below

func init() {
	u, err := url.Parse("https://proxy.golang.org")
	if err != nil {
		log.Fatal(err)
	}
	rp := httputil.NewSingleHostReverseProxy(u)
	rp.ModifyResponse = func(res *http.Response) error {
		r := res.Request
		if res.StatusCode/100 != 2 && r != nil {
			log.Printf("modproxy: proxying HTTP %s response from backend for %s, %s %s", res.Status, r.RemoteAddr, r.Method, r.RequestURI)
		}
		return nil
	}
	proxyGolangOrg = rp
}

func listenAndServeInternalModuleProxy() {
	err := http.ListenAndServe(":8123", http.HandlerFunc(proxyModuleCache))
	log.Fatalf("error running internal module proxy: %v", err)
}

// proxyModuleCache proxies requests to https://proxy.golang.org/
func proxyModuleCache(w http.ResponseWriter, r *http.Request) {
	// Delete any Host header so it's the one sent to the backend
	// is proxy.golang.org by the default ReverseProxy.Director
	// setting the URL.Host. (Host takes priority for Host header,
	// if present)
	r.Host = ""
	proxyGolangOrg.ServeHTTP(w, r)
}
<commit_msg>cmd/coordinator: don't log about proxying 410 responses from proxy.golang.org<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux darwin

package main

import (
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

var proxyGolangOrg *httputil.ReverseProxy // initialized just below

func init() {
	u, err := url.Parse("https://proxy.golang.org")
	if err != nil {
		log.Fatal(err)
	}
	rp := httputil.NewSingleHostReverseProxy(u)
	rp.ModifyResponse = func(res *http.Response) error {
		r := res.Request
		if res.StatusCode/100 != 2 && res.StatusCode != 410 && r != nil {
			log.Printf("modproxy: proxying HTTP %s response from backend for %s, %s %s", res.Status, r.RemoteAddr, r.Method, r.RequestURI)
		}
		return nil
	}
	proxyGolangOrg = rp
}

func listenAndServeInternalModuleProxy() {
	err := http.ListenAndServe(":8123", http.HandlerFunc(proxyModuleCache))
	log.Fatalf("error running internal module proxy: %v", err)
}

// proxyModuleCache proxies requests to https://proxy.golang.org/
func proxyModuleCache(w http.ResponseWriter, r *http.Request) {
	// Delete any Host header so it's the one sent to the backend
	// is proxy.golang.org by the default ReverseProxy.Director
	// setting the URL.Host. (Host takes priority for Host header,
	// if present)
	r.Host = ""
	proxyGolangOrg.ServeHTTP(w, r)
}
<|endoftext|>


<commit_before>// package pybr stands for pinyin by range
package pybr

import (
	"bytes"

	"github.com/axgle/mahonia"
)

var (
	pyTableLen = len(pyValue)
)

type Pinyin struct {
	//分隔符
	Split string
	//是否首字母大写
	Upper bool
}

func New(split string) Pinyin {
	return Pinyin{
		Split: split,
		Upper: true,
	}
}

//get chinese pinyin number code
//param s must be chinese character with utf8 encoding
func Code(s string) int {
	//print(s)
	gbkString := UTF8ToGBK(s)
	//print(len(gbkString))
	var i1, i2 int
	i1 = int(gbkString[0])
	i2 = int(gbkString[1])
	return i1*256 + i2 - 65536
}

// converts utf8 chinese to pinyin
func (c *Pinyin) Convert(s string) string {
	pyString := bytes.NewBufferString("")
	var str string
	var code int

	for _, rune := range s {
		str = string(rune)
		//println(str + ": " + pyString.String())

		if rune <= '~' {
			pyString.WriteString(str)
		} else {
			code = Code(str)
			if code > 0 && code < 160 {
				pyString.WriteString(str)
			} else {
				if v, ok := tableMap[code]; ok { //map by table
					pyString.WriteString(v + c.Split)
				} else {
					found := false
					for i := pyTableLen - 1; i >= 0; i-- {
						if pyValue[i] <= code {
							pyString.WriteString(pyName[i] + c.Split)
							found = true
							break
						}
					}
					if !found {
						pyString.WriteString(str)
					}
				}
			}
		}
	}
	return pyString.String()
}

//define func UTF8ToGBK by using mahonia package
var enc = mahonia.NewEncoder("gbk")

func UTF8ToGBK(s string) string {
	return enc.ConvertString(s)
}
<commit_msg>- [+] add back the .Upper control<commit_after>// package pybr stands for pinyin by range
package pybr

import (
	"bytes"
	"strings"

	"github.com/axgle/mahonia"
)

var (
	pyTableLen = len(pyValue)
)

type Pinyin struct {
	//分隔符
	Split string
	//是否首字母大写
	Upper bool
}

func New(split string) Pinyin {
	return Pinyin{
		Split: split,
		Upper: true,
	}
}

//get chinese pinyin number code
//param s must be chinese character with utf8 encoding
func Code(s string) int {
	//print(s)
	gbkString := UTF8ToGBK(s)
	//print(len(gbkString))
	var i1, i2 int
	i1 = int(gbkString[0])
	i2 = int(gbkString[1])
	return i1*256 + i2 - 65536
}

// converts utf8 chinese to pinyin
func (c *Pinyin) Convert(s string) string {
	pyString := bytes.NewBufferString("")
	var str string
	var code int

	for _, rune := range s {
		str = string(rune)
		//println(str + ": " + pyString.String())

		if rune <= '~' {
			pyString.WriteString(str)
		} else {
			code = Code(str)
			if code > 0 && code < 160 {
				pyString.WriteString(str)
			} else {
				if v, ok := tableMap[code]; ok { //map by table
					pyString.WriteString(v + c.Split)
				} else {
					found := false
					for i := pyTableLen - 1; i >= 0; i-- {
						if pyValue[i] <= code {
							py := pyName[i]
							if !c.Upper {
								py = strings.ToLower(py)
							}
							pyString.WriteString(py + c.Split)
							found = true
							break
						}
					}
					if !found {
						pyString.WriteString(str)
					}
				}
			}
		}
	}
	return pyString.String()
}

//define func UTF8ToGBK by using mahonia package
var enc = mahonia.NewEncoder("gbk")

func UTF8ToGBK(s string) string {
	return enc.ConvertString(s)
}
<|endoftext|>