<commit_before>package hummingbird

import (
	"testing"
)

func TestUnpicklingVersion1Map(t *testing.T) {
	data, err := PickleLoads([]byte("(dp1\nS'hi'\np2\nS'there'\np3\ns."))
	if err != nil {
		t.Fatal("Error parsing pickle: " + err.Error())
	}
	if dataVal, ok := data.(map[interface{}]interface{}); ok {
		if dataVal["hi"] != "there" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func TestUnpicklingVersion2Map(t *testing.T) {
	data, err := PickleLoads([]byte("\x80\x02}q\x01U\x02hiq\x02U\x05thereq\x03s."))
	if err != nil {
		t.Fatal("Error parsing pickle: " + err.Error())
	}
	if dataVal, ok := data.(map[interface{}]interface{}); ok {
		if dataVal["hi"] != "there" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func PickleRoundTrip(t *testing.T, v interface{}) (interface{}) {
  	str := PickleDumps(v)
	ret, err := PickleLoads(str)
	if err != nil {
	  	t.Fatal("Error parsing pickle: " + err.Error())
	}
	return ret
}

func TestRoundTrip1(t *testing.T) {
  	if dataVal, ok := PickleRoundTrip(t, 2).(int64); ok {
	  	if dataVal != 2 {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func TestRoundTrip2(t *testing.T) {
  	if dataVal, ok := PickleRoundTrip(t, "hi").(string); ok {
	  	if dataVal != "hi" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func TestRoundTrip3(t *testing.T) {
	data := map[string]string{"1": "test1", "2": "test2"}
  	if dataVal, ok := PickleRoundTrip(t, data).(map[interface{}]interface{}); ok {
	  	if dataVal["1"] != "test1" {
			t.Fatal("Return data not correct.")
		}
	  	if dataVal["2"] != "test2" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}
<commit_msg>go fmt<commit_after>package hummingbird

import (
	"testing"
)

func TestUnpicklingVersion1Map(t *testing.T) {
	data, err := PickleLoads([]byte("(dp1\nS'hi'\np2\nS'there'\np3\ns."))
	if err != nil {
		t.Fatal("Error parsing pickle: " + err.Error())
	}
	if dataVal, ok := data.(map[interface{}]interface{}); ok {
		if dataVal["hi"] != "there" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func TestUnpicklingVersion2Map(t *testing.T) {
	data, err := PickleLoads([]byte("\x80\x02}q\x01U\x02hiq\x02U\x05thereq\x03s."))
	if err != nil {
		t.Fatal("Error parsing pickle: " + err.Error())
	}
	if dataVal, ok := data.(map[interface{}]interface{}); ok {
		if dataVal["hi"] != "there" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func PickleRoundTrip(t *testing.T, v interface{}) interface{} {
	str := PickleDumps(v)
	ret, err := PickleLoads(str)
	if err != nil {
		t.Fatal("Error parsing pickle: " + err.Error())
	}
	return ret
}

func TestRoundTrip1(t *testing.T) {
	if dataVal, ok := PickleRoundTrip(t, 2).(int64); ok {
		if dataVal != 2 {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func TestRoundTrip2(t *testing.T) {
	if dataVal, ok := PickleRoundTrip(t, "hi").(string); ok {
		if dataVal != "hi" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}

func TestRoundTrip3(t *testing.T) {
	data := map[string]string{"1": "test1", "2": "test2"}
	if dataVal, ok := PickleRoundTrip(t, data).(map[interface{}]interface{}); ok {
		if dataVal["1"] != "test1" {
			t.Fatal("Return data not correct.")
		}
		if dataVal["2"] != "test2" {
			t.Fatal("Return data not correct.")
		}
	} else {
		t.Fatal("Invalid return type.")
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compact

import (
	"fmt"
	"reflect"
	"testing"
)

func TestRangeNodes(t *testing.T) {
	n := func(level uint, index uint64) NodeID {
		return NewNodeID(level, index)
	}
	for _, tc := range []struct {
		begin uint64
		end   uint64
		want  []NodeID
	}{
		// Empty ranges.
		{end: 0, want: []NodeID{}},
		{begin: 10, end: 10, want: []NodeID{}},
		{begin: 1024, end: 1024, want: []NodeID{}},
		// One entry.
		{begin: 10, end: 11, want: []NodeID{n(0, 10)}},
		{begin: 1024, end: 1025, want: []NodeID{n(0, 1024)}},
		{begin: 1025, end: 1026, want: []NodeID{n(0, 1025)}},
		// Two entries.
		{begin: 10, end: 12, want: []NodeID{n(1, 5)}},
		{begin: 1024, end: 1026, want: []NodeID{n(1, 512)}},
		{begin: 1025, end: 1027, want: []NodeID{n(0, 1025), n(0, 1026)}},
		// Only right border.
		{end: 1, want: []NodeID{n(0, 0)}},
		{end: 2, want: []NodeID{n(1, 0)}},
		{end: 3, want: []NodeID{n(1, 0), n(0, 2)}},
		{end: 4, want: []NodeID{n(2, 0)}},
		{end: 5, want: []NodeID{n(2, 0), n(0, 4)}},
		{end: 15, want: []NodeID{n(3, 0), n(2, 2), n(1, 6), n(0, 14)}},
		{end: 100, want: []NodeID{n(6, 0), n(5, 2), n(2, 24)}},
		{end: 513, want: []NodeID{n(9, 0), n(0, 512)}},
		{end: uint64(1) << 63, want: []NodeID{n(63, 0)}},
		{end: (uint64(1) << 63) + (uint64(1) << 57), want: []NodeID{n(63, 0), n(57, 64)}},
		// Only left border.
		{begin: 0, end: 16, want: []NodeID{n(4, 0)}},
		{begin: 1, end: 16, want: []NodeID{n(0, 1), n(1, 1), n(2, 1), n(3, 1)}},
		{begin: 2, end: 16, want: []NodeID{n(1, 1), n(2, 1), n(3, 1)}},
		{begin: 3, end: 16, want: []NodeID{n(0, 3), n(2, 1), n(3, 1)}},
		{begin: 4, end: 16, want: []NodeID{n(2, 1), n(3, 1)}},
		{begin: 6, end: 16, want: []NodeID{n(1, 3), n(3, 1)}},
		{begin: 8, end: 16, want: []NodeID{n(3, 1)}},
		{begin: 11, end: 16, want: []NodeID{n(0, 11), n(2, 3)}},
		// Two-sided.
		{begin: 1, end: 31, want: []NodeID{n(0, 1), n(1, 1), n(2, 1), n(3, 1), n(3, 2), n(2, 6), n(1, 14), n(0, 30)}},
		{begin: 1, end: 17, want: []NodeID{n(0, 1), n(1, 1), n(2, 1), n(3, 1), n(0, 16)}},
	} {
		t.Run(fmt.Sprintf("range:%d:%d", tc.begin, tc.end), func(t *testing.T) {
			if got, want := RangeNodes(tc.begin, tc.end), tc.want; !reflect.DeepEqual(got, tc.want) {
				t.Fatalf("RangeNodes: got %v, want %v", got, want)
			}
		})
	}
}
<commit_msg>compact: Add generated test for RangeNodes<commit_after>// Copyright 2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compact

import (
	"fmt"
	"testing"

	"github.com/google/go-cmp/cmp"
)

func TestRangeNodes(t *testing.T) {
	n := func(level uint, index uint64) NodeID {
		return NewNodeID(level, index)
	}
	for _, tc := range []struct {
		begin uint64
		end   uint64
		want  []NodeID
	}{
		// Empty ranges.
		{end: 0, want: []NodeID{}},
		{begin: 10, end: 10, want: []NodeID{}},
		{begin: 1024, end: 1024, want: []NodeID{}},
		// One entry.
		{begin: 10, end: 11, want: []NodeID{n(0, 10)}},
		{begin: 1024, end: 1025, want: []NodeID{n(0, 1024)}},
		{begin: 1025, end: 1026, want: []NodeID{n(0, 1025)}},
		// Two entries.
		{begin: 10, end: 12, want: []NodeID{n(1, 5)}},
		{begin: 1024, end: 1026, want: []NodeID{n(1, 512)}},
		{begin: 1025, end: 1027, want: []NodeID{n(0, 1025), n(0, 1026)}},
		// Only right border.
		{end: 1, want: []NodeID{n(0, 0)}},
		{end: 2, want: []NodeID{n(1, 0)}},
		{end: 3, want: []NodeID{n(1, 0), n(0, 2)}},
		{end: 4, want: []NodeID{n(2, 0)}},
		{end: 5, want: []NodeID{n(2, 0), n(0, 4)}},
		{end: 15, want: []NodeID{n(3, 0), n(2, 2), n(1, 6), n(0, 14)}},
		{end: 100, want: []NodeID{n(6, 0), n(5, 2), n(2, 24)}},
		{end: 513, want: []NodeID{n(9, 0), n(0, 512)}},
		{end: uint64(1) << 63, want: []NodeID{n(63, 0)}},
		{end: (uint64(1) << 63) + (uint64(1) << 57), want: []NodeID{n(63, 0), n(57, 64)}},
		// Only left border.
		{begin: 0, end: 16, want: []NodeID{n(4, 0)}},
		{begin: 1, end: 16, want: []NodeID{n(0, 1), n(1, 1), n(2, 1), n(3, 1)}},
		{begin: 2, end: 16, want: []NodeID{n(1, 1), n(2, 1), n(3, 1)}},
		{begin: 3, end: 16, want: []NodeID{n(0, 3), n(2, 1), n(3, 1)}},
		{begin: 4, end: 16, want: []NodeID{n(2, 1), n(3, 1)}},
		{begin: 6, end: 16, want: []NodeID{n(1, 3), n(3, 1)}},
		{begin: 8, end: 16, want: []NodeID{n(3, 1)}},
		{begin: 11, end: 16, want: []NodeID{n(0, 11), n(2, 3)}},
		// Two-sided.
		{begin: 1, end: 31, want: []NodeID{n(0, 1), n(1, 1), n(2, 1), n(3, 1), n(3, 2), n(2, 6), n(1, 14), n(0, 30)}},
		{begin: 1, end: 17, want: []NodeID{n(0, 1), n(1, 1), n(2, 1), n(3, 1), n(0, 16)}},
	} {
		t.Run(fmt.Sprintf("range:%d:%d", tc.begin, tc.end), func(t *testing.T) {
			got := RangeNodes(tc.begin, tc.end)
			if diff := cmp.Diff(got, tc.want); diff != "" {
				t.Fatalf("RangeNodes: diff(-want +got):\n%s", diff)
			}
		})
	}
}

func TestGenNodeRanges(t *testing.T) {
	const size = uint64(512)
	for begin := uint64(0); begin <= size; begin++ {
		for end := begin; end <= size; end++ {
			got := RangeNodes(begin, end)
			want := refRangeNodes(NewNodeID(63, 0), begin, end)
			if diff := cmp.Diff(got, want); diff != "" {
				t.Fatalf("RangeNodes(%d, %d): diff(-want +got):\n%s", begin, end, diff)
			}
		}
	}
}

// refRangeNodes returns node IDs that comprise the [begin, end) compact range.
// This is a reference implementation for cross-cthehecking.
func refRangeNodes(root NodeID, begin, end uint64) []NodeID {
	b, e := root.Coverage()
	if end <= b || begin >= e {
		return []NodeID{}
	}
	if b >= begin && e <= end {
		return []NodeID{root}
	}
	return append(
		refRangeNodes(NewNodeID(root.Level-1, root.Index*2), begin, end),
		refRangeNodes(NewNodeID(root.Level-1, root.Index*2+1), begin, end)...)
}
<|endoftext|>


<commit_before>package allyourbase

import (
	"fmt"
	"math"
)

func ConvertToBase(inputBase int, inputDigits []int, outputBase int) (outputDigits []int, e error) {
	base10 := getBase10Input(inputBase, inputDigits)
	if base10 == 0 {
		return []int{0}, nil
	}
	for base10 > 0 {
		digit := base10 % outputBase
		outputDigits = append([]int{digit}, outputDigits...)
		base10 = base10 / outputBase
	}
	return outputDigits, nil
}

func getBase10Input(inputBase int, inputDigits []int) (base10Input int) {
	for i, digit := range reverse(inputDigits) {
		base10Input += powInt(inputBase, i) * digit
	}
	fmt.Printf("getBase10Input(%d, %v)=%d\n", inputBase, inputDigits, base10Input)
	return base10Input
}

func reverse(input []int) (reversed []int) {
	for i := len(input) - 1; i >= 0; i-- {
		reversed = append(reversed, input[i])
	}
	return reversed
}

func powInt(x, y int) int {
	return int(math.Pow(float64(x), float64(y)))
}
<commit_msg>Return error for invalid base<commit_after>package allyourbase

import (
	"errors"
	"fmt"
	"math"
)

func ConvertToBase(inputBase int, inputDigits []int, outputBase int) (outputDigits []int, e error) {
	if inputBase < 2 {
		return []int{}, errors.New("input base must be >= 2")
	}
	base10 := getBase10Input(inputBase, inputDigits)
	if base10 == 0 {
		return []int{0}, nil
	}
	for base10 > 0 {
		digit := base10 % outputBase
		outputDigits = append([]int{digit}, outputDigits...)
		base10 = base10 / outputBase
	}
	return outputDigits, nil
}

func getBase10Input(inputBase int, inputDigits []int) (base10Input int) {
	for i, digit := range reverse(inputDigits) {
		base10Input += powInt(inputBase, i) * digit
	}
	fmt.Printf("getBase10Input(%d, %v)=%d\n", inputBase, inputDigits, base10Input)
	return base10Input
}

func reverse(input []int) (reversed []int) {
	for i := len(input) - 1; i >= 0; i-- {
		reversed = append(reversed, input[i])
	}
	return reversed
}

func powInt(x, y int) int {
	return int(math.Pow(float64(x), float64(y)))
}
<|endoftext|>


<commit_before>package isbn

import (
	"log"
	"strconv"
	"strings"
	"unicode"
)

func IsValidISBN(isbn string) bool {
	isbnWithoutDashes := strings.ReplaceAll(isbn, "-", "")
	if !isValidISBN(isbnWithoutDashes) {
		return false
	}
	return isbnSum(isbnWithoutDashes)%11 == 0
}

func isbnSum(isbn string) (sum int) {
	for i, r := range isbn {
		value := valueForCheckDigit(string(r))
		multiplier := 10 - i
		sum += value * multiplier
	}
	log.Printf("isbnSum(%s): %d\n", isbn, sum)
	return sum
}

func valueForCheckDigit(r string) int {
	if r == "X" || r == "x" {
		return 10
	}
	value, err := strconv.Atoi(r)
	if err != nil {
		log.Fatalf("failed to convert %v to int", r)
	}
	return value
}

func isValidISBN(isbn string) bool {
	return isValidCheckDigit(rune(isbn[len(isbn)-1])) && isValidPrefix(isbn) && isValidLength(isbn)
}

func isValidCheckDigit(checkDigit rune) bool {
	return unicode.IsDigit(checkDigit) || checkDigit == 'X'
}

// Prefix in this context is the entire ISBN excluding the check digit
func isValidPrefix(isbn string) bool {
	for _, c := range isbn[0 : len(isbn)-1] {
		if c != '-' && !unicode.IsDigit(c) {
			return false
		}
	}
	return true
}

func isValidLength(isbn string) bool {
	return len(isbn) == 10
}
<commit_msg>Solve isbn verifier<commit_after>package isbn

import (
	"log"
	"strconv"
	"strings"
	"unicode"
)

func IsValidISBN(isbn string) bool {
	isbnWithoutDashes := strings.ReplaceAll(isbn, "-", "")
	if !isValidISBN(isbnWithoutDashes) {
		return false
	}
	return isbnSum(isbnWithoutDashes)%11 == 0
}

func isbnSum(isbn string) (sum int) {
	for i, r := range isbn {
		value := valueForCheckDigit(string(r))
		multiplier := 10 - i
		sum += value * multiplier
	}
	log.Printf("isbnSum(%s): %d\n", isbn, sum)
	return sum
}

func valueForCheckDigit(r string) int {
	if r == "X" || r == "x" {
		return 10
	}
	value, err := strconv.Atoi(r)
	if err != nil {
		log.Fatalf("failed to convert %v to int", r)
	}
	return value
}

func isValidISBN(isbn string) bool {
	return isValidLength(isbn) && isValidCheckDigit(isbn) && isValidPrefix(isbn)
}

func isValidCheckDigit(isbn string) bool {
	if len(isbn) < 1 {
		return false
	}
	r := rune(isbn[len(isbn)-1])
	return unicode.IsDigit(r) || r == 'X'
}

// Prefix in this context is the entire ISBN excluding the check digit
func isValidPrefix(isbn string) bool {
	for _, c := range isbn[0 : len(isbn)-1] {
		if c != '-' && !unicode.IsDigit(c) {
			return false
		}
	}
	return true
}

func isValidLength(isbn string) bool {
	return len(isbn) == 10
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"

	"cloud.google.com/go/logging"

	topk "github.com/dgryski/go-topk"
	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"
)

type originAllower struct {
	m        map[string]struct{}
	ns       *expvar.Map
	hostname string
	gclog    logClient

	mu                 *sync.RWMutex
	topKAllDomains     *topk.Stream
	topKOfflistDomains *topk.Stream
}

type logClient interface {
	Log(logging.Entry)
	Flush()
}

func newOriginAllower(blockedDomains []string, hostname string, gclog logClient, ns *expvar.Map) *originAllower {
	mu := &sync.RWMutex{}
	topKAllDomains := topk.New(100)
	topKOfflistDomains := topk.New(100)
	lifetime := new(expvar.Map).Init()
	ns.Set("lifetime", lifetime)
	lifetime.Set("top_all_domains", expvar.Func(func() interface{} {
		mu.RLock()
		defer mu.RUnlock()
		return topKAllDomains.Keys()
	}))
	lifetime.Set("top_offlist_domains", expvar.Func(func() interface{} {
		mu.RLock()
		defer mu.RUnlock()
		return topKOfflistDomains.Keys()
	}))

	oa := &originAllower{
		m:                  make(map[string]struct{}),
		ns:                 ns,
		hostname:           hostname,
		gclog:              gclog,
		mu:                 mu,
		topKAllDomains:     topKAllDomains,
		topKOfflistDomains: topKOfflistDomains,
	}
	for _, d := range blockedDomains {
		oa.m[d] = struct{}{}
	}
	return oa
}

func (oa *originAllower) Allow(r *http.Request) (string, bool) {
	origin := r.Header.Get("Origin")
	referrer := r.Header.Get("Referer")

	apiKey := r.FormValue("key")
	userAgent := r.Header.Get("User-Agent")

	remoteIP, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		log.Printf("error splitting %#v as host:port: %s", r.RemoteAddr, err)
		remoteIP = "0.0.0.0"
	}
	entry := &apiLogEntry{
		DetectedDomain: "",
		Allowed:        false,
		APIKey:         apiKey,
		Headers: headers{
			Origin:    origin,
			Referrer:  referrer,
			UserAgent: userAgent,
		},
	}
	defer func() {
		go oa.countRequest(entry, r, remoteIP)
	}()

	if origin == "" && referrer == "" {
		entry.Allowed = true
		return "", true
	}
	if origin != "" {
		domain, ok := oa.checkDomain(origin)
		entry.DetectedDomain = domain
		entry.Allowed = ok
		if !ok {
			entry.RejectionReason = rejectionConfig
		}
		return domain, ok
	}
	if referrer != "" {
		domain, ok := oa.checkDomain(referrer)
		entry.DetectedDomain = domain
		entry.Allowed = ok
		if !ok {
			entry.RejectionReason = rejectionConfig
		}
		return domain, ok
	}

	return "", false
}

// checkDomain checks if the detected domain from the request headers and
// whether domain is allowed to make requests against howsmyssl's API.
func (oa *originAllower) checkDomain(d string) (string, bool) {
	domain, err := effectiveDomain(d)
	if err != nil {
		// TODO(jmhodges): replace this len check with false when we use top-k
		return "", len(oa.m) == 0
	}
	_, isBlocked := oa.m[domain]
	// TODO(jmhodges): remove this len check when we use top-k
	return domain, !isBlocked || len(oa.m) == 0
}

func (oa *originAllower) countRequest(entry *apiLogEntry, r *http.Request, remoteIP string) {
	oa.gclog.Log(logging.Entry{
		Payload:     entry,
		HTTPRequest: &logging.HTTPRequest{Request: r, RemoteIP: remoteIP},
		Labels: map[string]string{
			"server_hostname": oa.hostname,
			"app":             "howsmyssl",
		},
	})

	if entry.DetectedDomain == "" {
		return
	}

	oa.mu.Lock()
	defer oa.mu.Unlock()
	oa.topKAllDomains.Insert(entry.DetectedDomain, 1)
	if !entry.Allowed {
		oa.topKOfflistDomains.Insert(entry.DetectedDomain, 1)
	}
}

func effectiveDomain(str string) (string, error) {
	u, err := url.Parse(str)
	if err != nil {
		return "", err
	}
	host := u.Host
	if host == "" {
		return "", fmt.Errorf("unparsable domain string %#v", str)
	}
	i := strings.Index(host, ":")
	if i >= 0 {
		host = host[:i]
	}

	if host == "localhost" {
		return "localhost", nil
	}
	d, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		return "", err
	}
	return d, nil
}

func loadOriginsConfig(fp string) *originsConfig {
	f, err := os.Open(fp)
	if err != nil {
		log.Fatalf("unable to open origins config file %#v: %s", fp, err)
	}
	defer f.Close()
	jc := &originsConfig{}
	err = json.NewDecoder(f).Decode(jc)
	if err != nil {
		log.Fatalf("unable to parse origins config file %#v: %s", fp, err)
	}
	for _, a := range jc.BlockedOrigins {
		if strings.HasPrefix(a, "http://") || strings.HasPrefix(a, "https://") {
			log.Fatalf("origins config file (%#v) should have only domains without the leading scheme. For example, %#v should not have the protocol scheme at its beginning.", fp, a)
		}
		if strings.Contains(a, "/") {
			log.Fatalf("origins config file (%#v) should have only domains without a path after it. For example, %#v should not have a trailing path.", fp, a)
		}
	}
	return jc
}

type originsConfig struct {
	// BlockedOrigins are domains that are not to be allowed as referrers to the
	// API. They should not have a scheme or path, but only the domain, as in
	// "example.com".
	BlockedOrigins []string `json:"blocked_origins"`
}

type rejectionReason string

const rejectionConfig = rejectionReason("config")

type apiLogEntry struct {
	DetectedDomain  string          `json:"detected_domain"`
	Allowed         bool            `json:"allowed"`
	APIKey          string          `json:"api_key"`
	RejectionReason rejectionReason `json:"rejection_reason"`
	Headers         headers         `json:"headers"`
}

type headers struct {
	Origin    string `json:"origin"`
	Referrer  string `json:"referrer"`
	UserAgent string `json:"user_agent"`
}

func loadGoogleServiceAccount(fp string) *googleConfig {
	bs, err := ioutil.ReadFile(fp)
	if err != nil {
		log.Fatalf("unable to read Google service account config %#v: %s", fp, err)
	}
	c := &googleConfig{}
	err = json.Unmarshal(bs, c)
	if err != nil {
		log.Fatalf("unable to parse project ID from Google service account config %#v: %s", fp, err)
	}
	if c.ProjectID == "" {
		log.Fatalf("blank project ID in Google service account config %#v: %s", fp, err)
	}
	jwtConf, err := google.JWTConfigFromJSON(bs, logging.WriteScope)
	if err != nil {
		log.Fatalf("unable to parse Google service account config %#v: %s", fp, err)
	}
	c.conf = jwtConf
	return c
}

type googleConfig struct {
	ProjectID string `json:"project_id"`

	conf *jwt.Config
}

var _ logClient = nullLogClient{}

type nullLogClient struct{}

func (n nullLogClient) Log(e logging.Entry) {
}

func (n nullLogClient) Flush() {
}
<commit_msg>log full detected domain, not just the eTLD+1 (#181)<commit_after>package main

import (
	"encoding/json"
	"expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"

	"cloud.google.com/go/logging"

	topk "github.com/dgryski/go-topk"
	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2/google"
	"golang.org/x/oauth2/jwt"
)

type originAllower struct {
	m        map[string]struct{}
	ns       *expvar.Map
	hostname string
	gclog    logClient

	mu                 *sync.RWMutex
	topKAllDomains     *topk.Stream
	topKOfflistDomains *topk.Stream
}

type logClient interface {
	Log(logging.Entry)
	Flush()
}

func newOriginAllower(blockedDomains []string, hostname string, gclog logClient, ns *expvar.Map) *originAllower {
	mu := &sync.RWMutex{}
	topKAllDomains := topk.New(100)
	topKOfflistDomains := topk.New(100)
	lifetime := new(expvar.Map).Init()
	ns.Set("lifetime", lifetime)
	lifetime.Set("top_all_domains", expvar.Func(func() interface{} {
		mu.RLock()
		defer mu.RUnlock()
		return topKAllDomains.Keys()
	}))
	lifetime.Set("top_offlist_domains", expvar.Func(func() interface{} {
		mu.RLock()
		defer mu.RUnlock()
		return topKOfflistDomains.Keys()
	}))

	oa := &originAllower{
		m:                  make(map[string]struct{}),
		ns:                 ns,
		hostname:           hostname,
		gclog:              gclog,
		mu:                 mu,
		topKAllDomains:     topKAllDomains,
		topKOfflistDomains: topKOfflistDomains,
	}
	for _, d := range blockedDomains {
		oa.m[d] = struct{}{}
	}
	return oa
}

func (oa *originAllower) Allow(r *http.Request) (string, bool) {
	origin := r.Header.Get("Origin")
	referrer := r.Header.Get("Referer")

	apiKey := r.FormValue("key")
	userAgent := r.Header.Get("User-Agent")

	remoteIP, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		log.Printf("error splitting %#v as host:port: %s", r.RemoteAddr, err)
		remoteIP = "0.0.0.0"
	}
	entry := &apiLogEntry{
		DetectedDomain:     "",
		DetectedFullDomain: "",
		Allowed:            false,
		APIKey:             apiKey,
		Headers: headers{
			Origin:    origin,
			Referrer:  referrer,
			UserAgent: userAgent,
		},
	}
	defer func() {
		go oa.countRequest(entry, r, remoteIP)
	}()

	if origin == "" && referrer == "" {
		entry.Allowed = true
		return "", true
	}
	if origin != "" {
		domain, fullDomain, ok := oa.checkDomain(origin)
		entry.DetectedDomain = domain
		entry.DetectedFullDomain = fullDomain
		entry.Allowed = ok
		if !ok {
			entry.RejectionReason = rejectionConfig
		}
		return domain, ok
	}
	if referrer != "" {
		domain, fullDomain, ok := oa.checkDomain(referrer)
		entry.DetectedDomain = domain
		entry.DetectedFullDomain = fullDomain
		entry.Allowed = ok
		if !ok {
			entry.RejectionReason = rejectionConfig
		}
		return domain, ok
	}

	return "", false
}

// checkDomain checks if the detected domain from the request headers and
// whether domain is allowed to make requests against howsmyssl's API.
func (oa *originAllower) checkDomain(d string) (string, string, bool) {
	domain, fullDomain, err := effectiveDomain(d)
	if err != nil {
		// TODO(jmhodges): replace this len check with false when we use top-k
		return "", "", len(oa.m) == 0
	}
	_, isBlocked := oa.m[domain]
	// TODO(jmhodges): remove this len check when we use top-k
	return domain, fullDomain, !isBlocked || len(oa.m) == 0
}

func (oa *originAllower) countRequest(entry *apiLogEntry, r *http.Request, remoteIP string) {
	oa.gclog.Log(logging.Entry{
		Payload:     entry,
		HTTPRequest: &logging.HTTPRequest{Request: r, RemoteIP: remoteIP},
		Labels: map[string]string{
			"server_hostname": oa.hostname,
			"app":             "howsmyssl",
		},
	})

	if entry.DetectedDomain == "" {
		return
	}

	oa.mu.Lock()
	defer oa.mu.Unlock()
	oa.topKAllDomains.Insert(entry.DetectedDomain, 1)
	if !entry.Allowed {
		oa.topKOfflistDomains.Insert(entry.DetectedDomain, 1)
	}
}

func effectiveDomain(str string) (string, string, error) {
	u, err := url.Parse(str)
	if err != nil {
		return "", "", err
	}
	host := u.Host
	if host == "" {
		return "", "", fmt.Errorf("unparsable domain string %#v", str)
	}
	i := strings.Index(host, ":")
	if i >= 0 {
		host = host[:i]
	}

	if host == "localhost" {
		return "localhost", "localhost", nil
	}
	d, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		return "", "", err
	}
	return d, host, nil
}

func loadOriginsConfig(fp string) *originsConfig {
	f, err := os.Open(fp)
	if err != nil {
		log.Fatalf("unable to open origins config file %#v: %s", fp, err)
	}
	defer f.Close()
	jc := &originsConfig{}
	err = json.NewDecoder(f).Decode(jc)
	if err != nil {
		log.Fatalf("unable to parse origins config file %#v: %s", fp, err)
	}
	for _, a := range jc.BlockedOrigins {
		if strings.HasPrefix(a, "http://") || strings.HasPrefix(a, "https://") {
			log.Fatalf("origins config file (%#v) should have only domains without the leading scheme. For example, %#v should not have the protocol scheme at its beginning.", fp, a)
		}
		if strings.Contains(a, "/") {
			log.Fatalf("origins config file (%#v) should have only domains without a path after it. For example, %#v should not have a trailing path.", fp, a)
		}
	}
	return jc
}

type originsConfig struct {
	// BlockedOrigins are domains that are not to be allowed as referrers to the
	// API. They should not have a scheme or path, but only the domain, as in
	// "example.com".
	BlockedOrigins []string `json:"blocked_origins"`
}

type rejectionReason string

const rejectionConfig = rejectionReason("config")

type apiLogEntry struct {
	DetectedDomain     string          `json:"detected_domain"`
	DetectedFullDomain string          `json:"detected_full_domain"`
	Allowed            bool            `json:"allowed"`
	APIKey             string          `json:"api_key"`
	RejectionReason    rejectionReason `json:"rejection_reason"`
	ReferrerHost       string          `json:referrer_host"`
	Headers            headers         `json:"headers"`
}

type headers struct {
	Origin    string `json:"origin"`
	Referrer  string `json:"referrer"`
	UserAgent string `json:"user_agent"`
}

func loadGoogleServiceAccount(fp string) *googleConfig {
	bs, err := ioutil.ReadFile(fp)
	if err != nil {
		log.Fatalf("unable to read Google service account config %#v: %s", fp, err)
	}
	c := &googleConfig{}
	err = json.Unmarshal(bs, c)
	if err != nil {
		log.Fatalf("unable to parse project ID from Google service account config %#v: %s", fp, err)
	}
	if c.ProjectID == "" {
		log.Fatalf("blank project ID in Google service account config %#v: %s", fp, err)
	}
	jwtConf, err := google.JWTConfigFromJSON(bs, logging.WriteScope)
	if err != nil {
		log.Fatalf("unable to parse Google service account config %#v: %s", fp, err)
	}
	c.conf = jwtConf
	return c
}

type googleConfig struct {
	ProjectID string `json:"project_id"`

	conf *jwt.Config
}

var _ logClient = nullLogClient{}

type nullLogClient struct{}

func (n nullLogClient) Log(e logging.Entry) {
}

func (n nullLogClient) Flush() {
}
<|endoftext|>


<commit_before>package grandcentral

import (
	"fmt"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	dhtpb "github.com/jbenet/go-ipfs/routing/dht/pb"
	proxy "github.com/jbenet/go-ipfs/routing/grandcentral/proxy"
	util "github.com/jbenet/go-ipfs/util"
	errors "github.com/jbenet/go-ipfs/util/debugerror"
)

// Server handles routing queries using a database backend
type Server struct {
	local           peer.ID
	routingBackend  datastore.ThreadSafeDatastore
	peerstore       peer.Peerstore
	*proxy.Loopback // so server can be injected into client
}

// NewServer creates a new GrandCentral routing Server
func NewServer(ds datastore.ThreadSafeDatastore, ps peer.Peerstore, local peer.ID) (*Server, error) {
	s := &Server{local, ds, ps, nil}
	s.Loopback = &proxy.Loopback{
		Handler: s,
		Local:   local,
	}
	return s, nil
}

// HandleLocalRequest implements the proxy.RequestHandler interface. This is
// where requests are received from the outside world.
func (s *Server) HandleRequest(ctx context.Context, p peer.ID, req *dhtpb.Message) *dhtpb.Message {
	_, response := s.handleMessage(ctx, p, req) // ignore response peer. it's local.
	return response
}

func (s *Server) handleMessage(
	ctx context.Context, p peer.ID, req *dhtpb.Message) (peer.ID, *dhtpb.Message) {

	defer log.EventBegin(ctx, "routingMessageReceived", req, p).Done()

	var response = dhtpb.NewMessage(req.GetType(), req.GetKey(), req.GetClusterLevel())
	switch req.GetType() {

	case dhtpb.Message_GET_VALUE:
		rawRecord, err := getRoutingRecord(s.routingBackend, util.Key(req.GetKey()))
		if err != nil {
			return "", nil
		}
		response.Record = rawRecord
		// TODO before merging: if we know any providers for the requested value, return those.
		return p, response

	case dhtpb.Message_PUT_VALUE:
		// TODO before merging: verifyRecord(req.GetRecord())
		putRoutingRecord(s.routingBackend, util.Key(req.GetKey()), req.GetRecord())
		return p, req // TODO before merging: verify that we should return record

	case dhtpb.Message_FIND_NODE:
		p := s.peerstore.PeerInfo(peer.ID(req.GetKey()))
		pri := []dhtpb.PeerRoutingInfo{
			dhtpb.PeerRoutingInfo{
				PeerInfo: p,
				// Connectedness: TODO
			},
		}
		response.CloserPeers = dhtpb.PeerRoutingInfosToPBPeers(pri)
		return p.ID, response

	case dhtpb.Message_ADD_PROVIDER:
		// FIXME(btc): do we want to store these locally? I think the
		// storeProvidersToPeerstore behavior is straight from the DHT message
		// handler.
		storeProvidersToPeerstore(s.peerstore, p, req.GetProviderPeers())

		if err := putRoutingProviders(s.routingBackend, util.Key(req.GetKey()), req.GetProviderPeers()); err != nil {
			return "", nil
		}
		return "", nil

	case dhtpb.Message_GET_PROVIDERS:
		providers, err := getRoutingProviders(s.routingBackend, util.Key(req.GetKey()))
		if err != nil {
			return "", nil
		}
		response.ProviderPeers = providers
		return p, response

	case dhtpb.Message_PING:
		return p, req
	default:
	}
	return "", nil
}

var _ proxy.RequestHandler = &Server{}
var _ proxy.Proxy = &Server{}

func getRoutingRecord(ds datastore.Datastore, k util.Key) (*dhtpb.Record, error) {
	dskey := k.DsKey()
	val, err := ds.Get(dskey)
	if err != nil {
		return nil, errors.Wrap(err)
	}
	recordBytes, ok := val.([]byte)
	if !ok {
		return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
	}
	var record dhtpb.Record
	if err := proto.Unmarshal(recordBytes, &record); err != nil {
		return nil, errors.New("failed to unmarshal dht record from datastore")
	}
	return &record, nil
}

func putRoutingRecord(ds datastore.Datastore, k util.Key, value *dhtpb.Record) error {
	data, err := proto.Marshal(value)
	if err != nil {
		return err
	}
	dskey := k.DsKey()
	// TODO namespace
	if err := ds.Put(dskey, data); err != nil {
		return err
	}
	return nil
}

func putRoutingProviders(ds datastore.Datastore, k util.Key, newRecords []*dhtpb.Message_Peer) error {
	log.Event(context.Background(), "putRoutingProviders", &k)
	oldRecords, err := getRoutingProviders(ds, k)
	if err != nil {
		return err
	}
	mergedRecords := make(map[string]*dhtpb.Message_Peer)
	for _, provider := range oldRecords {
		mergedRecords[provider.GetId()] = provider // add original records
	}
	for _, provider := range newRecords {
		mergedRecords[provider.GetId()] = provider // overwrite old record if new exists
	}
	var protomsg dhtpb.Message
	protomsg.ProviderPeers = make([]*dhtpb.Message_Peer, 0, len(mergedRecords))
	for _, provider := range mergedRecords {
		protomsg.ProviderPeers = append(protomsg.ProviderPeers, provider)
	}
	data, err := proto.Marshal(&protomsg)
	if err != nil {
		return err
	}
	return ds.Put(providerKey(k), data)
}

func storeProvidersToPeerstore(ps peer.Peerstore, p peer.ID, providers []*dhtpb.Message_Peer) {
	for _, provider := range providers {
		providerID := peer.ID(provider.GetId())
		if providerID != p {
			log.Errorf("provider message came from third-party %s", p)
			continue
		}
		for _, maddr := range provider.Addresses() {
			// as a router, we want to store addresses for peers who have provided
			ps.AddAddr(p, maddr, peer.AddressTTL)
		}
	}
}

func getRoutingProviders(ds datastore.Datastore, k util.Key) ([]*dhtpb.Message_Peer, error) {
	e := log.EventBegin(context.Background(), "getProviders", &k)
	defer e.Done()
	var providers []*dhtpb.Message_Peer
	if v, err := ds.Get(providerKey(k)); err == nil {
		if data, ok := v.([]byte); ok {
			var msg dhtpb.Message
			if err := proto.Unmarshal(data, &msg); err != nil {
				return nil, err
			}
			providers = append(providers, msg.GetProviderPeers()...)
		}
	}
	return providers, nil
}

func providerKey(k util.Key) datastore.Key {
	return datastore.KeyWithNamespaces([]string{"routing", "providers", k.String()})
}
<commit_msg>remove TODO<commit_after>package grandcentral

import (
	"fmt"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	dhtpb "github.com/jbenet/go-ipfs/routing/dht/pb"
	proxy "github.com/jbenet/go-ipfs/routing/grandcentral/proxy"
	util "github.com/jbenet/go-ipfs/util"
	errors "github.com/jbenet/go-ipfs/util/debugerror"
)

// Server handles routing queries using a database backend
type Server struct {
	local           peer.ID
	routingBackend  datastore.ThreadSafeDatastore
	peerstore       peer.Peerstore
	*proxy.Loopback // so server can be injected into client
}

// NewServer creates a new GrandCentral routing Server
func NewServer(ds datastore.ThreadSafeDatastore, ps peer.Peerstore, local peer.ID) (*Server, error) {
	s := &Server{local, ds, ps, nil}
	s.Loopback = &proxy.Loopback{
		Handler: s,
		Local:   local,
	}
	return s, nil
}

// HandleLocalRequest implements the proxy.RequestHandler interface. This is
// where requests are received from the outside world.
func (s *Server) HandleRequest(ctx context.Context, p peer.ID, req *dhtpb.Message) *dhtpb.Message {
	_, response := s.handleMessage(ctx, p, req) // ignore response peer. it's local.
	return response
}

func (s *Server) handleMessage(
	ctx context.Context, p peer.ID, req *dhtpb.Message) (peer.ID, *dhtpb.Message) {

	defer log.EventBegin(ctx, "routingMessageReceived", req, p).Done()

	var response = dhtpb.NewMessage(req.GetType(), req.GetKey(), req.GetClusterLevel())
	switch req.GetType() {

	case dhtpb.Message_GET_VALUE:
		rawRecord, err := getRoutingRecord(s.routingBackend, util.Key(req.GetKey()))
		if err != nil {
			return "", nil
		}
		response.Record = rawRecord
		return p, response

	case dhtpb.Message_PUT_VALUE:
		// TODO before merging: verifyRecord(req.GetRecord())
		putRoutingRecord(s.routingBackend, util.Key(req.GetKey()), req.GetRecord())
		return p, req // TODO before merging: verify that we should return record

	case dhtpb.Message_FIND_NODE:
		p := s.peerstore.PeerInfo(peer.ID(req.GetKey()))
		pri := []dhtpb.PeerRoutingInfo{
			dhtpb.PeerRoutingInfo{
				PeerInfo: p,
				// Connectedness: TODO
			},
		}
		response.CloserPeers = dhtpb.PeerRoutingInfosToPBPeers(pri)
		return p.ID, response

	case dhtpb.Message_ADD_PROVIDER:
		// FIXME(btc): do we want to store these locally? I think the
		// storeProvidersToPeerstore behavior is straight from the DHT message
		// handler.
		storeProvidersToPeerstore(s.peerstore, p, req.GetProviderPeers())

		if err := putRoutingProviders(s.routingBackend, util.Key(req.GetKey()), req.GetProviderPeers()); err != nil {
			return "", nil
		}
		return "", nil

	case dhtpb.Message_GET_PROVIDERS:
		providers, err := getRoutingProviders(s.routingBackend, util.Key(req.GetKey()))
		if err != nil {
			return "", nil
		}
		response.ProviderPeers = providers
		return p, response

	case dhtpb.Message_PING:
		return p, req
	default:
	}
	return "", nil
}

var _ proxy.RequestHandler = &Server{}
var _ proxy.Proxy = &Server{}

func getRoutingRecord(ds datastore.Datastore, k util.Key) (*dhtpb.Record, error) {
	dskey := k.DsKey()
	val, err := ds.Get(dskey)
	if err != nil {
		return nil, errors.Wrap(err)
	}
	recordBytes, ok := val.([]byte)
	if !ok {
		return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
	}
	var record dhtpb.Record
	if err := proto.Unmarshal(recordBytes, &record); err != nil {
		return nil, errors.New("failed to unmarshal dht record from datastore")
	}
	return &record, nil
}

func putRoutingRecord(ds datastore.Datastore, k util.Key, value *dhtpb.Record) error {
	data, err := proto.Marshal(value)
	if err != nil {
		return err
	}
	dskey := k.DsKey()
	// TODO namespace
	if err := ds.Put(dskey, data); err != nil {
		return err
	}
	return nil
}

func putRoutingProviders(ds datastore.Datastore, k util.Key, newRecords []*dhtpb.Message_Peer) error {
	log.Event(context.Background(), "putRoutingProviders", &k)
	oldRecords, err := getRoutingProviders(ds, k)
	if err != nil {
		return err
	}
	mergedRecords := make(map[string]*dhtpb.Message_Peer)
	for _, provider := range oldRecords {
		mergedRecords[provider.GetId()] = provider // add original records
	}
	for _, provider := range newRecords {
		mergedRecords[provider.GetId()] = provider // overwrite old record if new exists
	}
	var protomsg dhtpb.Message
	protomsg.ProviderPeers = make([]*dhtpb.Message_Peer, 0, len(mergedRecords))
	for _, provider := range mergedRecords {
		protomsg.ProviderPeers = append(protomsg.ProviderPeers, provider)
	}
	data, err := proto.Marshal(&protomsg)
	if err != nil {
		return err
	}
	return ds.Put(providerKey(k), data)
}

func storeProvidersToPeerstore(ps peer.Peerstore, p peer.ID, providers []*dhtpb.Message_Peer) {
	for _, provider := range providers {
		providerID := peer.ID(provider.GetId())
		if providerID != p {
			log.Errorf("provider message came from third-party %s", p)
			continue
		}
		for _, maddr := range provider.Addresses() {
			// as a router, we want to store addresses for peers who have provided
			ps.AddAddr(p, maddr, peer.AddressTTL)
		}
	}
}

func getRoutingProviders(ds datastore.Datastore, k util.Key) ([]*dhtpb.Message_Peer, error) {
	e := log.EventBegin(context.Background(), "getProviders", &k)
	defer e.Done()
	var providers []*dhtpb.Message_Peer
	if v, err := ds.Get(providerKey(k)); err == nil {
		if data, ok := v.([]byte); ok {
			var msg dhtpb.Message
			if err := proto.Unmarshal(data, &msg); err != nil {
				return nil, err
			}
			providers = append(providers, msg.GetProviderPeers()...)
		}
	}
	return providers, nil
}

func providerKey(k util.Key) datastore.Key {
	return datastore.KeyWithNamespaces([]string{"routing", "providers", k.String()})
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"errors"
	"fmt"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/text"

	"go.chromium.org/luci/rts/filegraph"
)

var cmdQuery = &subcommands.Command{
	UsageLine: `query [flags] SOURCE_FILE [SOURCE_FILE...]`,
	ShortDesc: "print graph files in the distance-ascending order",
	LongDesc: text.Doc(`
		Print graph files in the distance-ascending order from SOURCE_FILEs.

		Each output line has format "<distance> <filename>",
		where the filename is forward-slash-separated and has "//" prefix.
		Example: "0.4 //foo/bar.cpp".

		All SOURCE_FILEs must be in the same git repository.
		Does not print unreachable files.
	`),
	CommandRun: func() subcommands.CommandRun {
		r := &queryRun{}
		r.git.RegisterFlags(&r.Flags)
		r.Flags.BoolVar(&r.git.q.Reversed, "reversed", false, "Follow incoming edges instead of outgoing")
		return r
	},
}

type queryRun struct {
	baseCommandRun
	git gitGraph
}

func (r *queryRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)
	if err := r.git.Validate(); err != nil {
		return r.done(err)
	}
	if len(args) == 0 {
		return r.done(errors.New("expected filenames as positional arguments"))
	}

	var err error
	if r.git.q.Sources, err = r.git.loadSyncedNodes(ctx, args...); err != nil {
		return r.done(err)
	}

	r.git.q.Run(func(sp *filegraph.ShortestPath) bool {
		fmt.Printf("%.2f %s\n", sp.Distance, sp.Node.Name())
		return ctx.Err() == nil
	})
	return 0
}
<commit_msg>[filegraph] Embed gitGraph in queryRun<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"errors"
	"fmt"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/text"

	"go.chromium.org/luci/rts/filegraph"
)

var cmdQuery = &subcommands.Command{
	UsageLine: `query [flags] SOURCE_FILE [SOURCE_FILE...]`,
	ShortDesc: "print graph files in the distance-ascending order",
	LongDesc: text.Doc(`
		Print graph files in the distance-ascending order from SOURCE_FILEs.

		Each output line has format "<distance> <filename>",
		where the filename is forward-slash-separated and has "//" prefix.
		Example: "0.4 //foo/bar.cpp".

		All SOURCE_FILEs must be in the same git repository.
		Does not print unreachable files.
	`),
	CommandRun: func() subcommands.CommandRun {
		r := &queryRun{}
		r.gitGraph.RegisterFlags(&r.Flags)
		r.Flags.BoolVar(&r.q.Reversed, "reversed", false, "Follow incoming edges instead of outgoing")
		return r
	},
}

type queryRun struct {
	baseCommandRun
	gitGraph
}

func (r *queryRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)
	if err := r.gitGraph.Validate(); err != nil {
		return r.done(err)
	}
	if len(args) == 0 {
		return r.done(errors.New("expected filenames as positional arguments"))
	}

	var err error
	if r.q.Sources, err = r.loadSyncedNodes(ctx, args...); err != nil {
		return r.done(err)
	}

	r.q.Run(func(sp *filegraph.ShortestPath) bool {
		fmt.Printf("%.2f %s\n", sp.Distance, sp.Node.Name())
		return ctx.Err() == nil
	})
	return 0
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"os"
	"syscall"

	"golang.org/x/sys/unix"
	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
	"gvisor.googlesource.com/gvisor/pkg/seccomp"
)

// allowedSyscalls is the set of syscalls executed by the gofer.
var allowedSyscalls = seccomp.SyscallRules{
	syscall.SYS_ACCEPT: {},
	syscall.SYS_ARCH_PRCTL: []seccomp.Rule{
		{seccomp.AllowValue(linux.ARCH_GET_FS)},
		{seccomp.AllowValue(linux.ARCH_SET_FS)},
	},
	syscall.SYS_CLOCK_GETTIME: {},
	syscall.SYS_CLONE: []seccomp.Rule{
		{
			seccomp.AllowValue(
				syscall.CLONE_VM |
					syscall.CLONE_FS |
					syscall.CLONE_FILES |
					syscall.CLONE_SIGHAND |
					syscall.CLONE_SYSVSEM |
					syscall.CLONE_THREAD),
		},
	},
	syscall.SYS_CLOSE:     {},
	syscall.SYS_DUP:       {},
	syscall.SYS_EPOLL_CTL: {},
	syscall.SYS_EPOLL_PWAIT: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(0),
		},
	},
	syscall.SYS_EVENTFD2: []seccomp.Rule{
		{
			seccomp.AllowValue(0),
			seccomp.AllowValue(0),
		},
	},
	syscall.SYS_EXIT:       {},
	syscall.SYS_EXIT_GROUP: {},
	syscall.SYS_FCHMOD:     {},
	syscall.SYS_FCHOWNAT:   {},
	syscall.SYS_FCNTL: []seccomp.Rule{
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.F_GETFL),
		},
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.F_SETFL),
		},
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.F_GETFD),
		},
	},
	syscall.SYS_FSTAT:     {},
	syscall.SYS_FSTATFS:   {},
	syscall.SYS_FSYNC:     {},
	syscall.SYS_FTRUNCATE: {},
	syscall.SYS_FUTEX: {
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG),
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(0),
		},
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG),
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(0),
		},
	},
	syscall.SYS_GETDENTS64:   {},
	syscall.SYS_GETPID:       {},
	unix.SYS_GETRANDOM:       {},
	syscall.SYS_GETTID:       {},
	syscall.SYS_GETTIMEOFDAY: {},
	syscall.SYS_LINKAT:       {},
	syscall.SYS_LSEEK:        {},
	syscall.SYS_MKDIRAT:      {},
	syscall.SYS_MMAP: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MAP_SHARED),
		},
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MAP_PRIVATE | syscall.MAP_ANONYMOUS),
		},
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MAP_PRIVATE | syscall.MAP_ANONYMOUS | syscall.MAP_FIXED),
		},
	},
	syscall.SYS_MPROTECT:   {},
	syscall.SYS_MUNMAP:     {},
	syscall.SYS_NANOSLEEP:  {},
	syscall.SYS_NEWFSTATAT: {},
	syscall.SYS_OPENAT:     {},
	syscall.SYS_POLL:       {},
	syscall.SYS_PREAD64:    {},
	syscall.SYS_PWRITE64:   {},
	syscall.SYS_READ:       {},
	syscall.SYS_READLINKAT: {},
	syscall.SYS_RECVMSG: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MSG_DONTWAIT | syscall.MSG_TRUNC),
		},
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MSG_DONTWAIT | syscall.MSG_TRUNC | syscall.MSG_PEEK),
		},
	},
	syscall.SYS_RENAMEAT:        {},
	syscall.SYS_RESTART_SYSCALL: {},
	syscall.SYS_RT_SIGPROCMASK:  {},
	syscall.SYS_SCHED_YIELD:     {},
	syscall.SYS_SENDMSG: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MSG_DONTWAIT | syscall.MSG_NOSIGNAL),
		},
	},
	syscall.SYS_SHUTDOWN: []seccomp.Rule{
		{seccomp.AllowAny{}, seccomp.AllowValue(syscall.SHUT_RDWR)},
	},
	syscall.SYS_SIGALTSTACK: {},
	syscall.SYS_SYMLINKAT:   {},
	syscall.SYS_TGKILL: []seccomp.Rule{
		{
			seccomp.AllowValue(uint64(os.Getpid())),
		},
	},
	syscall.SYS_UNLINKAT:  {},
	syscall.SYS_UTIMENSAT: {},
	syscall.SYS_WRITE:     {},
}
<commit_msg>Add MADVISE to fsgofer seccomp profile<commit_after>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"os"
	"syscall"

	"golang.org/x/sys/unix"
	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
	"gvisor.googlesource.com/gvisor/pkg/seccomp"
)

// allowedSyscalls is the set of syscalls executed by the gofer.
var allowedSyscalls = seccomp.SyscallRules{
	syscall.SYS_ACCEPT: {},
	syscall.SYS_ARCH_PRCTL: []seccomp.Rule{
		{seccomp.AllowValue(linux.ARCH_GET_FS)},
		{seccomp.AllowValue(linux.ARCH_SET_FS)},
	},
	syscall.SYS_CLOCK_GETTIME: {},
	syscall.SYS_CLONE: []seccomp.Rule{
		{
			seccomp.AllowValue(
				syscall.CLONE_VM |
					syscall.CLONE_FS |
					syscall.CLONE_FILES |
					syscall.CLONE_SIGHAND |
					syscall.CLONE_SYSVSEM |
					syscall.CLONE_THREAD),
		},
	},
	syscall.SYS_CLOSE:     {},
	syscall.SYS_DUP:       {},
	syscall.SYS_EPOLL_CTL: {},
	syscall.SYS_EPOLL_PWAIT: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(0),
		},
	},
	syscall.SYS_EVENTFD2: []seccomp.Rule{
		{
			seccomp.AllowValue(0),
			seccomp.AllowValue(0),
		},
	},
	syscall.SYS_EXIT:       {},
	syscall.SYS_EXIT_GROUP: {},
	syscall.SYS_FCHMOD:     {},
	syscall.SYS_FCHOWNAT:   {},
	syscall.SYS_FCNTL: []seccomp.Rule{
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.F_GETFL),
		},
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.F_SETFL),
		},
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.F_GETFD),
		},
	},
	syscall.SYS_FSTAT:     {},
	syscall.SYS_FSTATFS:   {},
	syscall.SYS_FSYNC:     {},
	syscall.SYS_FTRUNCATE: {},
	syscall.SYS_FUTEX: {
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG),
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(0),
		},
		seccomp.Rule{
			seccomp.AllowAny{},
			seccomp.AllowValue(linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG),
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(0),
		},
	},
	syscall.SYS_GETDENTS64:   {},
	syscall.SYS_GETPID:       {},
	unix.SYS_GETRANDOM:       {},
	syscall.SYS_GETTID:       {},
	syscall.SYS_GETTIMEOFDAY: {},
	syscall.SYS_LINKAT:       {},
	syscall.SYS_LSEEK:        {},
	syscall.SYS_MADVISE:      {},
	syscall.SYS_MKDIRAT:      {},
	syscall.SYS_MMAP: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MAP_SHARED),
		},
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MAP_PRIVATE | syscall.MAP_ANONYMOUS),
		},
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MAP_PRIVATE | syscall.MAP_ANONYMOUS | syscall.MAP_FIXED),
		},
	},
	syscall.SYS_MPROTECT:   {},
	syscall.SYS_MUNMAP:     {},
	syscall.SYS_NANOSLEEP:  {},
	syscall.SYS_NEWFSTATAT: {},
	syscall.SYS_OPENAT:     {},
	syscall.SYS_POLL:       {},
	syscall.SYS_PREAD64:    {},
	syscall.SYS_PWRITE64:   {},
	syscall.SYS_READ:       {},
	syscall.SYS_READLINKAT: {},
	syscall.SYS_RECVMSG: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MSG_DONTWAIT | syscall.MSG_TRUNC),
		},
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MSG_DONTWAIT | syscall.MSG_TRUNC | syscall.MSG_PEEK),
		},
	},
	syscall.SYS_RENAMEAT:        {},
	syscall.SYS_RESTART_SYSCALL: {},
	syscall.SYS_RT_SIGPROCMASK:  {},
	syscall.SYS_SCHED_YIELD:     {},
	syscall.SYS_SENDMSG: []seccomp.Rule{
		{
			seccomp.AllowAny{},
			seccomp.AllowAny{},
			seccomp.AllowValue(syscall.MSG_DONTWAIT | syscall.MSG_NOSIGNAL),
		},
	},
	syscall.SYS_SHUTDOWN: []seccomp.Rule{
		{seccomp.AllowAny{}, seccomp.AllowValue(syscall.SHUT_RDWR)},
	},
	syscall.SYS_SIGALTSTACK: {},
	syscall.SYS_SYMLINKAT:   {},
	syscall.SYS_TGKILL: []seccomp.Rule{
		{
			seccomp.AllowValue(uint64(os.Getpid())),
		},
	},
	syscall.SYS_UNLINKAT:  {},
	syscall.SYS_UTIMENSAT: {},
	syscall.SYS_WRITE:     {},
}
<|endoftext|>


<commit_before>package storage

import (
	"archive/zip"
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"io"
	"io/ioutil"
	"path"
	"strings"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.ReadCloser) (io.ReadCloser, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition := strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result = string(result[pathRoot:])
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}

	return result
}

func copyStorageContent(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	if err != nil {
		return err
	}
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath == toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
			if strings.HasPrefix(objectRelativePath, "/") {
				objectRelativePath = string(objectRelativePath[1:])
			}
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}

		if object.IsContent() {
			reader, err := sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			defer reader.Close()

			if modifyContentHandler != nil {

				content, err := ioutil.ReadAll(reader)
				if err != nil {
					return err
				}
				reader = ioutil.NopCloser(bytes.NewReader(content))
				if toolbox.IsASCIIText(string(content)) {
					reader, err = modifyContentHandler(reader)
					if err != nil {
						err = fmt.Errorf("unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
						return err
					}
				}
			}
			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if destinationObject != nil && destinationObject.IsFolder() {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName {
						if !strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationURL, sourceName)
						}

					}
				}
			}

			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return err
			}

		} else {
			err = copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL)
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<commit_msg>patched contend modification handler<commit_after>package storage

import (
	"archive/zip"
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"io"
	"io/ioutil"
	"path"
	"strings"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.ReadCloser) (io.ReadCloser, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition := strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result = string(result[pathRoot:])
	}
	if strings.HasSuffix(result, "/") {
		result = string(result[:len(result)-1])
	}

	return result
}

func copyStorageContent(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	if err != nil {
		return err
	}
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath == toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
			if strings.HasPrefix(objectRelativePath, "/") {
				objectRelativePath = string(objectRelativePath[1:])
			}
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}

		if object.IsContent() {
			reader, err := sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			defer reader.Close()

			if modifyContentHandler != nil {

				content, err := ioutil.ReadAll(reader)
				if err != nil {
					return err
				}
				reader = ioutil.NopCloser(bytes.NewReader(content))
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
					return err
				}

			}
			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if destinationObject != nil && destinationObject.IsFolder() {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName {
						if !strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationURL, sourceName)
						}

					}
				}
			}

			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return err
			}

		} else {
			err = copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL)
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<|endoftext|>


<commit_before>package proxy

import (
	"errors"
	"io"
	"net"
	"time"

	"v2ray.com/core/app"
	"v2ray.com/core/app/proxyman"
	"v2ray.com/core/common/alloc"
	v2io "v2ray.com/core/common/io"
	"v2ray.com/core/common/log"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/ray"
)

const (
	APP_ID = 7
)

type OutboundProxy struct {
	outboundManager proxyman.OutboundHandlerManager
}

func NewOutboundProxy(space app.Space) *OutboundProxy {
	proxy := new(OutboundProxy)
	space.InitializeApplication(func() error {
		if !space.HasApp(proxyman.APP_ID_OUTBOUND_MANAGER) {
			return errors.New("Proxy: Outbound handler manager not found.")
		}
		proxy.outboundManager = space.GetApp(proxyman.APP_ID_OUTBOUND_MANAGER).(proxyman.OutboundHandlerManager)
		return nil
	})
	return proxy
}

func (this *OutboundProxy) RegisterDialer() {
	internet.ProxyDialer = this.Dial
}

func (this *OutboundProxy) Dial(src v2net.Address, dest v2net.Destination, options internet.DialerOptions) (internet.Connection, error) {
	handler := this.outboundManager.GetHandler(options.Proxy.Tag)
	if handler == nil {
		log.Warning("Proxy: Failed to get outbound handler with tag: ", options.Proxy.Tag)
		return internet.Dial(src, dest, internet.DialerOptions{
			Stream: options.Stream,
		})
	}
	stream := ray.NewRay()
	go handler.Dispatch(dest, alloc.NewLocalBuffer(32).Clear(), stream)
	return NewProxyConnection(src, dest, stream), nil
}

func (this *OutboundProxy) Release() {

}

type ProxyConnection struct {
	stream     ray.Ray
	closed     bool
	localAddr  net.Addr
	remoteAddr net.Addr

	reader *v2io.ChanReader
	writer *v2io.ChainWriter
}

func NewProxyConnection(src v2net.Address, dest v2net.Destination, stream ray.Ray) *ProxyConnection {
	return &ProxyConnection{
		stream: stream,
		localAddr: &net.TCPAddr{
			IP:   []byte{0, 0, 0, 0},
			Port: 0,
		},
		remoteAddr: &net.TCPAddr{
			IP:   []byte{0, 0, 0, 0},
			Port: 0,
		},
		reader: v2io.NewChanReader(stream.InboundOutput()),
		writer: v2io.NewChainWriter(stream.InboundInput()),
	}
}

func (this *ProxyConnection) Read(b []byte) (int, error) {
	if this.closed {
		return 0, io.EOF
	}
	return this.reader.Read(b)
}

func (this *ProxyConnection) Write(b []byte) (int, error) {
	if this.closed {
		return 0, io.ErrClosedPipe
	}
	return this.writer.Write(b)
}

func (this *ProxyConnection) Close() error {
	this.closed = true
	this.stream.InboundInput().Close()
	this.stream.InboundOutput().Release()
	return nil
}

func (this *ProxyConnection) LocalAddr() net.Addr {
	return this.localAddr
}

func (this *ProxyConnection) RemoteAddr() net.Addr {
	return this.remoteAddr
}

func (this *ProxyConnection) SetDeadline(t time.Time) error {
	return nil
}

func (this *ProxyConnection) SetReadDeadline(t time.Time) error {
	return nil
}

func (this *ProxyConnection) SetWriteDeadline(t time.Time) error {
	return nil
}

func (this *ProxyConnection) Reusable() bool {
	return false
}

func (this *ProxyConnection) SetReusable(bool) {

}
<commit_msg>small fixes<commit_after>package proxy

import (
	"errors"
	"io"
	"net"
	"time"

	"v2ray.com/core/app"
	"v2ray.com/core/app/proxyman"
	v2io "v2ray.com/core/common/io"
	"v2ray.com/core/common/log"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/ray"
)

const (
	APP_ID = 7
)

type OutboundProxy struct {
	outboundManager proxyman.OutboundHandlerManager
}

func NewOutboundProxy(space app.Space) *OutboundProxy {
	proxy := new(OutboundProxy)
	space.InitializeApplication(func() error {
		if !space.HasApp(proxyman.APP_ID_OUTBOUND_MANAGER) {
			return errors.New("Proxy: Outbound handler manager not found.")
		}
		proxy.outboundManager = space.GetApp(proxyman.APP_ID_OUTBOUND_MANAGER).(proxyman.OutboundHandlerManager)
		return nil
	})
	return proxy
}

func (this *OutboundProxy) RegisterDialer() {
	internet.ProxyDialer = this.Dial
}

func (this *OutboundProxy) Dial(src v2net.Address, dest v2net.Destination, options internet.DialerOptions) (internet.Connection, error) {
	handler := this.outboundManager.GetHandler(options.Proxy.Tag)
	if handler == nil {
		log.Warning("Proxy: Failed to get outbound handler with tag: ", options.Proxy.Tag)
		return internet.Dial(src, dest, internet.DialerOptions{
			Stream: options.Stream,
		})
	}
	log.Info("Proxy: Dialing to ", dest)
	stream := ray.NewRay()
	go handler.Dispatch(dest, nil, stream)
	return NewProxyConnection(src, dest, stream), nil
}

func (this *OutboundProxy) Release() {

}

type ProxyConnection struct {
	stream     ray.Ray
	closed     bool
	localAddr  net.Addr
	remoteAddr net.Addr

	reader *v2io.ChanReader
	writer *v2io.ChainWriter
}

func NewProxyConnection(src v2net.Address, dest v2net.Destination, stream ray.Ray) *ProxyConnection {
	return &ProxyConnection{
		stream: stream,
		localAddr: &net.TCPAddr{
			IP:   []byte{0, 0, 0, 0},
			Port: 0,
		},
		remoteAddr: &net.TCPAddr{
			IP:   []byte{0, 0, 0, 0},
			Port: 0,
		},
		reader: v2io.NewChanReader(stream.InboundOutput()),
		writer: v2io.NewChainWriter(stream.InboundInput()),
	}
}

func (this *ProxyConnection) Read(b []byte) (int, error) {
	if this.closed {
		return 0, io.EOF
	}
	return this.reader.Read(b)
}

func (this *ProxyConnection) Write(b []byte) (int, error) {
	if this.closed {
		return 0, io.ErrClosedPipe
	}
	return this.writer.Write(b)
}

func (this *ProxyConnection) Close() error {
	this.closed = true
	this.stream.InboundInput().Close()
	this.stream.InboundOutput().Release()
	this.reader.Release()
	this.writer.Release()
	return nil
}

func (this *ProxyConnection) LocalAddr() net.Addr {
	return this.localAddr
}

func (this *ProxyConnection) RemoteAddr() net.Addr {
	return this.remoteAddr
}

func (this *ProxyConnection) SetDeadline(t time.Time) error {
	return nil
}

func (this *ProxyConnection) SetReadDeadline(t time.Time) error {
	return nil
}

func (this *ProxyConnection) SetWriteDeadline(t time.Time) error {
	return nil
}

func (this *ProxyConnection) Reusable() bool {
	return false
}

func (this *ProxyConnection) SetReusable(bool) {

}
<|endoftext|>


<commit_before>package main

import (
    "appengine"
    "appengine/datastore"
    "strconv"
    "time"
)

type DB struct {
    Context appengine.Context
}

func (db *DB) FetchUser(user string) {
    vineApi := VineRequest{db.Context}
    data, err := vineApi.GetUser(user)

    var userMeta StoredUserMeta
    var userData StoredUserData

    userId := strconv.FormatFloat(data["userId"].(float64), 'f', -1, 64)

    userMetaTemp, err := db.GetUserMeta(userId)

    if err == datastore.ErrNoSuchEntity {
        userMeta = StoredUserMeta{
            Username: data["username"].(string),
            Location: data["location"].(string),
            Description: data["description"].(string),
            Verified: data["verified"].(float64) == 1.0,
            AvatarUrl: data["avatarUrl"].(string),
        }
        if data["vanityUrls"] != nil {
            userMeta.VanityUrl = data["vanityUrls"].([]interface{})[0].(string)
        }

        if data["background"] != nil {
            userMeta.Background = data["background"].(string)
        }

        if userMeta.Verified {
            userMeta.VerifiedDate = time.Now()
        }

        userData = StoredUserData{
            LastUpdated: time.Now(),
            Followers: []float64{data["followerCount"].(float64),},
            Following: []float64{data["followingCount"].(float64),},
            Loops: []float64{data["loopCount"].(float64),},
            AuthoredPosts: []float64{data["authoredPostCount"].(float64),},
            Reposts: []float64{data["postCount"].(float64) - data["authoredPostCount"].(float64),},
            Likes: []float64{data["likeCount"].(float64),},
            Updated: []time.Time{time.Now(),},
        }

    } else {

        userMeta = userMetaTemp.(StoredUserMeta)

        if userMeta.Location != data["location"].(string) {
            userMeta.Previous.Location = append(userMeta.Previous.Location, PreviousLocation{userMeta.Location, time.Now()})
            userMeta.Location = data["location"].(string)
        }

        if userMeta.Username != data["username"].(string) {
            userMeta.Previous.Username = append(userMeta.Previous.Username, PreviousUsername{userMeta.Username, time.Now()})
            userMeta.Username = data["username"].(string)
        }

        if userMeta.Description != data["description"].(string) {
            userMeta.Previous.Description = append(userMeta.Previous.Description, PreviousDescription{userMeta.Description, time.Now()})
            userMeta.Description = data["description"].(string)
        }

        if userMeta.Background != data["background"].(string) {
            userMeta.Previous.Background = append(userMeta.Previous.Background, PreviousBackground{userMeta.Background, time.Now()})
            userMeta.Background = data["background"].(string)
        }

        userDataTemp, err := db.GetUserData(userId)
        userData = userDataTemp.(StoredUserData)

        if err != datastore.ErrNoSuchEntity {
            userData.LastUpdated = time.Now()
            userData.Followers = append(userData.Followers, data["followerCount"].(float64))
            userData.Following = append(userData.Following, data["followingCount"].(float64))
            userData.Loops = append(userData.Loops, data["loopCount"].(float64))
            userData.AuthoredPosts = append(userData.AuthoredPosts, data["authoredPostCount"].(float64))
            userData.Reposts = append(userData.Reposts, data["postCount"].(float64) - data["authoredPostCount"].(float64))
            userData.Likes = append(userData.Followers, data["likeCount"].(float64))
            userData.Updated = append(userData.Updated, time.Now())
        }
    }

    dataKey := datastore.NewKey(db.Context, "UserData", userId, 0, nil)
    metaKey := datastore.NewKey(db.Context, "UserMeta", userId, 0, nil)

    datastore.Put(db.Context, dataKey, &userData)
    datastore.Put(db.Context, metaKey, &userMeta)
}

func (db *DB) GetUserData(user string) (interface{}, error) {

    var data *StoredUserData

    key := datastore.NewKey(db.Context, "UserMeta", user, 0, nil)
    err := datastore.Get(db.Context, key, &data)

    if err != nil {
        return nil, err
    } else {
        return data, nil
    }
}

func (db *DB) GetUserMeta(user string) (interface{}, error) {

    var meta *StoredUserMeta

	key := datastore.NewKey(db.Context, "UserData", user, 0, nil)
    err := datastore.Get(db.Context, key, &meta)

    if err != nil {
        return nil, err
    } else {
        return meta, nil
    }
}<commit_msg>Prevent private users from being scraped.<commit_after>package main

import (
    "appengine"
    "appengine/datastore"
    "strconv"
    "time"
)

type DB struct {
    Context appengine.Context
}

func (db *DB) FetchUser(user string) {
    vineApi := VineRequest{db.Context}
    data, err := vineApi.GetUser(user)

    if data["private"].(float64) == 1.0 {
        return
    }

    var userMeta StoredUserMeta
    var userData StoredUserData

    userId := strconv.FormatFloat(data["userId"].(float64), 'f', -1, 64)

    userMetaTemp, err := db.GetUserMeta(userId)

    if err == datastore.ErrNoSuchEntity {
        userMeta = StoredUserMeta{
            Username: data["username"].(string),
            Location: data["location"].(string),
            Description: data["description"].(string),
            Verified: data["verified"].(float64) == 1.0,
            AvatarUrl: data["avatarUrl"].(string),
        }
        if data["vanityUrls"] != nil {
            userMeta.VanityUrl = data["vanityUrls"].([]interface{})[0].(string)
        }

        if data["background"] != nil {
            userMeta.Background = data["background"].(string)
        }

        if userMeta.Verified {
            userMeta.VerifiedDate = time.Now()
        }

        userData = StoredUserData{
            LastUpdated: time.Now(),
            Followers: []float64{data["followerCount"].(float64),},
            Following: []float64{data["followingCount"].(float64),},
            Loops: []float64{data["loopCount"].(float64),},
            AuthoredPosts: []float64{data["authoredPostCount"].(float64),},
            Reposts: []float64{data["postCount"].(float64) - data["authoredPostCount"].(float64),},
            Likes: []float64{data["likeCount"].(float64),},
            Updated: []time.Time{time.Now(),},
        }

    } else {

        userMeta = userMetaTemp.(StoredUserMeta)

        if userMeta.Location != data["location"].(string) {
            userMeta.Previous.Location = append(userMeta.Previous.Location, PreviousLocation{userMeta.Location, time.Now()})
            userMeta.Location = data["location"].(string)
        }

        if userMeta.Username != data["username"].(string) {
            userMeta.Previous.Username = append(userMeta.Previous.Username, PreviousUsername{userMeta.Username, time.Now()})
            userMeta.Username = data["username"].(string)
        }

        if userMeta.Description != data["description"].(string) {
            userMeta.Previous.Description = append(userMeta.Previous.Description, PreviousDescription{userMeta.Description, time.Now()})
            userMeta.Description = data["description"].(string)
        }

        if userMeta.Background != data["background"].(string) {
            userMeta.Previous.Background = append(userMeta.Previous.Background, PreviousBackground{userMeta.Background, time.Now()})
            userMeta.Background = data["background"].(string)
        }

        userDataTemp, err := db.GetUserData(userId)
        userData = userDataTemp.(StoredUserData)

        if err != datastore.ErrNoSuchEntity {
            userData.LastUpdated = time.Now()
            userData.Followers = append(userData.Followers, data["followerCount"].(float64))
            userData.Following = append(userData.Following, data["followingCount"].(float64))
            userData.Loops = append(userData.Loops, data["loopCount"].(float64))
            userData.AuthoredPosts = append(userData.AuthoredPosts, data["authoredPostCount"].(float64))
            userData.Reposts = append(userData.Reposts, data["postCount"].(float64) - data["authoredPostCount"].(float64))
            userData.Likes = append(userData.Followers, data["likeCount"].(float64))
            userData.Updated = append(userData.Updated, time.Now())
        }
    }

    dataKey := datastore.NewKey(db.Context, "UserData", userId, 0, nil)
    metaKey := datastore.NewKey(db.Context, "UserMeta", userId, 0, nil)

    datastore.Put(db.Context, dataKey, &userData)
    datastore.Put(db.Context, metaKey, &userMeta)
}

func (db *DB) GetUserData(user string) (interface{}, error) {

    var data *StoredUserData

    key := datastore.NewKey(db.Context, "UserMeta", user, 0, nil)
    err := datastore.Get(db.Context, key, &data)

    if err != nil {
        return nil, err
    } else {
        return data, nil
    }
}

func (db *DB) GetUserMeta(user string) (interface{}, error) {

    var meta *StoredUserMeta

	key := datastore.NewKey(db.Context, "UserData", user, 0, nil)
    err := datastore.Get(db.Context, key, &meta)

    if err != nil {
        return nil, err
    } else {
        return meta, nil
    }
}<|endoftext|>


<commit_before>package jiffy

import (
	"errors"
	"time"
)

var (
	// The maximum number of messages to buffer in a subscription.
	ResponseBufferSize = 100

	// The wait before timing out a publish.
	PublishTimeout = 10 * time.Minute
)

type Subscription struct {
	Name     string
	Topic    *Topic
	Response chan *Message
	uuid     string
	expireAt time.Time
}

func NewSubscription(name string, topic *Topic, ttl time.Duration) *Subscription {
	subscription := &Subscription{
		name,
		topic,
		make(chan *Message, ResponseBufferSize),
		UUID(),
		time.Now().Add(ttl),
	}
	return subscription
}

// Publishes a message to the subscription.
func (subscription *Subscription) Publish(message *Message) {
	ticker := time.NewTicker(PublishTimeout)
	select {
	case subscription.Response <- message:
	case <-ticker.C:
	}
}

// Deletes the subscription from its topic.
func (subscription *Subscription) Expire() {
	subscription.expireAt = time.Now()
}

// Expires the subscription.
func (subscription *Subscription) Expired() bool {
	return time.Now().After(subscription.expireAt)
}

var (
	ExpiredSubscription = errors.New("Subscription is expired")
)

// Extends the subscription's expiration by the input TTL.
func (subscription *Subscription) ExtendExpiration(ttl time.Duration) error {
	if !subscription.Active() {
		return ExpiredSubscription
	}
	subscription.expireAt = time.Now().Add(ttl)
	return nil
}

// Returns true if the subscription is active on a topic.
func (subscription *Subscription) Active() bool {
	if subscription.Expired() {
		return false
	}
	if topicSubscription, ok := subscription.Topic.Subscriptions[subscription.Name]; ok {
		return topicSubscription.uuid == subscription.uuid
	}
	return false
}
<commit_msg>Moving things around<commit_after>package jiffy

import (
	"errors"
	"time"
)

var (
	// The maximum number of messages to buffer in a subscription.
	ResponseBufferSize = 100
	// The wait before timing out a publish.
	PublishTimeout      = 10 * time.Minute
	ExpiredSubscription = errors.New("Subscription is expired")
)

type Subscription struct {
	Name     string
	Topic    *Topic
	Response chan *Message
	uuid     string
	expireAt time.Time
}

func NewSubscription(name string, topic *Topic, ttl time.Duration) *Subscription {
	subscription := &Subscription{
		name,
		topic,
		make(chan *Message, ResponseBufferSize),
		UUID(),
		time.Now().Add(ttl),
	}
	return subscription
}

// Publishes a message to the subscription.
func (subscription *Subscription) Publish(message *Message) {
	ticker := time.NewTicker(PublishTimeout)
	select {
	case subscription.Response <- message:
	case <-ticker.C:
	}
}

// Deletes the subscription from its topic.
func (subscription *Subscription) Expire() {
	subscription.expireAt = time.Now()
}

// Expires the subscription.
func (subscription *Subscription) Expired() bool {
	return time.Now().After(subscription.expireAt)
}

// Extends the subscription's expiration by the input TTL.
func (subscription *Subscription) ExtendExpiration(ttl time.Duration) error {
	if !subscription.Active() {
		return ExpiredSubscription
	}
	subscription.expireAt = time.Now().Add(ttl)
	return nil
}

// Returns true if the subscription is active on a topic.
func (subscription *Subscription) Active() bool {
	if subscription.Expired() {
		return false
	}
	if topicSubscription, ok := subscription.Topic.Subscriptions[subscription.Name]; ok {
		return topicSubscription.uuid == subscription.uuid
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"expvar"
	"flag"
	"fmt"
	"http"
	"io"
	"log"
	"os"
	"strconv"
)

// hello world, the web server
var helloRequests = expvar.NewInt("hello-requests")

func HelloServer(w http.ResponseWriter, req *http.Request) {
	helloRequests.Add(1)
	io.WriteString(w, "hello, world!\n")
}

// Simple counter server. POSTing to it will set the value.
type Counter struct {
	n int
}

// This makes Counter satisfy the expvar.Var interface, so we can export
// it directly.
func (ctr *Counter) String() string { return fmt.Sprintf("%d", ctr.n) }

func (ctr *Counter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	switch req.Method {
	case "GET":
		ctr.n++
	case "POST":
		buf := new(bytes.Buffer)
		io.Copy(buf, req.Body)
		body := buf.String()
		if n, err := strconv.Atoi(body); err != nil {
			fmt.Fprintf(w, "bad POST: %v\nbody: [%v]\n", err, body)
		} else {
			ctr.n = n
			fmt.Fprint(w, "counter reset\n")
		}
	}
	fmt.Fprintf(w, "counter = %d\n", ctr.n)
}

// simple flag server
var booleanflag = flag.Bool("boolean", true, "another flag for testing")

func FlagServer(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "Flags:\n")
	flag.VisitAll(func(f *flag.Flag) {
		if f.Value.String() != f.DefValue {
			fmt.Fprintf(w, "%s = %s [default = %s]\n", f.Name, f.Value.String(), f.DefValue)
		} else {
			fmt.Fprintf(w, "%s = %s\n", f.Name, f.Value.String())
		}
	})
}

// simple argument server
func ArgServer(w http.ResponseWriter, req *http.Request) {
	for _, s := range os.Args {
		fmt.Fprint(w, s, " ")
	}
}

// a channel (just for the fun of it)
type Chan chan int

func ChanCreate() Chan {
	c := make(Chan)
	go func(c Chan) {
		for x := 0; ; x++ {
			c <- x
		}
	}(c)
	return c
}

func (ch Chan) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, fmt.Sprintf("channel send #%d\n", <-ch))
}

// exec a program, redirecting output
func DateServer(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
	r, w, err := os.Pipe()
	if err != nil {
		fmt.Fprintf(rw, "pipe: %s\n", err)
		return
	}

	p, err := os.StartProcess("/bin/date", []string{"date"}, &os.ProcAttr{Files: []*os.File{nil, w, w}})
	defer r.Close()
	w.Close()
	if err != nil {
		fmt.Fprintf(rw, "fork/exec: %s\n", err)
		return
	}
	defer p.Release()
	io.Copy(rw, r)
	wait, err := p.Wait(0)
	if err != nil {
		fmt.Fprintf(rw, "wait: %s\n", err)
		return
	}
	if !wait.Exited() || wait.ExitStatus() != 0 {
		fmt.Fprintf(rw, "date: %v\n", wait)
		return
	}
}

func Logger(w http.ResponseWriter, req *http.Request) {
	log.Print(req.URL.Raw)
	w.WriteHeader(404)
	w.Write([]byte("oops"))
}

var webroot = flag.String("root", "/home/rsc", "web root directory")

func main() {
	flag.Parse()

	// The counter is published as a variable directly.
	ctr := new(Counter)
	http.Handle("/counter", ctr)
	expvar.Publish("counter", ctr)

	http.Handle("/", http.HandlerFunc(Logger))
	http.Handle("/go/", http.FileServer(*webroot, "/go/"))
	http.Handle("/flags", http.HandlerFunc(FlagServer))
	http.Handle("/args", http.HandlerFunc(ArgServer))
	http.Handle("/go/hello", http.HandlerFunc(HelloServer))
	http.Handle("/chan", ChanCreate())
	http.Handle("/date", http.HandlerFunc(DateServer))
	err := http.ListenAndServe(":12345", nil)
	if err != nil {
		log.Panicln("ListenAndServe:", err)
	}
}
<commit_msg>http: update triv.go with gofix<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"expvar"
	"flag"
	"fmt"
	"http"
	"io"
	"log"
	"os"
	"strconv"
)

// hello world, the web server
var helloRequests = expvar.NewInt("hello-requests")

func HelloServer(w http.ResponseWriter, req *http.Request) {
	helloRequests.Add(1)
	io.WriteString(w, "hello, world!\n")
}

// Simple counter server. POSTing to it will set the value.
type Counter struct {
	n int
}

// This makes Counter satisfy the expvar.Var interface, so we can export
// it directly.
func (ctr *Counter) String() string { return fmt.Sprintf("%d", ctr.n) }

func (ctr *Counter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	switch req.Method {
	case "GET":
		ctr.n++
	case "POST":
		buf := new(bytes.Buffer)
		io.Copy(buf, req.Body)
		body := buf.String()
		if n, err := strconv.Atoi(body); err != nil {
			fmt.Fprintf(w, "bad POST: %v\nbody: [%v]\n", err, body)
		} else {
			ctr.n = n
			fmt.Fprint(w, "counter reset\n")
		}
	}
	fmt.Fprintf(w, "counter = %d\n", ctr.n)
}

// simple flag server
var booleanflag = flag.Bool("boolean", true, "another flag for testing")

func FlagServer(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "Flags:\n")
	flag.VisitAll(func(f *flag.Flag) {
		if f.Value.String() != f.DefValue {
			fmt.Fprintf(w, "%s = %s [default = %s]\n", f.Name, f.Value.String(), f.DefValue)
		} else {
			fmt.Fprintf(w, "%s = %s\n", f.Name, f.Value.String())
		}
	})
}

// simple argument server
func ArgServer(w http.ResponseWriter, req *http.Request) {
	for _, s := range os.Args {
		fmt.Fprint(w, s, " ")
	}
}

// a channel (just for the fun of it)
type Chan chan int

func ChanCreate() Chan {
	c := make(Chan)
	go func(c Chan) {
		for x := 0; ; x++ {
			c <- x
		}
	}(c)
	return c
}

func (ch Chan) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, fmt.Sprintf("channel send #%d\n", <-ch))
}

// exec a program, redirecting output
func DateServer(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
	r, w, err := os.Pipe()
	if err != nil {
		fmt.Fprintf(rw, "pipe: %s\n", err)
		return
	}

	p, err := os.StartProcess("/bin/date", []string{"date"}, &os.ProcAttr{Files: []*os.File{nil, w, w}})
	defer r.Close()
	w.Close()
	if err != nil {
		fmt.Fprintf(rw, "fork/exec: %s\n", err)
		return
	}
	defer p.Release()
	io.Copy(rw, r)
	wait, err := p.Wait(0)
	if err != nil {
		fmt.Fprintf(rw, "wait: %s\n", err)
		return
	}
	if !wait.Exited() || wait.ExitStatus() != 0 {
		fmt.Fprintf(rw, "date: %v\n", wait)
		return
	}
}

func Logger(w http.ResponseWriter, req *http.Request) {
	log.Print(req.URL.Raw)
	w.WriteHeader(404)
	w.Write([]byte("oops"))
}

var webroot = flag.String("root", "/home/rsc", "web root directory")

func main() {
	flag.Parse()

	// The counter is published as a variable directly.
	ctr := new(Counter)
	http.Handle("/counter", ctr)
	expvar.Publish("counter", ctr)

	http.Handle("/", http.HandlerFunc(Logger))
	http.Handle("/go/", http.StripPrefix("/go/", http.FileServer(http.Dir(*webroot))))
	http.Handle("/flags", http.HandlerFunc(FlagServer))
	http.Handle("/args", http.HandlerFunc(ArgServer))
	http.Handle("/go/hello", http.HandlerFunc(HelloServer))
	http.Handle("/chan", ChanCreate())
	http.Handle("/date", http.HandlerFunc(DateServer))
	err := http.ListenAndServe(":12345", nil)
	if err != nil {
		log.Panicln("ListenAndServe:", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"once"
)

// A Ticker holds a synchronous channel that delivers `ticks' of a clock
// at intervals.
type Ticker struct {
	C        <-chan int64 // The channel on which the ticks are delivered.
	c        chan<- int64 // The same channel, but the end we use.
	ns       int64
	shutdown bool
	nextTick int64
	next     *Ticker
}

// Stop turns off a ticker.  After Stop, no more ticks will be sent.
func (t *Ticker) Stop() { t.shutdown = true }

// Tick is a convenience wrapper for NewTicker providing access to the ticking
// channel only.  Useful for clients that have no need to shut down the ticker.
func Tick(ns int64) <-chan int64 {
	if ns <= 0 {
		return nil
	}
	return NewTicker(ns).C
}

type alarmer struct {
	wakeUp   chan bool // wakeup signals sent/received here
	wakeMeAt chan int64
	wakeTime int64
}

// Set alarm to go off at time ns, if not already set earlier.
func (a *alarmer) set(ns int64) {
	// If there's no wakeLoop or the next tick we expect is too late, start a new wakeLoop
	if a.wakeMeAt == nil || a.wakeTime > ns {
		// Stop previous wakeLoop.
		if a.wakeMeAt != nil {
			a.wakeMeAt <- -1
		}
		a.wakeMeAt = make(chan int64, 10)
		go wakeLoop(a.wakeMeAt, a.wakeUp)
		a.wakeMeAt <- ns
	}
}

// Channel to notify tickerLoop of new Tickers being created.
var newTicker chan *Ticker

func startTickerLoop() {
	newTicker = make(chan *Ticker)
	go tickerLoop()
}

// wakeLoop delivers ticks at scheduled times, sleeping until the right moment.
// If another, earlier Ticker is created while it sleeps, tickerLoop() will start a new
// wakeLoop but they will share the wakeUp channel and signal that this one
// is done by giving it a negative time request.
func wakeLoop(wakeMeAt chan int64, wakeUp chan bool) {
	for {
		wakeAt := <-wakeMeAt
		if wakeAt < 0 { // tickerLoop has started another wakeLoop
			return
		}
		now := Nanoseconds()
		if wakeAt > now {
			Sleep(wakeAt - now)
			now = Nanoseconds()
		}
		wakeUp <- true
	}
}

// A single tickerLoop serves all ticks to Tickers.  It waits for two events:
// either the creation of a new Ticker or a tick from the alarm,
// signalling a time to wake up one or more Tickers.
func tickerLoop() {
	// Represents the next alarm to be delivered.
	var alarm alarmer
	// All wakeLoops deliver wakeups to this channel.
	alarm.wakeUp = make(chan bool, 10)
	var now, prevTime, wakeTime int64
	var tickers *Ticker
	for {
		select {
		case t := <-newTicker:
			// Add Ticker to list
			t.next = tickers
			tickers = t
			// Arrange for a new alarm if this one precedes the existing one.
			alarm.set(t.nextTick)
		case <-alarm.wakeUp:
			now = Nanoseconds()
			// Ignore an old time due to a dying wakeLoop
			if now < prevTime {
				continue
			}
			wakeTime = now + 1e15 // very long in the future
			var prev *Ticker = nil
			// Scan list of tickers, delivering updates to those
			// that need it and determining the next wake time.
			// TODO(r): list should be sorted in time order.
			for t := tickers; t != nil; t = t.next {
				if t.shutdown {
					// Ticker is done; remove it from list.
					if prev == nil {
						tickers = t.next
					} else {
						prev.next = t.next
					}
					continue
				}
				if tickers.nextTick <= now {
					if len(t.c) == 0 {
						// Only send if there's room.  We must not block.
						// The channel is allocated with a one-element
						// buffer, which is sufficient: if he hasn't picked
						// up the last tick, no point in sending more.
						t.c <- now
					}
					t.nextTick += t.ns
					if t.nextTick <= now {
						// Still behind; advance in one big step.
						t.nextTick += (now - t.nextTick + t.ns) / t.ns * t.ns
					}
					if t.nextTick > now && t.nextTick < wakeTime {
						wakeTime = t.nextTick
					}
				}
				prev = t
			}
			if tickers != nil {
				// Please send wakeup at earliest required time.
				// If there are no tickers, don't bother.
				alarm.wakeMeAt <- wakeTime
			}
		}
		prevTime = now
	}
}

// Ticker returns a new Ticker containing a channel that will
// send the time, in nanoseconds, every ns nanoseconds.  It adjusts the
// intervals to make up for pauses in delivery of the ticks.
func NewTicker(ns int64) *Ticker {
	if ns <= 0 {
		return nil
	}
	c := make(chan int64, 1) //  See comment on send in tickerLoop
	t := &Ticker{c, c, ns, false, Nanoseconds() + ns, nil}
	once.Do(startTickerLoop)
	// must be run in background so global Tickers can be created
	go func() { newTicker <- t }()
	return t
}
<commit_msg>fix bug in tick<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"once"
)

// A Ticker holds a synchronous channel that delivers `ticks' of a clock
// at intervals.
type Ticker struct {
	C        <-chan int64 // The channel on which the ticks are delivered.
	c        chan<- int64 // The same channel, but the end we use.
	ns       int64
	shutdown bool
	nextTick int64
	next     *Ticker
}

// Stop turns off a ticker.  After Stop, no more ticks will be sent.
func (t *Ticker) Stop() { t.shutdown = true }

// Tick is a convenience wrapper for NewTicker providing access to the ticking
// channel only.  Useful for clients that have no need to shut down the ticker.
func Tick(ns int64) <-chan int64 {
	if ns <= 0 {
		return nil
	}
	return NewTicker(ns).C
}

type alarmer struct {
	wakeUp   chan bool // wakeup signals sent/received here
	wakeMeAt chan int64
	wakeTime int64
}

// Set alarm to go off at time ns, if not already set earlier.
func (a *alarmer) set(ns int64) {
	// If there's no wakeLoop or the next tick we expect is too late, start a new wakeLoop
	if a.wakeMeAt == nil || a.wakeTime > ns {
		// Stop previous wakeLoop.
		if a.wakeMeAt != nil {
			a.wakeMeAt <- -1
		}
		a.wakeMeAt = make(chan int64, 10)
		go wakeLoop(a.wakeMeAt, a.wakeUp)
		a.wakeTime = ns
		a.wakeMeAt <- ns
	}
}

// Channel to notify tickerLoop of new Tickers being created.
var newTicker chan *Ticker

func startTickerLoop() {
	newTicker = make(chan *Ticker)
	go tickerLoop()
}

// wakeLoop delivers ticks at scheduled times, sleeping until the right moment.
// If another, earlier Ticker is created while it sleeps, tickerLoop() will start a new
// wakeLoop but they will share the wakeUp channel and signal that this one
// is done by giving it a negative time request.
func wakeLoop(wakeMeAt chan int64, wakeUp chan bool) {
	for {
		wakeAt := <-wakeMeAt
		if wakeAt < 0 { // tickerLoop has started another wakeLoop
			return
		}
		now := Nanoseconds()
		if wakeAt > now {
			Sleep(wakeAt - now)
			now = Nanoseconds()
		}
		wakeUp <- true
	}
}

// A single tickerLoop serves all ticks to Tickers.  It waits for two events:
// either the creation of a new Ticker or a tick from the alarm,
// signalling a time to wake up one or more Tickers.
func tickerLoop() {
	// Represents the next alarm to be delivered.
	var alarm alarmer
	// All wakeLoops deliver wakeups to this channel.
	alarm.wakeUp = make(chan bool, 10)
	var now, prevTime, wakeTime int64
	var tickers *Ticker
	for {
		select {
		case t := <-newTicker:
			// Add Ticker to list
			t.next = tickers
			tickers = t
			// Arrange for a new alarm if this one precedes the existing one.
			alarm.set(t.nextTick)
		case <-alarm.wakeUp:
			now = Nanoseconds()
			// Ignore an old time due to a dying wakeLoop
			if now < prevTime {
				continue
			}
			wakeTime = now + 1e15 // very long in the future
			var prev *Ticker = nil
			// Scan list of tickers, delivering updates to those
			// that need it and determining the next wake time.
			// TODO(r): list should be sorted in time order.
			for t := tickers; t != nil; t = t.next {
				if t.shutdown {
					// Ticker is done; remove it from list.
					if prev == nil {
						tickers = t.next
					} else {
						prev.next = t.next
					}
					continue
				}
				if t.nextTick <= now {
					if len(t.c) == 0 {
						// Only send if there's room.  We must not block.
						// The channel is allocated with a one-element
						// buffer, which is sufficient: if he hasn't picked
						// up the last tick, no point in sending more.
						t.c <- now
					}
					t.nextTick += t.ns
					if t.nextTick <= now {
						// Still behind; advance in one big step.
						t.nextTick += (now - t.nextTick + t.ns) / t.ns * t.ns
					}
				}
				if t.nextTick < wakeTime {
					wakeTime = t.nextTick
				}
				prev = t
			}
			if tickers != nil {
				// Please send wakeup at earliest required time.
				// If there are no tickers, don't bother.
				alarm.wakeMeAt <- wakeTime
			}
		}
		prevTime = now
	}
}

// Ticker returns a new Ticker containing a channel that will
// send the time, in nanoseconds, every ns nanoseconds.  It adjusts the
// intervals to make up for pauses in delivery of the ticks.
func NewTicker(ns int64) *Ticker {
	if ns <= 0 {
		return nil
	}
	c := make(chan int64, 1) //  See comment on send in tickerLoop
	t := &Ticker{c, c, ns, false, Nanoseconds() + ns, nil}
	once.Do(startTickerLoop)
	// must be run in background so global Tickers can be created
	go func() { newTicker <- t }()
	return t
}
<|endoftext|>