<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

import (
	"strings"
	"testing"

	"github.com/go-test/deep"
)

var checkerInvalidPrograms = []struct {
	name    string
	program string
	errors  []string
}{
	{"undefined named capture group",
		"/blurgh/ { $undef++\n }\n",
		[]string{"undefined named capture group:1:12-17: Capture group `$undef' was not defined by a regular expression visible to this scope.\n\tTry using `(?P<undef>...)' to name the capture group."}},

	{"out of bounds capref",
		"/(blyurg)/ { $2++ \n}\n",
		[]string{"out of bounds capref:1:14-15: Capture group `$2' was not defined by a regular expression " +
			"visible to this scope.\n\tCheck that there are at least 2 pairs of parentheses."},
	},

	{"undefined decorator",
		"@foo {}\n",
		[]string{"undefined decorator:1:1-4: Decorator `foo' not defined.\n\tTry adding a definition `def foo {}' earlier in the program."}},

	{"undefined identifier",
		"// { x++ \n}\n",
		[]string{"undefined identifier:1:6: Identifier `x' not declared.\n\tTry adding `counter x' to the top of the program."},
	},

	{"invalid regex",
		"/foo(/ {}\n",
		[]string{"invalid regex:1:1-6: error parsing regexp: missing closing ): `foo(`"}},

	{"invalid regex 2",
		"/blurg(?P<x.)/ {}\n",
		[]string{"invalid regex 2:1:1-14: error parsing regexp: invalid named capture: `(?P<x.)`"}},

	{"invalid regex 3",
		"/blurg(?P<x>[[:alph:]])/ {}\n",
		[]string{"invalid regex 3:1:1-24: error parsing regexp: invalid character class range: `[:alph:]`"}},

	{"duplicate declaration",
		"counter foo\ncounter foo\n",
		[]string{"duplicate declaration:2:9-11: Redeclaration of metric `foo' previously declared at duplicate declaration:1:9-11"}},

	// 	{"indexedExpr parameter count",
	// 		`counter foo by a, b
	// counter bar by a, b
	// counter quux by a
	// /(\d+)/ {
	//   foo[$1]++
	//   bar[$1][0]++
	//   quux[$1][0]++
	// }
	// 	`,
	// 		[]string{"indexedExpr parameter count:5:3-8: index lookup: type mismatch: \"→ typeVar1 typeVar2 typeVar3\" != \"→ Int typeVar9\"",
	// 			"indexedExpr parameter count:7:3-12: Too many keys for metric"}},

	{"builtin parameter mismatch",
		`/(\d+)/ {
	  strptime()
	}
    /(\d+)/ {
	  timestamp()
	}
	`,
		[]string{"builtin parameter mismatch:2:13: call to `strptime': type mismatch; expected \"→ String String None\" received \"→ typeVar5\""}},
}

func TestCheckInvalidPrograms(t *testing.T) {
	for _, tc := range checkerInvalidPrograms {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			ast, err := Parse(tc.name, strings.NewReader(tc.program))
			if err != nil {
				t.Fatal(err)
			}
			err = Check(ast)
			if err == nil {
				t.Fatal("check didn't fail")
			}

			diff := deep.Equal(
				strings.Join(tc.errors, "\n"),        // want
				strings.TrimRight(err.Error(), "\n")) // got
			if diff != nil {
				t.Error(diff)
				s := Sexp{}
				s.emitTypes = true
				t.Log(s.Dump(ast))
			}
		})
	}
}

var checkerValidPrograms = []struct {
	name    string
	program string
}{
	{"capture group",
		`counter foo
/(.*)/ {
  foo += $1
}
`,
	},
	{"shadowed positionals",
		`counter foo
/(.*)/ {
  foo += $1
  /bar(\d+)/ {
   foo += $1
  }
}
`},
	{"sibling positionals",
		`counter foo
/(.*)/ {
  foo += $1
}
/bar(\d+)/ {
   foo += $1
}
`},

	{"index expression",
		`counter foo by a, b
/(\d)/ {
  foo[1,$1] = 3
}`},

	// TODO(jaq): Issue 73, 68
	// {"odd indexes",
	// 	`counter foo by a,b,c
	// /(\d) (\d)/ {
	//   foo[$1,$2][0]++
	// }
	// `},
	{"implicit int",
		`counter foo
/$/ {
  foo++
}`},
	{"function return value",
		`len("foo") > 0 {}`},
	{"conversions",
		`counter i
	counter f
	/(.*)/ {
	  i = int($1)
	  f = float($1)
	}
	`},
}

func TestCheckValidPrograms(t *testing.T) {
	for _, tc := range checkerValidPrograms {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			ast, err := Parse(tc.name, strings.NewReader(tc.program))
			if err != nil {
				t.Fatal(err)
			}
			err = Check(ast)
			s := Sexp{}
			s.emitTypes = true
			t.Log("Typed AST:\n" + s.Dump(ast))
			if err != nil {
				t.Errorf("check failed: %s", err)
			}
		})
	}
}

var checkerTypeExpressionTests = []struct {
	name     string
	expr     astNode
	expected Type
}{
	{"Int + Int -> Int",
		&binaryExprNode{lhs: &intConstNode{position{}, 1},
			rhs: &intConstNode{position{}, 1},
			op:  PLUS},
		Int,
	},
	{"Int + Float -> Float",
		&binaryExprNode{lhs: &intConstNode{position{}, 1},
			rhs: &floatConstNode{position{}, 1.0},
			op:  PLUS},
		Float,
	},
	{"⍺ + Float -> Float",
		&binaryExprNode{lhs: &idNode{pos: position{}, sym: &Symbol{Name: "i", Kind: VarSymbol, Type: NewTypeVariable()}},
			rhs: &caprefNode{pos: position{}, sym: &Symbol{Kind: CaprefSymbol, Type: Float}},
			op:  PLUS},
		Float,
	},
}

func TestCheckTypeExpressions(t *testing.T) {
	defaultCompareUnexportedFields := deep.CompareUnexportedFields
	deep.CompareUnexportedFields = true
	defer func() { deep.CompareUnexportedFields = defaultCompareUnexportedFields }()

	for _, tc := range checkerTypeExpressionTests {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			err := Check(tc.expr)
			if err != nil {
				t.Fatalf("check error: %s", err)
			}

			diff := deep.Equal(tc.expected, tc.expr.Type().Root())
			if len(diff) > 0 {
				t.Error(diff)
				s := Sexp{}
				s.emitTypes = true
				t.Log("Typed AST:\n" + s.Dump(tc.expr))
			}
		})
	}
}
<commit_msg>Switch to using go-cmp Diff for TestCheckInvalidPrograms.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

import (
	"strings"
	"testing"

	go_cmp "github.com/google/go-cmp/cmp"
)

var checkerInvalidPrograms = []struct {
	name    string
	program string
	errors  []string
}{
	{"undefined named capture group",
		"/blurgh/ { $undef++\n }\n",
		[]string{"undefined named capture group:1:12-17: Capture group `$undef' was not defined by a regular expression visible to this scope.", "\tTry using `(?P<undef>...)' to name the capture group."}},

	{"out of bounds capref",
		"/(blyurg)/ { $2++ \n}\n",
		[]string{"out of bounds capref:1:14-15: Capture group `$2' was not defined by a regular expression " +
			"visible to this scope.", "\tCheck that there are at least 2 pairs of parentheses."},
	},

	{"undefined decorator",
		"@foo {}\n",
		[]string{"undefined decorator:1:1-4: Decorator `foo' not defined.", "\tTry adding a definition `def foo {}' earlier in the program."}},

	{"undefined identifier",
		"// { x++ \n}\n",
		[]string{"undefined identifier:1:6: Identifier `x' not declared.", "\tTry adding `counter x' to the top of the program."},
	},

	{"invalid regex",
		"/foo(/ {}\n",
		[]string{"invalid regex:1:1-6: error parsing regexp: missing closing ): `foo(`"}},

	{"invalid regex 2",
		"/blurg(?P<x.)/ {}\n",
		[]string{"invalid regex 2:1:1-14: error parsing regexp: invalid named capture: `(?P<x.)`"}},

	{"invalid regex 3",
		"/blurg(?P<x>[[:alph:]])/ {}\n",
		[]string{"invalid regex 3:1:1-24: error parsing regexp: invalid character class range: `[:alph:]`"}},

	{"duplicate declaration",
		"counter foo\ncounter foo\n",
		[]string{"duplicate declaration:2:9-11: Redeclaration of metric `foo' previously declared at duplicate declaration:1:9-11"}},

	{"indexedExpr parameter count",
		`counter foo by a, b
	counter bar by a, b
	counter quux by a
	/(\d+)/ {
	  foo[$1]++
	  bar[$1][0]++
	  quux[$1][0]++
	}
		`,
		[]string{"indexedExpr parameter count:5:4-9: index lookup: type mismatch: expected \"→ typeVar5 typeVar6 typeVar7\" received \"→ Int typeVar13\"",
			"indexedExpr parameter count:6:4-9: Too many keys for metric"}},

	{"builtin parameter mismatch",
		`/(\d+)/ {
	  strptime()
	}
    /(\d+)/ {
	  timestamp()
	}
	`,
		[]string{"builtin parameter mismatch:2:13: call to `strptime': type mismatch; expected \"→ String String None\" received \"→ typeVar5\""}},
}

func TestCheckInvalidPrograms(t *testing.T) {
	for _, tc := range checkerInvalidPrograms {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			ast, err := Parse(tc.name, strings.NewReader(tc.program))
			if err != nil {
				t.Fatal(err)
			}
			err = Check(ast)
			if err == nil {
				t.Fatal("check didn't fail")
			}

			diff := go_cmp.Diff(
				tc.errors,                        // want
				strings.Split(err.Error(), "\n")) // got
			if diff != "" {
				t.Errorf("Diff %s", diff)
				s := Sexp{}
				s.emitTypes = true
				t.Log(s.Dump(ast))
			}
		})
	}
}

var checkerValidPrograms = []struct {
	name    string
	program string
}{
	{"capture group",
		`counter foo
/(.*)/ {
  foo += $1
}
`,
	},
	{"shadowed positionals",
		`counter foo
/(.*)/ {
  foo += $1
  /bar(\d+)/ {
   foo += $1
  }
}
`},
	{"sibling positionals",
		`counter foo
/(.*)/ {
  foo += $1
}
/bar(\d+)/ {
   foo += $1
}
`},

	{"index expression",
		`counter foo by a, b
/(\d)/ {
  foo[1,$1] = 3
}`},

	// TODO(jaq): Issue 73, 68
	// {"odd indexes",
	// 	`counter foo by a,b,c
	// /(\d) (\d)/ {
	//   foo[$1,$2][0]++
	// }
	// `},
	{"implicit int",
		`counter foo
/$/ {
  foo++
}`},
	{"function return value",
		`len("foo") > 0 {}`},
	{"conversions",
		`counter i
	counter f
	/(.*)/ {
	  i = int($1)
	  f = float($1)
	}
	`},
}

func TestCheckValidPrograms(t *testing.T) {
	for _, tc := range checkerValidPrograms {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			ast, err := Parse(tc.name, strings.NewReader(tc.program))
			if err != nil {
				t.Fatal(err)
			}
			err = Check(ast)
			s := Sexp{}
			s.emitTypes = true
			t.Log("Typed AST:\n" + s.Dump(ast))
			if err != nil {
				t.Errorf("check failed: %s", err)
			}
		})
	}
}

var checkerTypeExpressionTests = []struct {
	name     string
	expr     astNode
	expected Type
}{
	{"Int + Int -> Int",
		&binaryExprNode{lhs: &intConstNode{position{}, 1},
			rhs: &intConstNode{position{}, 1},
			op:  PLUS},
		Int,
	},
	{"Int + Float -> Float",
		&binaryExprNode{lhs: &intConstNode{position{}, 1},
			rhs: &floatConstNode{position{}, 1.0},
			op:  PLUS},
		Float,
	},
	{"⍺ + Float -> Float",
		&binaryExprNode{lhs: &idNode{pos: position{}, sym: &Symbol{Name: "i", Kind: VarSymbol, Type: NewTypeVariable()}},
			rhs: &caprefNode{pos: position{}, sym: &Symbol{Kind: CaprefSymbol, Type: Float}},
			op:  PLUS},
		Float,
	},
}

func TestCheckTypeExpressions(t *testing.T) {
	for _, tc := range checkerTypeExpressionTests {
		tc := tc
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()
			err := Check(tc.expr)
			if err != nil {
				t.Fatalf("check error: %s", err)
			}

			diff := go_cmp.Diff(tc.expected, tc.expr.Type().Root())
			if diff != "" {
				t.Error(diff)
				s := Sexp{}
				s.emitTypes = true
				t.Log("Typed AST:\n" + s.Dump(tc.expr))
			}
		})
	}
}
<|endoftext|>


<commit_before>package minimalweather

import (
	"encoding/json"
	"github.com/gorilla/mux"
	"github.com/elcuervo/geoip"
	"log"
	"fmt"
	"math"
	"strings"
	"os"
	"net/http"
        "html/template"
	"strconv"
)

var c = Pool.Get()

type CityWeather struct {
        City        City        `json:"city"`
	Weather     Weather     `json:"weather"`
        JSON        string      `json:"-"`
}

func outputWeatherAsJSON(current_city City, current_weather Weather) []byte {
	city_weather := &CityWeather{
		City:        current_city,
		Weather:     current_weather}

	out, _ := json.Marshal(city_weather)

	return out
}

func weatherByCity(w http.ResponseWriter, req *http.Request) {
	city_name := mux.Vars(req)["city"]

	log.Println("By Name:", city_name)

	current_city := <-FindByName(city_name)
	current_weather := <-GetWeather(current_city.Coords)

	if current_city.Error != nil {
		http.NotFound(w, req)
	} else {
		out := outputWeatherAsJSON(current_city, current_weather)
		w.Write(out)
	}
}

func weatherByCoords(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)

	lat, _ := strconv.ParseFloat(vars["lat"], 64)
	lng, _ := strconv.ParseFloat(vars["lng"], 64)

	log.Println("By Coords:", lat, lng)

	coords := Coordinates{ lat, lng }
	current_city := FindByCoords(coords)
	current_weather := <-GetWeather(coords)

	out := outputWeatherAsJSON(<-current_city, current_weather)

	w.Write(out)
}

func geolocate(req *http.Request) geoip.Geolocation {
        var user_addr string
        var current_env = os.Getenv("DEVELOPMENT")

        if current_env != "" {
                user_addr = "186.54.253.75"
        } else {
                user_addr = req.RemoteAddr
        }

        return <-GetLocation(user_addr)
}

func homepage(w http.ResponseWriter, req *http.Request) {
        var cw *CityWeather
        var coords Coordinates

        current_cookie, err := req.Cookie("mw-location")

        if err == nil {
                log.Println("From Cookie cache")
                parts := strings.Split(current_cookie.Value, "|")
                lat, _ := strconv.ParseFloat(parts[0], 64)
                lng, _ := strconv.ParseFloat(parts[1], 64)
                coords = Coordinates{ lat, lng }
        } else {
                log.Println("From IP geolocation")
                geo := geolocate(req)
                coords = Coordinates{ geo.Location.Latitude, geo.Location.Longitude }
        }

        city := <-FindByCoords(coords)
        weather := <-GetWeather(city.Coords)

        cw = &CityWeather{ City: city, Weather: weather }
        lat, lng := city.Coords.Lat, city.Coords.Lng

        cookie := &http.Cookie{
                Name: "mw-location",
                Value: fmt.Sprintf("%f|%f", lat, lng),
                Path: "/",
        }

        http.SetCookie(w, cookie)

        t, _ := template.ParseFiles("./website/index.html")
        out, err := json.Marshal(cw)
        cw.JSON = string(out)
        cw.Weather.Temperature = math.Floor(cw.Weather.Temperature)
        err = t.Execute(w, cw)
        if err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
        }
}

func Handler() *mux.Router {
	r := mux.NewRouter()

	r.HandleFunc("/weather/{city}", weatherByCity).Methods("GET")
	r.HandleFunc("/weather/{lat}/{lng}", weatherByCoords).Methods("GET")

	r.PathPrefix("/assets").Handler(http.FileServer(http.Dir("./website/")))

	r.HandleFunc("/", homepage).Methods("GET")

	return r
}
<commit_msg>Get remote ip.. fuck you heroku<commit_after>package minimalweather

import (
	"encoding/json"
	"github.com/gorilla/mux"
	"github.com/elcuervo/geoip"
	"log"
	"fmt"
	"math"
	"strings"
	"net/http"
        "html/template"
	"strconv"
)

var c = Pool.Get()

type CityWeather struct {
        City        City        `json:"city"`
	Weather     Weather     `json:"weather"`
        JSON        string      `json:"-"`
}

func outputWeatherAsJSON(current_city City, current_weather Weather) []byte {
	city_weather := &CityWeather{
		City:        current_city,
		Weather:     current_weather}

	out, _ := json.Marshal(city_weather)

	return out
}

func weatherByCity(w http.ResponseWriter, req *http.Request) {
	city_name := mux.Vars(req)["city"]

	log.Println("By Name:", city_name)

	current_city := <-FindByName(city_name)
	current_weather := <-GetWeather(current_city.Coords)

	if current_city.Error != nil {
		http.NotFound(w, req)
	} else {
		out := outputWeatherAsJSON(current_city, current_weather)
		w.Write(out)
	}
}

func weatherByCoords(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)

	lat, _ := strconv.ParseFloat(vars["lat"], 64)
	lng, _ := strconv.ParseFloat(vars["lng"], 64)

	log.Println("By Coords:", lat, lng)

	coords := Coordinates{ lat, lng }
	current_city := FindByCoords(coords)
	current_weather := <-GetWeather(coords)

	out := outputWeatherAsJSON(<-current_city, current_weather)

	w.Write(out)

}

func ipFromRemote(s string) string {
        index := strings.LastIndex(s, ":")
        if index == -1 {
                return s
        }
        return s[:index]
}

func ipAddress(r *http.Request) string {
        hdr := r.Header
        hdrRealIp := hdr.Get("X-Real-Ip")
        log.Println("real:", hdrRealIp)
        hdrForwardedFor := hdr.Get("X-Forwarded-For")
        log.Println("forwarded:", hdrForwardedFor)

        if hdrRealIp == "" && hdrForwardedFor == "" {
                log.Println("from remote:", ipFromRemote(r.RemoteAddr))
                return ipFromRemote(r.RemoteAddr)
        }

        if hdrForwardedFor != "" {
                // X-Forwarded-For is potentially a list of addresses separated with ","
                parts := strings.Split(hdrForwardedFor, ",")
                for i, p := range parts {
                        parts[i] = strings.TrimSpace(p)
                }
                log.Println("parts:", parts)
                // TODO: should return first non-local address
                return parts[0]
        }
        return hdrRealIp
}

func geolocate(req *http.Request) geoip.Geolocation {
        var user_addr string
        user_addr = ipAddress(req)
        log.Println(user_addr)

        return <-GetLocation(user_addr)
}

func homepage(w http.ResponseWriter, req *http.Request) {
        var cw *CityWeather
        var coords Coordinates

        current_cookie, err := req.Cookie("mw-location")

        if err == nil {
                log.Println("From Cookie cache")
                parts := strings.Split(current_cookie.Value, "|")
                lat, _ := strconv.ParseFloat(parts[0], 64)
                lng, _ := strconv.ParseFloat(parts[1], 64)
                coords = Coordinates{ lat, lng }
        } else {
                log.Println("From geolocation")
                geo := geolocate(req)
                coords = Coordinates{ geo.Location.Latitude, geo.Location.Longitude }
        }

        city := <-FindByCoords(coords)
        weather := <-GetWeather(city.Coords)

        cw = &CityWeather{ City: city, Weather: weather }
        lat, lng := city.Coords.Lat, city.Coords.Lng

        cookie := &http.Cookie{
                Name: "mw-location",
                Value: fmt.Sprintf("%f|%f", lat, lng),
                Path: "/",
        }

        http.SetCookie(w, cookie)

        t, _ := template.ParseFiles("./website/index.html")
        out, err := json.Marshal(cw)
        cw.JSON = string(out)
        cw.Weather.Temperature = math.Floor(cw.Weather.Temperature)
        err = t.Execute(w, cw)
        if err != nil {
                http.Error(w, err.Error(), http.StatusInternalServerError)
        }
}

func Handler() *mux.Router {
	r := mux.NewRouter()

	r.HandleFunc("/weather/{city}", weatherByCity).Methods("GET")
	r.HandleFunc("/weather/{lat}/{lng}", weatherByCoords).Methods("GET")

	r.PathPrefix("/assets").Handler(http.FileServer(http.Dir("./website/")))

	r.HandleFunc("/", homepage).Methods("GET")

	return r
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package layout

import (
	"context"
	"flag"
	"path/filepath"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/os/file"
)

// resolvedLayout is the data file layout discovered by layout()
// Call layout() instead of using this directly.
var resolvedLayout FileLayout

var (
	runfiles = flag.String("runfiles", "", "Location of the runfiles manifest")
)

func layout(ctx context.Context) (out FileLayout) {
	if resolvedLayout != nil {
		return resolvedLayout
	}
	defer func() { resolvedLayout = out }()
	for _, dir := range []file.Path{
		file.ExecutablePath().Parent(),
		file.Abs("."),
	} {
		log.D(ctx, "Looking for package in %v", dir)

		// Check the regular package layout first:
		// pkg
		//  ├─── build.properties
		//  ├─── strings
		//  │     └─── en-us.stb
		//  ├─── gapid-<abi>.apk
		//  ├─── gapir
		//  ├─── gapis
		//  ├─── gapit
		//  ↓
		if layout, err := NewPkgLayout(dir, false); err == nil {
			return layout
		}
	}

	if *runfiles != "" {
		if layout, err := RunfilesLayout(file.Abs(*runfiles)); err == nil {
			return layout
		}
	}

	exe := file.ExecutablePath().System()
	if resolved, err := filepath.EvalSymlinks(exe); err == nil {
		exe = resolved
	}
	log.D(ctx, "Looking for runfiles manifest for %v", exe)
	if path := file.Abs(exe + ".runfiles_manifest"); path.Exists() {
		if layout, err := RunfilesLayout(path); err == nil {
			return layout
		}
	}

	return unknownLayout{}
}

// Strings returns the path to the binary string table.
func Strings(ctx context.Context) (file.Path, error) {
	return layout(ctx).Strings(ctx)
}

// Gapis returns the path to the gapis binary.
func Gapis(ctx context.Context) (file.Path, error) {
	return layout(ctx).Gapis(ctx)
}

// GapidApk returns the path to the gapid.apk corresponding to the given abi.
func GapidApk(ctx context.Context, abi *device.ABI) (file.Path, error) {
	return layout(ctx).GapidApk(ctx, abi)
}

// Gapir returns the path to the gapir binary.
func Gapir(ctx context.Context) (file.Path, error) {
	return layout(ctx).Gapir(ctx)
}

// Gapit returns the path to the gapir binary.
func Gapit(ctx context.Context) (file.Path, error) {
	return layout(ctx).Gapit(ctx)
}

// Library returns the path to the requested library.
func Library(ctx context.Context, lib LibraryType) (file.Path, error) {
	return layout(ctx).Library(ctx, lib)
}

// Json returns the path to the Vulkan layer JSON definition for the given library.
func Json(ctx context.Context, lib LibraryType) (file.Path, error) {
	return layout(ctx).Json(ctx, lib)
}

// GoArgs returns additional arguments to pass to go binaries.
func GoArgs(ctx context.Context) []string {
	return layout(ctx).GoArgs(ctx)
}
<commit_msg>Make the runfiles flag a hidden flag.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package layout

import (
	"context"
	"flag"
	"path/filepath"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/os/file"
)

// resolvedLayout is the data file layout discovered by layout()
// Call layout() instead of using this directly.
var resolvedLayout FileLayout

var (
	runfiles = flag.String("runfiles", "", "_Location of the runfiles manifest")
)

func layout(ctx context.Context) (out FileLayout) {
	if resolvedLayout != nil {
		return resolvedLayout
	}
	defer func() { resolvedLayout = out }()
	for _, dir := range []file.Path{
		file.ExecutablePath().Parent(),
		file.Abs("."),
	} {
		log.D(ctx, "Looking for package in %v", dir)

		// Check the regular package layout first:
		// pkg
		//  ├─── build.properties
		//  ├─── strings
		//  │     └─── en-us.stb
		//  ├─── gapid-<abi>.apk
		//  ├─── gapir
		//  ├─── gapis
		//  ├─── gapit
		//  ↓
		if layout, err := NewPkgLayout(dir, false); err == nil {
			return layout
		}
	}

	if *runfiles != "" {
		if layout, err := RunfilesLayout(file.Abs(*runfiles)); err == nil {
			return layout
		}
	}

	exe := file.ExecutablePath().System()
	if resolved, err := filepath.EvalSymlinks(exe); err == nil {
		exe = resolved
	}
	log.D(ctx, "Looking for runfiles manifest for %v", exe)
	if path := file.Abs(exe + ".runfiles_manifest"); path.Exists() {
		if layout, err := RunfilesLayout(path); err == nil {
			return layout
		}
	}

	return unknownLayout{}
}

// Strings returns the path to the binary string table.
func Strings(ctx context.Context) (file.Path, error) {
	return layout(ctx).Strings(ctx)
}

// Gapis returns the path to the gapis binary.
func Gapis(ctx context.Context) (file.Path, error) {
	return layout(ctx).Gapis(ctx)
}

// GapidApk returns the path to the gapid.apk corresponding to the given abi.
func GapidApk(ctx context.Context, abi *device.ABI) (file.Path, error) {
	return layout(ctx).GapidApk(ctx, abi)
}

// Gapir returns the path to the gapir binary.
func Gapir(ctx context.Context) (file.Path, error) {
	return layout(ctx).Gapir(ctx)
}

// Gapit returns the path to the gapir binary.
func Gapit(ctx context.Context) (file.Path, error) {
	return layout(ctx).Gapit(ctx)
}

// Library returns the path to the requested library.
func Library(ctx context.Context, lib LibraryType) (file.Path, error) {
	return layout(ctx).Library(ctx, lib)
}

// Json returns the path to the Vulkan layer JSON definition for the given library.
func Json(ctx context.Context, lib LibraryType) (file.Path, error) {
	return layout(ctx).Json(ctx, lib)
}

// GoArgs returns additional arguments to pass to go binaries.
func GoArgs(ctx context.Context) []string {
	return layout(ctx).GoArgs(ctx)
}
<|endoftext|>


<commit_before>// Package forge provides an api to deal with parsing configuration files
//
// Config file example:
//
//     # example.cfg
//     top_level = "a string";
//     primary {
//       primary_int = 500;
//       sub_section {
//         sub_float = 50.5;  # End of line comment
//       }
//     }
//     secondary {
//       secondary_bool = true;
//       secondary_null = null;
//
//       # Reference other config value
//       local_ref = .secondary_null;
//       global_ref = primary.sub_section.sub_float;
//
//       # Include all files matching the provided pattern
//       include "/etc/app/*.cfg";
//     }
//
//
// Config file format:
//
//     IDENTIFIER: [_a-zA-Z]+
//
//     BOOL: 'true' | 'false'
//     NULL: 'null'
//     INTEGER: [0-9]+
//     FLOAT: INTEGER '.' INTEGER
//     STRING: '"' .* '"'
//     REFERENCE: [IDENTIFIER] ('.' IDENTIFIER)+
//     VALUE: BOOL | NULL | INTEGER | FLOAT | STRING | REFERENCE
//
//     INCLUDE: 'include ' STRING ';'
//     DIRECTIVE: (IDENTIFIER '=' VALUE | INCLUDE) ';'
//     SECTION: IDENTIFIER '{' (DIRECTIVE | SECTION)* '}'
//     COMMENT: '#' .* NEWLINE '\n'
//
//     CONFIG_FILE: (COMMENT | DIRECTIVE | SECTION)*
//
//
// Values
//  * String:
//      Any value enclosed in double quotes (single quotes not allowed) (e.g. "string")
//  * Integer:
//      Any number without decimal places (e.g. 500)
//  * Float:
//      Any number with decimal places (e.g. 500.55)
//  * Boolean:
//      The identifiers 'true' or 'false' of any case (e.g. TRUE, True, true, FALSE, False, false)
//  * Null:
//      The identifier 'null' of any case (e.g. NULL, Null, null)
//  * Global reference:
//      An identifier which may contain periods, the references are resolved from the global
//      section (e.g. global_value, section.sub_section.value)
//  * Local reference:
//      An identifier which main contain periods which starts with a period, the references
//      are resolved from the settings current section (e.g. .value, .sub_section.value)
//
//
// Directives
//  * Comment:
//      A comment is a pound symbol ('#') followed by any text any which ends with a newline (e.g. '# I am a comment\n')
//      A comment can either be on a line of it's own or at the end of any line. Nothing can come after the comment
//      until after the newline.
//  * Directive:
//      A directive is a setting, a identifier and a value. They are in the format '<identifier> = <value>;'
//      All directives must end in a semicolon. The value can be any of the types defined above.
//  * Section:
//      A section is a grouping of directives under a common name. They are in the format '<section_name> { <directives> }'.
//      All sections must be wrapped in brackets ('{', '}') and must all have a name. They do not end in a semicolon.
//      Sections may be left empty, they do not have to contain any directives.
//  * Include:
//      An include statement tells the config parser to include the contents of another config file where the include
//      statement is defined. Includes are in the format 'include "<pattern>";'. The <pattern> can be any glob
//      like pattern which is compatible with `path.filepath.Match` http://golang.org/pkg/path/filepath/#Match
package forge

import (
	"bytes"
	"io"
	"strings"

	"github.com/brettlangdon/forge/config"
	"github.com/brettlangdon/forge/parser"
)

func ParseString(data string) (*config.SectionValue, error) {
	return parser.ParseReader(strings.NewReader(data))
}

func ParseBytes(data []byte) (*config.SectionValue, error) {
	return parser.ParseReader(bytes.NewReader(data))
}

func ParseFile(filename string) (*config.SectionValue, error) {
	return parser.ParseFile(filename)
}

func ParseReader(reader io.Reader) (*config.SectionValue, error) {
	return parser.ParseReader(reader)
}
<commit_msg>add exported function comments<commit_after>// Package forge provides an api to deal with parsing configuration files
//
// Config file example:
//
//     # example.cfg
//     top_level = "a string";
//     primary {
//       primary_int = 500;
//       sub_section {
//         sub_float = 50.5;  # End of line comment
//       }
//     }
//     secondary {
//       secondary_bool = true;
//       secondary_null = null;
//
//       # Reference other config value
//       local_ref = .secondary_null;
//       global_ref = primary.sub_section.sub_float;
//
//       # Include all files matching the provided pattern
//       include "/etc/app/*.cfg";
//     }
//
//
// Config file format:
//
//     IDENTIFIER: [_a-zA-Z]+
//
//     BOOL: 'true' | 'false'
//     NULL: 'null'
//     INTEGER: [0-9]+
//     FLOAT: INTEGER '.' INTEGER
//     STRING: '"' .* '"'
//     REFERENCE: [IDENTIFIER] ('.' IDENTIFIER)+
//     VALUE: BOOL | NULL | INTEGER | FLOAT | STRING | REFERENCE
//
//     INCLUDE: 'include ' STRING ';'
//     DIRECTIVE: (IDENTIFIER '=' VALUE | INCLUDE) ';'
//     SECTION: IDENTIFIER '{' (DIRECTIVE | SECTION)* '}'
//     COMMENT: '#' .* NEWLINE '\n'
//
//     CONFIG_FILE: (COMMENT | DIRECTIVE | SECTION)*
//
//
// Values
//  * String:
//      Any value enclosed in double quotes (single quotes not allowed) (e.g. "string")
//  * Integer:
//      Any number without decimal places (e.g. 500)
//  * Float:
//      Any number with decimal places (e.g. 500.55)
//  * Boolean:
//      The identifiers 'true' or 'false' of any case (e.g. TRUE, True, true, FALSE, False, false)
//  * Null:
//      The identifier 'null' of any case (e.g. NULL, Null, null)
//  * Global reference:
//      An identifier which may contain periods, the references are resolved from the global
//      section (e.g. global_value, section.sub_section.value)
//  * Local reference:
//      An identifier which main contain periods which starts with a period, the references
//      are resolved from the settings current section (e.g. .value, .sub_section.value)
//
//
// Directives
//  * Comment:
//      A comment is a pound symbol ('#') followed by any text any which ends with a newline (e.g. '# I am a comment\n')
//      A comment can either be on a line of it's own or at the end of any line. Nothing can come after the comment
//      until after the newline.
//  * Directive:
//      A directive is a setting, a identifier and a value. They are in the format '<identifier> = <value>;'
//      All directives must end in a semicolon. The value can be any of the types defined above.
//  * Section:
//      A section is a grouping of directives under a common name. They are in the format '<section_name> { <directives> }'.
//      All sections must be wrapped in brackets ('{', '}') and must all have a name. They do not end in a semicolon.
//      Sections may be left empty, they do not have to contain any directives.
//  * Include:
//      An include statement tells the config parser to include the contents of another config file where the include
//      statement is defined. Includes are in the format 'include "<pattern>";'. The <pattern> can be any glob
//      like pattern which is compatible with `path.filepath.Match` http://golang.org/pkg/path/filepath/#Match
package forge

import (
	"bytes"
	"io"
	"strings"

	"github.com/brettlangdon/forge/config"
	"github.com/brettlangdon/forge/parser"
)

// Parse a forge SectionValue from a string
func ParseString(data string) (*config.SectionValue, error) {
	return parser.ParseReader(strings.NewReader(data))
}

// Parse a forge SectionValue from a byte array
func ParseBytes(data []byte) (*config.SectionValue, error) {
	return parser.ParseReader(bytes.NewReader(data))
}

// Parse a forge SectionValue from a filename
func ParseFile(filename string) (*config.SectionValue, error) {
	return parser.ParseFile(filename)
}

// Parse a forge SectionValue from a io.Reader
func ParseReader(reader io.Reader) (*config.SectionValue, error) {
	return parser.ParseReader(reader)
}
<|endoftext|>


<commit_before>package builder

// nginx
const (
	NginxVersion           = "1.13.0"
	NginxDownloadURLPrefix = "http://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "8.40"
	PcreDownloadURLPrefix = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OpenSSLVersion           = "1.0.2k"
	OpenSSLDownloadURLPrefix = "http://www.openssl.org/source"
)

// zlib
const (
	ZlibVersion           = "1.2.11"
	ZlibDownloadURLPrefix = "http://zlib.net/fossils"
)

// openResty
const (
	OpenRestyVersion           = "1.11.2.3"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.2.0"
	TengineDownloadURLPrefix = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentZlib
	ComponentMax
)
<commit_msg>change the URLs accessible by HTTPS to HTTPS<commit_after>package builder

// nginx
const (
	NginxVersion           = "1.13.0"
	NginxDownloadURLPrefix = "https://nginx.org/download"
)

// pcre
const (
	PcreVersion           = "8.40"
	PcreDownloadURLPrefix = "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre"
)

// openssl
const (
	OpenSSLVersion           = "1.0.2k"
	OpenSSLDownloadURLPrefix = "https://www.openssl.org/source"
)

// zlib
const (
	ZlibVersion           = "1.2.11"
	ZlibDownloadURLPrefix = "https://zlib.net/fossils"
)

// openResty
const (
	OpenRestyVersion           = "1.11.2.3"
	OpenRestyDownloadURLPrefix = "https://openresty.org/download"
)

// tengine
const (
	TengineVersion           = "2.2.0"
	TengineDownloadURLPrefix = "http://tengine.taobao.org/download"
)

// component enumerations
const (
	ComponentNginx = iota
	ComponentOpenResty
	ComponentTengine
	ComponentPcre
	ComponentOpenSSL
	ComponentZlib
	ComponentMax
)
<|endoftext|>


<commit_before>package name

import (
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
	e "github.com/ipfs/go-ipfs/core/commands/e"
	namesys "github.com/ipfs/go-ipfs/namesys"
	nsopts "github.com/ipfs/go-ipfs/namesys/opts"

	cmds "gx/ipfs/QmPTfgFTo9PFr1PvPKyKoeMgBvYPh6cX3aDP7DHKVbnCbi/go-ipfs-cmds"
	logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log"
	offline "gx/ipfs/QmSNe4MWVxZWk6UxxW2z2EKofFo4GdFzud1vfn1iVby3mj/go-ipfs-routing/offline"
	"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
	path "gx/ipfs/QmX7uSbkNz76yNwBhuwYwRbhihLnJqM73VTCjS3UMJud9A/go-path"
)

var log = logging.Logger("core/commands/ipns")

type ResolvedPath struct {
	Path path.Path
}

var IpnsCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Resolve IPNS names.",
		ShortDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default name used is the node's own PeerID,
which is the hash of its public key.
`,
		LongDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default name used is the node's own PeerID,
which is the hash of its public key.

You can use the 'ipfs key' commands to list and generate more names and their
respective keys.

Examples:

Resolve the value of your name:

  > ipfs name resolve
  /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of another name:

  > ipfs name resolve QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ
  /ipfs/QmSiTko9JZyabH56y2fussEt1A5oDqsFXB3CkvAqraFryz

Resolve the value of a dnslink:

  > ipfs name resolve ipfs.io
  /ipfs/QmaBvfZooxWkrv7D3r8LS9moNjzD2o525XMZze69hhoxf5

`,
	},

	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("name", false, false, "The IPNS name to resolve. Defaults to your node's peerID."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption("recursive", "r", "Resolve until the result is not an IPNS name."),
		cmdkit.BoolOption("nocache", "n", "Do not use cached entries."),
		cmdkit.UintOption("dht-record-count", "dhtrc", "Number of records to request for DHT resolution."),
		cmdkit.StringOption("dht-timeout", "dhtt", "Max time to collect values during DHT resolution eg \"30s\". Pass 0 for no timeout."),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := cmdenv.GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		if !n.OnlineMode() {
			err := n.SetupOfflineRouting()
			if err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
		}

		nocache, _ := req.Options["nocache"].(bool)
		local, _ := req.Options["local"].(bool)

		// default to nodes namesys resolver
		var resolver namesys.Resolver = n.Namesys

		if local && nocache {
			res.SetError(errors.New("cannot specify both local and nocache"), cmdkit.ErrNormal)
			return
		}

		if local {
			offroute := offline.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator)
			resolver = namesys.NewIpnsResolver(offroute)
		}

		if nocache {
			resolver = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), 0)
		}

		var name string
		if len(req.Arguments) == 0 {
			if n.Identity == "" {
				res.SetError(errors.New("identity not loaded"), cmdkit.ErrNormal)
				return
			}
			name = n.Identity.Pretty()

		} else {
			name = req.Arguments[0]
		}

		recursive, _ := req.Options["recursive"].(bool)
		rc, rcok := req.Options["dht-record-count"].(int)
		dhtt, dhttok := req.Options["dht-timeout"].(string)
		var ropts []nsopts.ResolveOpt
		if !recursive {
			ropts = append(ropts, nsopts.Depth(1))
		}
		if rcok {
			ropts = append(ropts, nsopts.DhtRecordCount(uint(rc)))
		}
		if dhttok {
			d, err := time.ParseDuration(dhtt)
			if err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
			if d < 0 {
				res.SetError(errors.New("DHT timeout value must be >= 0"), cmdkit.ErrNormal)
				return
			}
			ropts = append(ropts, nsopts.DhtTimeout(d))
		}

		if !strings.HasPrefix(name, "/ipns/") {
			name = "/ipns/" + name
		}

		output, err := resolver.Resolve(req.Context, name, ropts...)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// TODO: better errors (in the case of not finding the name, we get "failed to find any peer in table")

		cmds.EmitOnce(res, &ResolvedPath{output})
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			output, ok := v.(*ResolvedPath)
			if !ok {
				return e.TypeErr(output, v)
			}
			_, err := fmt.Fprintln(w, output.Path)
			return err
		}),
	},
	Type: ResolvedPath{},
}
<commit_msg>refactor ipns command option name<commit_after>package name

import (
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
	e "github.com/ipfs/go-ipfs/core/commands/e"
	namesys "github.com/ipfs/go-ipfs/namesys"
	nsopts "github.com/ipfs/go-ipfs/namesys/opts"

	cmds "gx/ipfs/QmPTfgFTo9PFr1PvPKyKoeMgBvYPh6cX3aDP7DHKVbnCbi/go-ipfs-cmds"
	logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log"
	offline "gx/ipfs/QmSNe4MWVxZWk6UxxW2z2EKofFo4GdFzud1vfn1iVby3mj/go-ipfs-routing/offline"
	"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
	path "gx/ipfs/QmX7uSbkNz76yNwBhuwYwRbhihLnJqM73VTCjS3UMJud9A/go-path"
)

var log = logging.Logger("core/commands/ipns")

type ResolvedPath struct {
	Path path.Path
}

const (
	recursiveOptionName      = "recursive"
	nocacheOptionName        = "nocache"
	dhtRecordCountOptionName = "dht-record-count"
	dhtTimeoutOptionName     = "dht-timeout"
)

var IpnsCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Resolve IPNS names.",
		ShortDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default name used is the node's own PeerID,
which is the hash of its public key.
`,
		LongDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default name used is the node's own PeerID,
which is the hash of its public key.

You can use the 'ipfs key' commands to list and generate more names and their
respective keys.

Examples:

Resolve the value of your name:

  > ipfs name resolve
  /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of another name:

  > ipfs name resolve QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ
  /ipfs/QmSiTko9JZyabH56y2fussEt1A5oDqsFXB3CkvAqraFryz

Resolve the value of a dnslink:

  > ipfs name resolve ipfs.io
  /ipfs/QmaBvfZooxWkrv7D3r8LS9moNjzD2o525XMZze69hhoxf5

`,
	},

	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("name", false, false, "The IPNS name to resolve. Defaults to your node's peerID."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption(recursiveOptionName, "r", "Resolve until the result is not an IPNS name."),
		cmdkit.BoolOption(nocacheOptionName, "n", "Do not use cached entries."),
		cmdkit.UintOption(dhtRecordCountOptionName, "dhtrc", "Number of records to request for DHT resolution."),
		cmdkit.StringOption(dhtTimeoutOptionName, "dhtt", "Max time to collect values during DHT resolution eg \"30s\". Pass 0 for no timeout."),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
		n, err := cmdenv.GetNode(env)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		if !n.OnlineMode() {
			err := n.SetupOfflineRouting()
			if err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
		}

		nocache, _ := req.Options["nocache"].(bool)
		local, _ := req.Options["local"].(bool)

		// default to nodes namesys resolver
		var resolver namesys.Resolver = n.Namesys

		if local && nocache {
			res.SetError(errors.New("cannot specify both local and nocache"), cmdkit.ErrNormal)
			return
		}

		if local {
			offroute := offline.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator)
			resolver = namesys.NewIpnsResolver(offroute)
		}

		if nocache {
			resolver = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), 0)
		}

		var name string
		if len(req.Arguments) == 0 {
			if n.Identity == "" {
				res.SetError(errors.New("identity not loaded"), cmdkit.ErrNormal)
				return
			}
			name = n.Identity.Pretty()

		} else {
			name = req.Arguments[0]
		}

		recursive, _ := req.Options[recursiveOptionName].(bool)
		rc, rcok := req.Options[dhtRecordCountOptionName].(int)
		dhtt, dhttok := req.Options[dhtTimeoutOptionName].(string)

		var ropts []nsopts.ResolveOpt
		if !recursive {
			ropts = append(ropts, nsopts.Depth(1))
		}
		if rcok {
			ropts = append(ropts, nsopts.DhtRecordCount(uint(rc)))
		}
		if dhttok {
			d, err := time.ParseDuration(dhtt)
			if err != nil {
				res.SetError(err, cmdkit.ErrNormal)
				return
			}
			if d < 0 {
				res.SetError(errors.New("DHT timeout value must be >= 0"), cmdkit.ErrNormal)
				return
			}
			ropts = append(ropts, nsopts.DhtTimeout(d))
		}

		if !strings.HasPrefix(name, "/ipns/") {
			name = "/ipns/" + name
		}

		output, err := resolver.Resolve(req.Context, name, ropts...)
		if err != nil {
			res.SetError(err, cmdkit.ErrNormal)
			return
		}

		// TODO: better errors (in the case of not finding the name, we get "failed to find any peer in table")

		cmds.EmitOnce(res, &ResolvedPath{output})
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
			output, ok := v.(*ResolvedPath)
			if !ok {
				return e.TypeErr(output, v)
			}
			_, err := fmt.Fprintln(w, output.Path)
			return err
		}),
	},
	Type: ResolvedPath{},
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	cmds "github.com/jbenet/go-ipfs/commands"
)

type Command struct {
	Name        string
	Subcommands []Command
}

// CommandsCmd takes in a root command,
// and returns a command that lists the subcommands in that root
func CommandsCmd(root *cmds.Command) *cmds.Command {
	return &cmds.Command{
		Helptext: cmds.HelpText{
			Tagline:          "List all available commands.",
			ShortDescription: `Lists all available commands (and subcommands) and exits.`,
		},

		Run: func(req cmds.Request) (interface{}, error) {
			root := outputCommand("ipfs", root)
			return &root, nil
		},
		Marshallers: map[cmds.EncodingType]cmds.Marshaller{
			cmds.Text: func(res cmds.Response) ([]byte, error) {
				v := res.Output().(*Command)
				s := formatCommand("", v)
				return []byte(s), nil
			},
		},
		Type: &Command{},
	}
}

func outputCommand(name string, cmd *cmds.Command) Command {
	output := Command{
		Name:        name,
		Subcommands: make([]Command, len(cmd.Subcommands)),
	}

	i := 0
	for name, sub := range cmd.Subcommands {
		output.Subcommands[i] = outputCommand(name, sub)
		i++
	}

	return output
}

func formatCommand(prefix string, cmd *Command) string {
	if len(prefix) > 0 {
		prefix += " "
	}
	s := fmt.Sprintf("%s%s\n", prefix, cmd.Name)

	prefix += cmd.Name
	for _, sub := range cmd.Subcommands {
		s += formatCommand(prefix, &sub)
	}

	return s
}
<commit_msg>cmds2/commands: better sorting<commit_after>package commands

import (
	"bytes"
	"sort"

	cmds "github.com/jbenet/go-ipfs/commands"
)

type Command struct {
	Name        string
	Subcommands []Command
}

// CommandsCmd takes in a root command,
// and returns a command that lists the subcommands in that root
func CommandsCmd(root *cmds.Command) *cmds.Command {
	return &cmds.Command{
		Helptext: cmds.HelpText{
			Tagline:          "List all available commands.",
			ShortDescription: `Lists all available commands (and subcommands) and exits.`,
		},

		Run: func(req cmds.Request) (interface{}, error) {
			root := cmd2outputCmd("ipfs", root)
			return &root, nil
		},
		Marshallers: map[cmds.EncodingType]cmds.Marshaller{
			cmds.Text: func(res cmds.Response) ([]byte, error) {
				v := res.Output().(*Command)
				var buf bytes.Buffer
				for _, s := range cmdPathStrings(v) {
					buf.Write([]byte(s + "\n"))
				}
				return buf.Bytes(), nil
			},
		},
		Type: &Command{},
	}
}

func cmd2outputCmd(name string, cmd *cmds.Command) Command {
	output := Command{
		Name:        name,
		Subcommands: make([]Command, len(cmd.Subcommands)),
	}

	i := 0
	for name, sub := range cmd.Subcommands {
		output.Subcommands[i] = cmd2outputCmd(name, sub)
		i++
	}

	return output
}

func cmdPathStrings(cmd *Command) []string {
	var cmds []string

	var recurse func(prefix string, cmd *Command)
	recurse = func(prefix string, cmd *Command) {
		cmds = append(cmds, prefix+cmd.Name)
		for _, sub := range cmd.Subcommands {
			recurse(prefix+cmd.Name+" ", &sub)
		}
	}

	recurse("", cmd)
	sort.Sort(sort.StringSlice(cmds))
	return cmds
}
<|endoftext|>


<commit_before>package index

import (
	"errors"
	"fmt"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	"log"
	"sort"
	"sync"
	"sync/atomic"
	"time"
)

// index/BufferedDeletesStream.java

type ApplyDeletesResult struct {
	// True if any actual deletes took place:
	anyDeletes bool

	// Curreng gen, for the merged segment:
	gen int64

	// If non-nil, contains segments that are 100% deleted
	allDeleted []*SegmentInfoPerCommit
}

type SegInfoByDelGen []*SegmentInfoPerCommit

func (a SegInfoByDelGen) Len() int           { return len(a) }
func (a SegInfoByDelGen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a SegInfoByDelGen) Less(i, j int) bool { return a[i].bufferedDeletesGen < a[j].bufferedDeletesGen }

type Query interface{}

type QueryAndLimit struct {
}

type CoalescedDeletes struct {
	_queries map[Query]int
}

func newCoalescedDeletes() *CoalescedDeletes {
	return &CoalescedDeletes{
		_queries: make(map[Query]int),
	}
}

func (cd *CoalescedDeletes) String() string {
	panic("not implemented yet")
}

func (cd *CoalescedDeletes) update(in *FrozenBufferedDeletes) {
	panic("not implemented yet")
}

func (cd *CoalescedDeletes) terms() []*Term {
	panic("not implemented yet")
}

func (cd *CoalescedDeletes) queries() []*QueryAndLimit {
	panic("not implemented yet")
}

/*
Tracks the stream of BufferedDeletes. When DocumentsWriterPerThread
flushes, its buffered deletes are appended to this stream. We later
apply these deletes (resolve them to the actual docIDs, per segment)
when a merge is started (only to the to-be-merged segments). We also
apply to all segments when NRT reader is pulled, commit/close is
called, or when too many deletes are buffered and must be flushed (by
RAM usage or by count).

Each packet is assigned a generation, and each flushed or merged
segment is also assigned a generation, so we can track when
BufferedDeletes packets to apply to any given segment.
*/
type BufferedDeletesStream struct {
	sync.Locker
	// TODO: maybe linked list?
	deletes []*FrozenBufferedDeletes

	// Starts at 1 so that SegmentInfos that have never had deletes
	// applied (whose bufferedDelGen defaults to 0) will be correct:
	nextGen int64

	// used only by assert
	lastDeleteTerm *Term

	infoStream util.InfoStream
	bytesUsed  int64 // atomic
	numTerms   int32 // atomic
}

func newBufferedDeletesStream(infoStream util.InfoStream) *BufferedDeletesStream {
	return &BufferedDeletesStream{
		Locker:     &sync.Mutex{},
		deletes:    make([]*FrozenBufferedDeletes, 0),
		nextGen:    1,
		infoStream: infoStream,
	}
}

func (ds *BufferedDeletesStream) any() bool {
	return atomic.LoadInt64(&ds.bytesUsed) != 0
}

/*
Resolves the buffered deleted Term/Query/docIDs, into actual deleted
docIDs in the liveDocs MutableBits for each SegmentReader.
*/
func (ds *BufferedDeletesStream) applyDeletes(readerPool *ReaderPool, infos []*SegmentInfoPerCommit) (*ApplyDeletesResult, error) {
	ds.Lock()
	defer ds.Unlock()

	if len(infos) == 0 {
		ds.nextGen++
		return &ApplyDeletesResult{false, ds.nextGen - 1, nil}, nil
	}

	t0 := time.Now()
	ds.assertDeleteStats()
	if !ds.any() {
		if ds.infoStream.IsEnabled("BD") {
			ds.infoStream.Message("BD", "applyDeletes: no deletes; skipping")
		}
		ds.nextGen++
		return &ApplyDeletesResult{false, ds.nextGen - 1, nil}, nil
	}

	if ds.infoStream.IsEnabled("BD") {
		ds.infoStream.Message("BD", "applyDeletes: infos=%v packetCount=%v", infos, len(ds.deletes))
	}

	gen := ds.nextGen
	ds.nextGen++

	infos2 := make([]*SegmentInfoPerCommit, len(infos))
	copy(infos2, infos)
	sort.Sort(SegInfoByDelGen(infos2))

	var coalescedDeletes *CoalescedDeletes
	var anyNewDeletes bool

	infosIDX := len(infos2) - 1
	delIDX := len(ds.deletes) - 1

	var allDeleted []*SegmentInfoPerCommit

	for infosIDX >= 0 {
		log.Printf("BD: cycle delIDX=%v infoIDX=%v", delIDX, infosIDX)

		var packet *FrozenBufferedDeletes
		if delIDX >= 0 {
			packet = ds.deletes[delIDX]
		}
		info := infos2[infosIDX]
		segGen := info.bufferedDeletesGen

		if packet != nil && segGen < packet.gen {
			log.Println("  coalesce")
			if coalescedDeletes == nil {
				coalescedDeletes = newCoalescedDeletes()
			}
			if !packet.isSegmentPrivate {
				// Only coalesce if we are NOT on a segment private del
				// packet: the segment private del packet must only be
				// applied to segments with the same delGen. yet, if a
				// segment is already deleted from the SI since it had no
				// more documents remaining after some del packets younger
				// than its segPrivate packet (higher delGen) have been
				// applied, the segPrivate packet has not been removed.
				coalescedDeletes.update(packet)
			}
			delIDX--

		} else if packet != nil && segGen == packet.gen {
			assertn(packet.isSegmentPrivate,
				"Packet and Segments deletegen can only match on a segment private del packet gen=%v",
				segGen)
			log.Println("  eq")

			// Lockorder: IW -> BD -> RP
			assert(readerPool.infoIsLive(info))
			rld := readerPool.get(info, true)
			reader, err := rld.reader(store.IO_CONTEXT_READ)
			if err != nil {
				return nil, err
			}
			delCount, segAllDeletes, err := func() (delCount int64, segAllDeletes bool, err error) {
				defer func() {
					err = mergeError(err, rld.release(reader))
					err = mergeError(err, readerPool.release(rld))
				}()
				if coalescedDeletes != nil {
					log.Println("    del coalesced")
					var delta int64
					delta, err = ds._applyTermDeletes(coalescedDeletes.terms(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
					delta, err = applyQueryDeletes(coalescedDeletes.queries(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
				}
				log.Println("    del exact")
				// Don't delete by Term here; DWPT already did that on flush:
				delta, err := applyQueryDeletes(packet.queries(), rld, reader)
				delCount += delta
				fullDelCount := rld.info.delCount + rld.pendingDeleteCount()
				assert(fullDelCount <= int(rld.info.info.docCount))
				return delCount, fullDelCount == int(rld.info.info.docCount), nil
			}()
			if err != nil {
				return nil, err
			}
			anyNewDeletes = anyNewDeletes || (delCount > 0)

			if segAllDeletes {
				allDeleted = append(allDeleted, info)
			}

			if ds.infoStream.IsEnabled("BD") {
				var suffix string
				if segAllDeletes {
					suffix = " 100%% deleted"
				}
				ds.infoStream.Message("BD", "Seg=%v segGen=%v segDeletes=[%v]; coalesced deletes=[%v] newDelCount=%v%v",
					info, segGen, packet, coalescedDeletes, delCount, suffix)
			}

			if coalescedDeletes == nil {
				coalescedDeletes = newCoalescedDeletes()
			}

			// Since we are on a segment private del packet we must not
			// update the coalescedDeletes here! We can simply advance to
			// the next packet and seginfo.
			delIDX--
			infosIDX--
			info.setBufferedDeletesGen(gen)

		} else {
			log.Println("  gt")

			if coalescedDeletes != nil {
				// Lock order: IW -> BD -> RP
				assert(readerPool.infoIsLive(info))
				rld := readerPool.get(info, true)
				reader, err := rld.reader(store.IO_CONTEXT_READ)
				if err != nil {
					return nil, err
				}
				delCount, segAllDeletes, err := func() (delCount int64, segAllDeletes bool, err error) {
					defer func() {
						err = mergeError(err, rld.release(reader))
						err = mergeError(err, readerPool.release(rld))
					}()
					delta, err := ds._applyTermDeletes(coalescedDeletes.terms(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
					delta, err = applyQueryDeletes(coalescedDeletes.queries(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
					fullDelCount := rld.info.delCount + rld.pendingDeleteCount()
					assert(fullDelCount <= int(rld.info.info.docCount))
					return delCount, fullDelCount == int(rld.info.info.docCount), nil
				}()
				if err != nil {
					return nil, err
				}
				anyNewDeletes = anyNewDeletes || (delCount > 0)

				if segAllDeletes {
					allDeleted = append(allDeleted, info)
				}

				if ds.infoStream.IsEnabled("BD") {
					var suffix string
					if segAllDeletes {
						suffix = " 100%% deleted"
					}
					ds.infoStream.Message("BD", "Seg=%v segGen=%v coalesced deletes=[%v] newDelCount=%v%v",
						info, segGen, coalescedDeletes, delCount, suffix)
				}
			}
			info.setBufferedDeletesGen(gen)

			infosIDX--
		}
	}

	ds.assertDeleteStats()
	if ds.infoStream.IsEnabled("BD") {
		ds.infoStream.Message("BD", "applyDeletes took %v", time.Now().Sub(t0))
	}

	return &ApplyDeletesResult{anyNewDeletes, gen, allDeleted}, nil
}

func mergeError(err, err2 error) error {
	if err == nil {
		return err2
	} else {
		return errors.New(fmt.Sprintf("%v\n  %v", err, err2))
	}
}

// Lock order IW -> BD
/*
Removes any BufferedDeletes that we no longer need to store because
all segments in the index have had the deletes applied.
*/
func (ds *BufferedDeletesStream) prune(infos *SegmentInfos) {
	panic("not implemented yet")
}

/* Delete by term */
func (ds *BufferedDeletesStream) _applyTermDeletes(terms []*Term,
	rld *ReadersAndLiveDocs, reader *SegmentReader) (int64, error) {
	panic("not implemented yet")
}

/* Delete by query */
func applyQueryDeletes(queries []*QueryAndLimit,
	rld *ReadersAndLiveDocs, reader *SegmentReader) (int64, error) {
	panic("not implemented yet")
}

func (ds *BufferedDeletesStream) assertDeleteStats() {
	var numTerms2 int
	var bytesUsed2 int64
	for _, packet := range ds.deletes {
		numTerms2 += packet.numTermDeletes
		bytesUsed2 += int64(packet.bytesUsed)
	}
	n1 := int(atomic.LoadInt32(&ds.numTerms))
	assertn(numTerms2 == n1, "numTerms2=%v vs %v", numTerms2, n1)
	n2 := int64(atomic.LoadInt64(&ds.bytesUsed))
	assertn(bytesUsed2 == n2, "bytesUsed2=%v vs %v", bytesUsed2, n2)
}
<commit_msg>implement DS.prune()<commit_after>package index

import (
	"errors"
	"fmt"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	"log"
	"math"
	"sort"
	"sync"
	"sync/atomic"
	"time"
)

// index/BufferedDeletesStream.java

type ApplyDeletesResult struct {
	// True if any actual deletes took place:
	anyDeletes bool

	// Curreng gen, for the merged segment:
	gen int64

	// If non-nil, contains segments that are 100% deleted
	allDeleted []*SegmentInfoPerCommit
}

type SegInfoByDelGen []*SegmentInfoPerCommit

func (a SegInfoByDelGen) Len() int           { return len(a) }
func (a SegInfoByDelGen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a SegInfoByDelGen) Less(i, j int) bool { return a[i].bufferedDeletesGen < a[j].bufferedDeletesGen }

type Query interface{}

type QueryAndLimit struct {
}

type CoalescedDeletes struct {
	_queries map[Query]int
}

func newCoalescedDeletes() *CoalescedDeletes {
	return &CoalescedDeletes{
		_queries: make(map[Query]int),
	}
}

func (cd *CoalescedDeletes) String() string {
	panic("not implemented yet")
}

func (cd *CoalescedDeletes) update(in *FrozenBufferedDeletes) {
	panic("not implemented yet")
}

func (cd *CoalescedDeletes) terms() []*Term {
	panic("not implemented yet")
}

func (cd *CoalescedDeletes) queries() []*QueryAndLimit {
	panic("not implemented yet")
}

/*
Tracks the stream of BufferedDeletes. When DocumentsWriterPerThread
flushes, its buffered deletes are appended to this stream. We later
apply these deletes (resolve them to the actual docIDs, per segment)
when a merge is started (only to the to-be-merged segments). We also
apply to all segments when NRT reader is pulled, commit/close is
called, or when too many deletes are buffered and must be flushed (by
RAM usage or by count).

Each packet is assigned a generation, and each flushed or merged
segment is also assigned a generation, so we can track when
BufferedDeletes packets to apply to any given segment.
*/
type BufferedDeletesStream struct {
	sync.Locker
	// TODO: maybe linked list?
	deletes []*FrozenBufferedDeletes

	// Starts at 1 so that SegmentInfos that have never had deletes
	// applied (whose bufferedDelGen defaults to 0) will be correct:
	nextGen int64

	// used only by assert
	lastDeleteTerm *Term

	infoStream util.InfoStream
	bytesUsed  int64 // atomic
	numTerms   int32 // atomic
}

func newBufferedDeletesStream(infoStream util.InfoStream) *BufferedDeletesStream {
	return &BufferedDeletesStream{
		Locker:     &sync.Mutex{},
		deletes:    make([]*FrozenBufferedDeletes, 0),
		nextGen:    1,
		infoStream: infoStream,
	}
}

func (ds *BufferedDeletesStream) any() bool {
	return atomic.LoadInt64(&ds.bytesUsed) != 0
}

/*
Resolves the buffered deleted Term/Query/docIDs, into actual deleted
docIDs in the liveDocs MutableBits for each SegmentReader.
*/
func (ds *BufferedDeletesStream) applyDeletes(readerPool *ReaderPool, infos []*SegmentInfoPerCommit) (*ApplyDeletesResult, error) {
	ds.Lock()
	defer ds.Unlock()

	if len(infos) == 0 {
		ds.nextGen++
		return &ApplyDeletesResult{false, ds.nextGen - 1, nil}, nil
	}

	t0 := time.Now()
	ds.assertDeleteStats()
	if !ds.any() {
		if ds.infoStream.IsEnabled("BD") {
			ds.infoStream.Message("BD", "applyDeletes: no deletes; skipping")
		}
		ds.nextGen++
		return &ApplyDeletesResult{false, ds.nextGen - 1, nil}, nil
	}

	if ds.infoStream.IsEnabled("BD") {
		ds.infoStream.Message("BD", "applyDeletes: infos=%v packetCount=%v", infos, len(ds.deletes))
	}

	gen := ds.nextGen
	ds.nextGen++

	infos2 := make([]*SegmentInfoPerCommit, len(infos))
	copy(infos2, infos)
	sort.Sort(SegInfoByDelGen(infos2))

	var coalescedDeletes *CoalescedDeletes
	var anyNewDeletes bool

	infosIDX := len(infos2) - 1
	delIDX := len(ds.deletes) - 1

	var allDeleted []*SegmentInfoPerCommit

	for infosIDX >= 0 {
		log.Printf("BD: cycle delIDX=%v infoIDX=%v", delIDX, infosIDX)

		var packet *FrozenBufferedDeletes
		if delIDX >= 0 {
			packet = ds.deletes[delIDX]
		}
		info := infos2[infosIDX]
		segGen := info.bufferedDeletesGen

		if packet != nil && segGen < packet.gen {
			log.Println("  coalesce")
			if coalescedDeletes == nil {
				coalescedDeletes = newCoalescedDeletes()
			}
			if !packet.isSegmentPrivate {
				// Only coalesce if we are NOT on a segment private del
				// packet: the segment private del packet must only be
				// applied to segments with the same delGen. yet, if a
				// segment is already deleted from the SI since it had no
				// more documents remaining after some del packets younger
				// than its segPrivate packet (higher delGen) have been
				// applied, the segPrivate packet has not been removed.
				coalescedDeletes.update(packet)
			}
			delIDX--

		} else if packet != nil && segGen == packet.gen {
			assertn(packet.isSegmentPrivate,
				"Packet and Segments deletegen can only match on a segment private del packet gen=%v",
				segGen)
			log.Println("  eq")

			// Lockorder: IW -> BD -> RP
			assert(readerPool.infoIsLive(info))
			rld := readerPool.get(info, true)
			reader, err := rld.reader(store.IO_CONTEXT_READ)
			if err != nil {
				return nil, err
			}
			delCount, segAllDeletes, err := func() (delCount int64, segAllDeletes bool, err error) {
				defer func() {
					err = mergeError(err, rld.release(reader))
					err = mergeError(err, readerPool.release(rld))
				}()
				if coalescedDeletes != nil {
					log.Println("    del coalesced")
					var delta int64
					delta, err = ds._applyTermDeletes(coalescedDeletes.terms(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
					delta, err = applyQueryDeletes(coalescedDeletes.queries(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
				}
				log.Println("    del exact")
				// Don't delete by Term here; DWPT already did that on flush:
				delta, err := applyQueryDeletes(packet.queries(), rld, reader)
				delCount += delta
				fullDelCount := rld.info.delCount + rld.pendingDeleteCount()
				assert(fullDelCount <= int(rld.info.info.docCount))
				return delCount, fullDelCount == int(rld.info.info.docCount), nil
			}()
			if err != nil {
				return nil, err
			}
			anyNewDeletes = anyNewDeletes || (delCount > 0)

			if segAllDeletes {
				allDeleted = append(allDeleted, info)
			}

			if ds.infoStream.IsEnabled("BD") {
				var suffix string
				if segAllDeletes {
					suffix = " 100%% deleted"
				}
				ds.infoStream.Message("BD", "Seg=%v segGen=%v segDeletes=[%v]; coalesced deletes=[%v] newDelCount=%v%v",
					info, segGen, packet, coalescedDeletes, delCount, suffix)
			}

			if coalescedDeletes == nil {
				coalescedDeletes = newCoalescedDeletes()
			}

			// Since we are on a segment private del packet we must not
			// update the coalescedDeletes here! We can simply advance to
			// the next packet and seginfo.
			delIDX--
			infosIDX--
			info.setBufferedDeletesGen(gen)

		} else {
			log.Println("  gt")

			if coalescedDeletes != nil {
				// Lock order: IW -> BD -> RP
				assert(readerPool.infoIsLive(info))
				rld := readerPool.get(info, true)
				reader, err := rld.reader(store.IO_CONTEXT_READ)
				if err != nil {
					return nil, err
				}
				delCount, segAllDeletes, err := func() (delCount int64, segAllDeletes bool, err error) {
					defer func() {
						err = mergeError(err, rld.release(reader))
						err = mergeError(err, readerPool.release(rld))
					}()
					delta, err := ds._applyTermDeletes(coalescedDeletes.terms(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
					delta, err = applyQueryDeletes(coalescedDeletes.queries(), rld, reader)
					if err != nil {
						return
					}
					delCount += delta
					fullDelCount := rld.info.delCount + rld.pendingDeleteCount()
					assert(fullDelCount <= int(rld.info.info.docCount))
					return delCount, fullDelCount == int(rld.info.info.docCount), nil
				}()
				if err != nil {
					return nil, err
				}
				anyNewDeletes = anyNewDeletes || (delCount > 0)

				if segAllDeletes {
					allDeleted = append(allDeleted, info)
				}

				if ds.infoStream.IsEnabled("BD") {
					var suffix string
					if segAllDeletes {
						suffix = " 100%% deleted"
					}
					ds.infoStream.Message("BD", "Seg=%v segGen=%v coalesced deletes=[%v] newDelCount=%v%v",
						info, segGen, coalescedDeletes, delCount, suffix)
				}
			}
			info.setBufferedDeletesGen(gen)

			infosIDX--
		}
	}

	ds.assertDeleteStats()
	if ds.infoStream.IsEnabled("BD") {
		ds.infoStream.Message("BD", "applyDeletes took %v", time.Now().Sub(t0))
	}

	return &ApplyDeletesResult{anyNewDeletes, gen, allDeleted}, nil
}

func mergeError(err, err2 error) error {
	if err == nil {
		return err2
	} else {
		return errors.New(fmt.Sprintf("%v\n  %v", err, err2))
	}
}

// Lock order IW -> BD
/*
Removes any BufferedDeletes that we no longer need to store because
all segments in the index have had the deletes applied.
*/
func (ds *BufferedDeletesStream) prune(infos *SegmentInfos) {
	ds.assertDeleteStats()
	var minGen int64 = math.MaxInt64
	for _, info := range infos.Segments {
		if info.bufferedDeletesGen < minGen {
			minGen = info.bufferedDeletesGen
		}
	}

	if ds.infoStream.IsEnabled("BD") {
		ds.infoStream.Message("BD", "prune sis=%v minGen=%v packetCount=%v",
			infos, minGen, len(ds.deletes))
	}
	for delIDX, limit := 0, len(ds.deletes); delIDX < limit; delIDX++ {
		if ds.deletes[delIDX].gen >= minGen {
			ds.pruneDeletes(delIDX)
			ds.assertDeleteStats()
			return
		}
	}

	// All deletes pruned
	ds.pruneDeletes(len(ds.deletes))
	assert(!ds.any())
	ds.assertDeleteStats()
}

func (ds *BufferedDeletesStream) pruneDeletes(count int) {
	if count > 0 {
		if ds.infoStream.IsEnabled("BD") {
			ds.infoStream.Message("BD", "pruneDeletes: prune %v packets; %v packets remain",
				count, len(ds.deletes)-count)
		}
		for delIDX := 0; delIDX < count; delIDX++ {
			packet := ds.deletes[delIDX]
			n := atomic.AddInt32(&ds.numTerms, -int32(packet.numTermDeletes))
			assert(n >= 0)
			n2 := atomic.AddInt64(&ds.bytesUsed, -int64(packet.bytesUsed))
			assert(n2 >= 0)
			ds.deletes[delIDX] = nil
		}
		ds.deletes = ds.deletes[count:]
	}
}

/* Delete by term */
func (ds *BufferedDeletesStream) _applyTermDeletes(terms []*Term,
	rld *ReadersAndLiveDocs, reader *SegmentReader) (int64, error) {
	panic("not implemented yet")
}

/* Delete by query */
func applyQueryDeletes(queries []*QueryAndLimit,
	rld *ReadersAndLiveDocs, reader *SegmentReader) (int64, error) {
	panic("not implemented yet")
}

func (ds *BufferedDeletesStream) assertDeleteStats() {
	var numTerms2 int
	var bytesUsed2 int64
	for _, packet := range ds.deletes {
		numTerms2 += packet.numTermDeletes
		bytesUsed2 += int64(packet.bytesUsed)
	}
	n1 := int(atomic.LoadInt32(&ds.numTerms))
	assertn(numTerms2 == n1, "numTerms2=%v vs %v", numTerms2, n1)
	n2 := int64(atomic.LoadInt64(&ds.bytesUsed))
	assertn(bytesUsed2 == n2, "bytesUsed2=%v vs %v", bytesUsed2, n2)
}
<|endoftext|>


<commit_before>package pager

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

var (
	Endpoint   = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
	ServiceKey = ""
)

type Pager struct {
	Endpoint   string
	ServiceKey string
}

func New(serviceKey string) *Pager {
	return &Pager{Endpoint, serviceKey}
}

// -- Trigger

func Trigger(description string) (incidentKey string, err error) {
	return trigger(description, "", map[string]interface{}{})
}

func TriggerIncidentKey(description string, key string) (incidentKey string, err error) {
	return trigger(description, key, map[string]interface{}{})
}

func TriggerWithDetails(description string, details map[string]interface{}) (incidentKey string, err error) {
	return trigger(description, "", details)
}

func TriggerIncidentKeyWithDetails(description string, key string, details map[string]interface{}) (incidentKey string, err error) {
	return trigger(description, key, details)
}

func trigger(description string, key string, details map[string]interface{}) (newIncidentKey string, err error) {
	p := Pager{Endpoint, ServiceKey}
	return p.trigger(description, key, details)
}

func (p *Pager) Trigger(description string) (incidentKey string, err error) {
	return p.trigger(description, "", map[string]interface{}{})
}

func (p *Pager) TriggerIncidentKey(description string, key string) (incidentKey string, err error) {
	return p.trigger(description, key, map[string]interface{}{})
}

func (p *Pager) TriggerWithDetails(description string, details map[string]interface{}) (incidentKey string, err error) {
	return p.trigger(description, "", details)
}

func (p *Pager) TriggerIncidentKeyWithDetails(description string, key string, details map[string]interface{}) (incidentKey string, err error) {
	return p.trigger(description, key, details)
}

func (p *Pager) trigger(description string, incidentKey string, details map[string]interface{}) (newIncidentKey string, err error) {
	payload, err := triggerIncidentJSON(p.ServiceKey, description, incidentKey, details)
	if err != nil {
		return "", err
	}

	respBody, err := sendPostRequestWithJSON(p.Endpoint, payload)
	if err != nil {
		return "", err
	}

	return respBody["incident_key"], nil
}

// -- Resolve

func ResolveIncidentKey(incidentKey string) (resp bool, err error) {
	return resolve(incidentKey)
}

func resolve(incidentKey string) (resp bool, err error) {
	p := Pager{Endpoint, ServiceKey}
	return p.resolve(incidentKey)
}

func (p *Pager) ResolveIncidentKey(incidentKey string) (resp bool, err error) {
	return p.resolve(incidentKey)
}

func (p *Pager) resolve(incidentKey string) (resp bool, err error) {
	payload, err := resolveIncidentJSON(p.ServiceKey, incidentKey)
	if err != nil {
		return false, err
	}

	respBody, err := sendPostRequestWithJSON(p.Endpoint, payload)
	if err != nil {
		return false, err
	}

	return respBody["status"] == "success", nil
}

// -- Utils

// sendPostRequestWithJSON sends a POST request with a JSON Payload
// to the specified endpoint
func sendPostRequestWithJSON(endpoint string, payload []byte) (respBody map[string]string, err error) {
	resp, err := http.Post(endpoint, "application/json", bytes.NewReader(payload))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	err = responseError(resp)
	if err != nil {
		return nil, err
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	respBody := map[string]string{}
	err = json.Unmarshal(bodyBytes, &respBody)
	if err != nil {
		return nil, err
	}

	return respBody, nil
}

// triggerIncidentJSON builds the JSON payload for triggering a new PagerDuty
// incident.
func triggerIncidentJSON(serviceKey string, description string, incidentKey string, details map[string]interface{}) (payload []byte, err error) {
	payload := map[string]interface{}{
		"service_key": serviceKey,
		"event_type":  "trigger",
		"description": description,
		"details":     details,
	}
	if len(incidentKey) > 0 {
		payload["incident_key"] = incidentKey
	}
	return json.Marshal(payload)
}

// resolveIncidentJSON builds the JSON payload for resoling a PagerDuty
// incident
func resolveIncidentJSON(serviceKey string, incidentKey string) (payload []byte, err error) {
	payload := map[string]interface{}{
		"service_key":  serviceKey,
		"event_type":   "resolve",
		"incident_key": incidentKey,
	}
	return json.Marshal(payload)
}

// responseError returns an error if the given PagerDuty response is an error
// response.
func responseError(resp *http.Response) (err error) {
	if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusNoContent {
		return nil
	}
	body, _ := ioutil.ReadAll(resp.Body)
	return fmt.Errorf("%s: %s", resp.Status, string(body))
}
<commit_msg>Fixed Utility Functions<commit_after>package pager

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

var (
	Endpoint   = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
	ServiceKey = ""
)

type Pager struct {
	Endpoint   string
	ServiceKey string
}

func New(serviceKey string) *Pager {
	return &Pager{Endpoint, serviceKey}
}

// -- Trigger

func Trigger(description string) (incidentKey string, err error) {
	return trigger(description, "", map[string]interface{}{})
}

func TriggerIncidentKey(description string, key string) (incidentKey string, err error) {
	return trigger(description, key, map[string]interface{}{})
}

func TriggerWithDetails(description string, details map[string]interface{}) (incidentKey string, err error) {
	return trigger(description, "", details)
}

func TriggerIncidentKeyWithDetails(description string, key string, details map[string]interface{}) (incidentKey string, err error) {
	return trigger(description, key, details)
}

func trigger(description string, key string, details map[string]interface{}) (newIncidentKey string, err error) {
	p := Pager{Endpoint, ServiceKey}
	return p.trigger(description, key, details)
}

func (p *Pager) Trigger(description string) (incidentKey string, err error) {
	return p.trigger(description, "", map[string]interface{}{})
}

func (p *Pager) TriggerIncidentKey(description string, key string) (incidentKey string, err error) {
	return p.trigger(description, key, map[string]interface{}{})
}

func (p *Pager) TriggerWithDetails(description string, details map[string]interface{}) (incidentKey string, err error) {
	return p.trigger(description, "", details)
}

func (p *Pager) TriggerIncidentKeyWithDetails(description string, key string, details map[string]interface{}) (incidentKey string, err error) {
	return p.trigger(description, key, details)
}

func (p *Pager) trigger(description string, incidentKey string, details map[string]interface{}) (newIncidentKey string, err error) {
	payload, err := triggerIncidentJSON(p.ServiceKey, description, incidentKey, details)
	if err != nil {
		return "", err
	}

	respBody, err := sendPostRequestWithJSON(p.Endpoint, payload)
	if err != nil {
		return "", err
	}

	return respBody["incident_key"], nil
}

// -- Resolve

func ResolveIncidentKey(incidentKey string) (resp bool, err error) {
	return resolve(incidentKey)
}

func resolve(incidentKey string) (resp bool, err error) {
	p := Pager{Endpoint, ServiceKey}
	return p.resolve(incidentKey)
}

func (p *Pager) ResolveIncidentKey(incidentKey string) (resp bool, err error) {
	return p.resolve(incidentKey)
}

func (p *Pager) resolve(incidentKey string) (resp bool, err error) {
	payload, err := resolveIncidentJSON(p.ServiceKey, incidentKey)
	if err != nil {
		return false, err
	}

	respBody, err := sendPostRequestWithJSON(p.Endpoint, payload)
	if err != nil {
		return false, err
	}

	return respBody["status"] == "success", nil
}

// -- Utils

// sendPostRequestWithJSON sends a POST request with a JSON Payload
// to the specified endpoint
func sendPostRequestWithJSON(endpoint string, payload []byte) (map[string]string, error) {
	resp, err := http.Post(endpoint, "application/json", bytes.NewReader(payload))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	err = responseError(resp)
	if err != nil {
		return nil, err
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	respBody := map[string]string{}
	err = json.Unmarshal(bodyBytes, &respBody)
	if err != nil {
		return nil, err
	}

	return respBody, nil
}

// triggerIncidentJSON builds the JSON payload for triggering a new PagerDuty
// incident.
func triggerIncidentJSON(serviceKey string, description string, incidentKey string, details map[string]interface{}) ([]byte, error) {
	payload := map[string]interface{}{
		"service_key": serviceKey,
		"event_type":  "trigger",
		"description": description,
		"details":     details,
	}
	if len(incidentKey) > 0 {
		payload["incident_key"] = incidentKey
	}
	return json.Marshal(payload)
}

// resolveIncidentJSON builds the JSON payload for resoling a PagerDuty
// incident
func resolveIncidentJSON(serviceKey string, incidentKey string) ([]byte, error) {
	payload := map[string]interface{}{
		"service_key":  serviceKey,
		"event_type":   "resolve",
		"incident_key": incidentKey,
	}
	return json.Marshal(payload)
}

// responseError returns an error if the given PagerDuty response is an error
// response.
func responseError(resp *http.Response) (err error) {
	if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusNoContent {
		return nil
	}
	body, _ := ioutil.ReadAll(resp.Body)
	return fmt.Errorf("%s: %s", resp.Status, string(body))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package godata

import (
	"bytes"
	"fmt"

	"github.com/google/btree"
)

// Frame represents multiple rows and multiple columns of data.
type Frame struct {
	bt *btree.BTree

	indexer Indexer
}

// NewFrame returns a Frame for the given indexer.
func NewFrame(indexer Indexer) *Frame {
	return &Frame{
		bt:      btree.New(2),
		indexer: indexer,
	}
}

// Put inserts the data into the frame, replacing and returning the existing
// data if an entry already exists. Returns error if the data cannot be
// indexed.
func (f *Frame) Put(data RowData) (RowData, error) {
	index, err := f.indexer.Index(data)
	if err != nil {
		return nil, fmt.Errorf("AddRow: %v", err)
	}

	row := f.bt.ReplaceOrInsert(Row{
		index: index,
		data:  data,
	})

	if row == nil {
		return nil, nil
	}

	return row.(Row).data, nil
}

// Get returns the data for the given key. Returns error if the given key is
// invalid. Returns nil if there is no data for the given key.
func (f *Frame) Get(key RowData) (RowData, error) {
	index, err := f.indexer.Index(key)
	if err != nil {
		return nil, err
	}

	got := f.bt.Get(index)
	if got == nil {
		return nil, nil
	}

	return got.(Row).data, nil
}

// Pop returns the data for the given key and deletes it from the Frame.
// Returns error if the given key is invalid. Returns nil if there is no data
// for the given key.
func (f *Frame) Pop(key RowData) (RowData, error) {
	index, err := f.indexer.Index(key)
	if err != nil {
		return nil, err
	}

	got := f.bt.Delete(index)
	if got == nil {
		return nil, nil
	}
	return got.(Row).data, nil
}

// rangeOptions represents a begin and end point for range functions.
type rangeOptions struct {
	greaterOrEqual RowData
	lessThan       RowData
}

// rangeArg mutate a rangeOptions based on a given argument.
type rangeArg func(*rangeOptions)

// rangeArgsToOptions converts the given rangeArgs into an options struct.
func rangeArgsToOptions(args []rangeArg) *rangeOptions {
	var opts rangeOptions
	for _, a := range args {
		a(&opts)
	}
	return &opts
}

// GreaterOrEqual returns a range option that filters on rows greater than or
// equal to the given value.
func GreaterOrEqual(val RowData) rangeArg {
	return func(opts *rangeOptions) {
		opts.greaterOrEqual = val
	}
}

// LessThan returns a range option that filters on rows less than or equal to
// the given value.
func LessThan(val RowData) rangeArg {
	return func(opts *rangeOptions) {
		opts.lessThan = val
	}
}

// RowAction performs an operation on the given row and optionally returns a
// value. RowAction must not mutate the RowData.
type RowAction func(RowData) (interface{}, error)

// rowAction performs an operation on the given Row. Actions may mutate the
// data, but must also re-insert the row into the btree to maintain
// synchronization of the index.
type rowAction func(Row) (interface{}, error)

// forRange performs an action for a given key range and returns the array of
// results, one for each row.
func (f *Frame) forRange(opts *rangeOptions, action rowAction) ([]interface{}, error) {
	var (
		returnError  error
		returnValues []interface{}
	)
	iterator := func(item btree.Item) bool {
		val, err := action(item.(Row))
		if err != nil {
			returnError = err
			return false
		}
		returnValues = append(returnValues, val)
		return true
	}

	if opts.lessThan == nil && opts.greaterOrEqual == nil {
		f.bt.Ascend(iterator)
	} else if opts.lessThan == nil {
		pivot, err := f.indexer.Index(opts.greaterOrEqual)
		if err != nil {
			return nil, err
		}
		f.bt.AscendGreaterOrEqual(pivot, iterator)
	} else if opts.greaterOrEqual == nil {
		pivot, err := f.indexer.Index(opts.lessThan)
		if err != nil {
			return nil, err
		}
		f.bt.AscendLessThan(pivot, iterator)
	} else {
		begin, err := f.indexer.Index(opts.greaterOrEqual)
		if err != nil {
			return nil, err
		}
		end, err := f.indexer.Index(opts.lessThan)
		if err != nil {
			return nil, err
		}
		f.bt.AscendRange(begin, end, iterator)
	}

	return returnValues, returnError
}

// String returns the string representation of the Frame.
//
// TODO: The output is ugly and potentially very long. Refactor the forRange to
// take a limit, and then print out a partial representation of the Frame if
// there are too many rows.
func (f *Frame) String() string {
	rows, err := f.GetRange()
	if err != nil {
		return fmt.Sprintf("Frame.String: %v", err)
	}
	var buf bytes.Buffer
	for i, r := range rows {
		buf.WriteString(fmt.Sprintf("%d: %v\n", i, r))
	}
	return buf.String()
}

// GetRange returns a list of all values in the given range. See GreaterOrEqual
// and LessThan. If no range is given, then this function returns all rows. If
// only a begin range is given, then this function returns all rows beginning
// with the given value. If only an end range is given, then this function
// returns all rows up to the given value.
func (f *Frame) GetRange(args ...rangeArg) ([]RowData, error) {
	opts := rangeArgsToOptions(args)
	rows, err := f.forRange(opts, func(row Row) (interface{}, error) {
		return row.data, nil
	})
	if err != nil {
		return nil, err
	}

	var castRows []RowData
	for _, r := range rows {
		castRows = append(castRows, r.(RowData))
	}
	return castRows, nil
}

// PopRange returns a list of all values in the given range and deletes them
// from the Frame. See GetRange for details on the arguments.
func (f *Frame) PopRange(args ...rangeArg) ([]RowData, error) {
	opts := rangeArgsToOptions(args)
	rows, err := f.forRange(opts, func(row Row) (interface{}, error) {
		return row, nil
	})
	if err != nil {
		return nil, err
	}

	var (
		indices []Index
		data    []RowData
	)
	for _, r := range rows {
		indices = append(indices, r.(Row).index)
		data = append(data, r.(Row).data)
	}

	for _, i := range indices {
		f.bt.Delete(i)
	}

	return data, nil
}

// Joined returns a new Frame object that contains the joined contents of the
// two frames. The indices of the frames must be compatible. The resulting
// Frame contains one row for each key in the union of keys for the left and
// right frames. The RowData contains a JoinResult for each column of data,
// where Left is populated with the left side contents, and Right is populated
// with the right side contents. Left and Right are nil if they don't exist in
// the left and right sides.
func (f *Frame) Joined(frame *Frame) (*Frame, error) {
	fr := NewFrame(JoinResultIndexer{f.indexer})

	// Add all left rows.
	all, err := f.GetRange()
	if err != nil {
		return nil, err
	}
	for _, row := range all {
		// Convert each column to a JoinResult.
		projected := make(map[string]interface{})
		for col, val := range row {
			projected[col] = &JoinResult{Left: val}
		}

		_, err := fr.Put(projected)
		if err != nil {
			return nil, err
		}
	}

	// Add all right rows.
	all, err = frame.GetRange()
	if err != nil {
		return nil, err
	}
	for _, row := range all {
		projected, err := fr.Get(row)
		if err != nil {
			return nil, err
		}
		if projected == nil {
			projected = make(map[string]interface{})
		}

		// Add Right to each column JoinResult, or add a JoinResult if the column doesn't exist yet.
		for col, val := range row {
			joinResult, ok := projected[col]
			if ok {
				jd, ok := joinResult.(*JoinResult)
				if !ok {
					return nil, fmt.Errorf("Frame: column %q in %v is not a JoinResult", col, joinResult)
				}
				jd.Right = val
			} else {
				projected[col] = &JoinResult{Right: val}
			}
		}
		fr.Put(projected)
		if err != nil {
			return nil, err
		}
	}

	return fr, nil
}

// WithIndexer returns a new Frame object with the same underlying data indexed
// by a new indexer. Returns error if the data cannot be indexed by the new
// indexer. Note that mutating rows in the returned Frame will also mutate the
// rows in the existing Frame. However, adding to or deleting rows from the
// returned Frame will not add to and delete from the existing Frame.
func (f *Frame) WithIndexer(indexer Indexer) (*Frame, error) {
	var returnErr error

	nf := NewFrame(indexer)
	iter := func(item btree.Item) bool {
		_, err := nf.Put(item.(Row).data)
		if err != nil {
			returnErr = err
			return false
		}
		return true
	}

	f.bt.Ascend(iter)

	return nf, returnErr
}
<commit_msg>Clarify that WithIndex may drop rows.<commit_after>/*
Copyright 2014 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package godata

import (
	"bytes"
	"fmt"

	"github.com/google/btree"
)

// Frame represents multiple rows and multiple columns of data.
type Frame struct {
	bt *btree.BTree

	indexer Indexer
}

// NewFrame returns a Frame for the given indexer.
func NewFrame(indexer Indexer) *Frame {
	return &Frame{
		bt:      btree.New(2),
		indexer: indexer,
	}
}

// Put inserts the data into the frame, replacing and returning the existing
// data if an entry already exists. Returns error if the data cannot be
// indexed.
func (f *Frame) Put(data RowData) (RowData, error) {
	index, err := f.indexer.Index(data)
	if err != nil {
		return nil, fmt.Errorf("AddRow: %v", err)
	}

	row := f.bt.ReplaceOrInsert(Row{
		index: index,
		data:  data,
	})

	if row == nil {
		return nil, nil
	}

	return row.(Row).data, nil
}

// Get returns the data for the given key. Returns error if the given key is
// invalid. Returns nil if there is no data for the given key.
func (f *Frame) Get(key RowData) (RowData, error) {
	index, err := f.indexer.Index(key)
	if err != nil {
		return nil, err
	}

	got := f.bt.Get(index)
	if got == nil {
		return nil, nil
	}

	return got.(Row).data, nil
}

// Pop returns the data for the given key and deletes it from the Frame.
// Returns error if the given key is invalid. Returns nil if there is no data
// for the given key.
func (f *Frame) Pop(key RowData) (RowData, error) {
	index, err := f.indexer.Index(key)
	if err != nil {
		return nil, err
	}

	got := f.bt.Delete(index)
	if got == nil {
		return nil, nil
	}
	return got.(Row).data, nil
}

// rangeOptions represents a begin and end point for range functions.
type rangeOptions struct {
	greaterOrEqual RowData
	lessThan       RowData
}

// rangeArg mutate a rangeOptions based on a given argument.
type rangeArg func(*rangeOptions)

// rangeArgsToOptions converts the given rangeArgs into an options struct.
func rangeArgsToOptions(args []rangeArg) *rangeOptions {
	var opts rangeOptions
	for _, a := range args {
		a(&opts)
	}
	return &opts
}

// GreaterOrEqual returns a range option that filters on rows greater than or
// equal to the given value.
func GreaterOrEqual(val RowData) rangeArg {
	return func(opts *rangeOptions) {
		opts.greaterOrEqual = val
	}
}

// LessThan returns a range option that filters on rows less than or equal to
// the given value.
func LessThan(val RowData) rangeArg {
	return func(opts *rangeOptions) {
		opts.lessThan = val
	}
}

// RowAction performs an operation on the given row and optionally returns a
// value. RowAction must not mutate the RowData.
type RowAction func(RowData) (interface{}, error)

// rowAction performs an operation on the given Row. Actions may mutate the
// data, but must also re-insert the row into the btree to maintain
// synchronization of the index.
type rowAction func(Row) (interface{}, error)

// forRange performs an action for a given key range and returns the array of
// results, one for each row.
func (f *Frame) forRange(opts *rangeOptions, action rowAction) ([]interface{}, error) {
	var (
		returnError  error
		returnValues []interface{}
	)
	iterator := func(item btree.Item) bool {
		val, err := action(item.(Row))
		if err != nil {
			returnError = err
			return false
		}
		returnValues = append(returnValues, val)
		return true
	}

	if opts.lessThan == nil && opts.greaterOrEqual == nil {
		f.bt.Ascend(iterator)
	} else if opts.lessThan == nil {
		pivot, err := f.indexer.Index(opts.greaterOrEqual)
		if err != nil {
			return nil, err
		}
		f.bt.AscendGreaterOrEqual(pivot, iterator)
	} else if opts.greaterOrEqual == nil {
		pivot, err := f.indexer.Index(opts.lessThan)
		if err != nil {
			return nil, err
		}
		f.bt.AscendLessThan(pivot, iterator)
	} else {
		begin, err := f.indexer.Index(opts.greaterOrEqual)
		if err != nil {
			return nil, err
		}
		end, err := f.indexer.Index(opts.lessThan)
		if err != nil {
			return nil, err
		}
		f.bt.AscendRange(begin, end, iterator)
	}

	return returnValues, returnError
}

// String returns the string representation of the Frame.
//
// TODO: The output is ugly and potentially very long. Refactor the forRange to
// take a limit, and then print out a partial representation of the Frame if
// there are too many rows.
func (f *Frame) String() string {
	rows, err := f.GetRange()
	if err != nil {
		return fmt.Sprintf("Frame.String: %v", err)
	}
	var buf bytes.Buffer
	for i, r := range rows {
		buf.WriteString(fmt.Sprintf("%d: %v\n", i, r))
	}
	return buf.String()
}

// GetRange returns a list of all values in the given range. See GreaterOrEqual
// and LessThan. If no range is given, then this function returns all rows. If
// only a begin range is given, then this function returns all rows beginning
// with the given value. If only an end range is given, then this function
// returns all rows up to the given value.
func (f *Frame) GetRange(args ...rangeArg) ([]RowData, error) {
	opts := rangeArgsToOptions(args)
	rows, err := f.forRange(opts, func(row Row) (interface{}, error) {
		return row.data, nil
	})
	if err != nil {
		return nil, err
	}

	var castRows []RowData
	for _, r := range rows {
		castRows = append(castRows, r.(RowData))
	}
	return castRows, nil
}

// PopRange returns a list of all values in the given range and deletes them
// from the Frame. See GetRange for details on the arguments.
func (f *Frame) PopRange(args ...rangeArg) ([]RowData, error) {
	opts := rangeArgsToOptions(args)
	rows, err := f.forRange(opts, func(row Row) (interface{}, error) {
		return row, nil
	})
	if err != nil {
		return nil, err
	}

	var (
		indices []Index
		data    []RowData
	)
	for _, r := range rows {
		indices = append(indices, r.(Row).index)
		data = append(data, r.(Row).data)
	}

	for _, i := range indices {
		f.bt.Delete(i)
	}

	return data, nil
}

// Joined returns a new Frame object that contains the joined contents of the
// two frames. The indices of the frames must be compatible. The resulting
// Frame contains one row for each key in the union of keys for the left and
// right frames. The RowData contains a JoinResult for each column of data,
// where Left is populated with the left side contents, and Right is populated
// with the right side contents. Left and Right are nil if they don't exist in
// the left and right sides.
func (f *Frame) Joined(frame *Frame) (*Frame, error) {
	fr := NewFrame(JoinResultIndexer{f.indexer})

	// Add all left rows.
	all, err := f.GetRange()
	if err != nil {
		return nil, err
	}
	for _, row := range all {
		// Convert each column to a JoinResult.
		projected := make(map[string]interface{})
		for col, val := range row {
			projected[col] = &JoinResult{Left: val}
		}

		_, err := fr.Put(projected)
		if err != nil {
			return nil, err
		}
	}

	// Add all right rows.
	all, err = frame.GetRange()
	if err != nil {
		return nil, err
	}
	for _, row := range all {
		projected, err := fr.Get(row)
		if err != nil {
			return nil, err
		}
		if projected == nil {
			projected = make(map[string]interface{})
		}

		// Add Right to each column JoinResult, or add a JoinResult if the column doesn't exist yet.
		for col, val := range row {
			joinResult, ok := projected[col]
			if ok {
				jd, ok := joinResult.(*JoinResult)
				if !ok {
					return nil, fmt.Errorf("Frame: column %q in %v is not a JoinResult", col, joinResult)
				}
				jd.Right = val
			} else {
				projected[col] = &JoinResult{Right: val}
			}
		}
		fr.Put(projected)
		if err != nil {
			return nil, err
		}
	}

	return fr, nil
}

// WithIndexer returns a new Frame object with the same underlying data indexed
// by a new indexer. Returns error if the data cannot be indexed by the new
// indexer. Note that mutating rows in the returned Frame will also mutate the
// rows in the existing Frame. However, adding to or deleting rows from the
// returned Frame will not add to and delete from the existing Frame.
// WithIndexer assumes that the given Indexer gives each existing row a unique
// Index. If rows share an index, then one of the rows will be dropped. The
// dropped row is not defined by the API, and is subject to change.
func (f *Frame) WithIndexer(indexer Indexer) (*Frame, error) {
	var returnErr error

	nf := NewFrame(indexer)
	iter := func(item btree.Item) bool {
		_, err := nf.Put(item.(Row).data)
		if err != nil {
			returnErr = err
			return false
		}
		return true
	}

	f.bt.Ascend(iter)

	return nf, returnErr
}
<|endoftext|>