<commit_before>package perigee

import (
  "fmt"
	"bytes"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

func TestNormal(t *testing.T) {
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	response, err := Request("GET", ts.URL, Options{})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}
	if response.StatusCode != 200 {
		t.Fatalf("response code %d is not 200", response.StatusCode)
	}
}

func TestOKCodes(t *testing.T) {
	expectCode := 201
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(expectCode)
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	options := Options{
		OkCodes: []int{expectCode},
	}
	results, err := Request("GET", ts.URL, options)
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}
	if results.StatusCode != expectCode {
		t.Fatalf("response code %d is not %d", results.StatusCode, expectCode)
	}
}

func TestLocation(t *testing.T) {
	newLocation := "http://www.example.com"
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Location", newLocation)
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	response, err := Request("GET", ts.URL, Options{})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	location, err := response.HttpResponse.Location()
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	if location.String() != newLocation {
		t.Fatalf("location returned \"%s\" is not \"%s\"", location.String(), newLocation)
	}
}

func TestHeaders(t *testing.T) {
	newLocation := "http://www.example.com"
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Location", newLocation)
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	response, err := Request("GET", ts.URL, Options{})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	location := response.HttpResponse.Header.Get("Location")
	if location == "" {
		t.Fatalf("Location should not empty")
	}

	if location != newLocation {
		t.Fatalf("location returned \"%s\" is not \"%s\"", location, newLocation)
	}
}

func TestCustomHeaders(t *testing.T) {
	var contentType, accept, contentLength string

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		m := map[string][]string(r.Header)
		contentType = m["Content-Type"][0]
		accept = m["Accept"][0]
		contentLength = m["Content-Length"][0]
	})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	_, err := Request("GET", ts.URL, Options{
		ContentLength: 5,
		ContentType:   "x-application/vb",
		Accept:        "x-application/c",
		ReqBody:       strings.NewReader("Hello"),
	})
	if err != nil {
		t.Fatalf(err.Error())
	}

	if contentType != "x-application/vb" {
		t.Fatalf("I expected x-application/vb; got ", contentType)
	}

	if contentLength != "5" {
		t.Fatalf("I expected 5 byte content length; got ", contentLength)
	}

	if accept != "x-application/c" {
		t.Fatalf("I expected x-application/c; got ", accept)
	}
}

func TestJson(t *testing.T) {
	newLocation := "http://www.example.com"
	jsonBytes := []byte(`{"foo": {"bar": "baz"}}`)
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Location", newLocation)
			w.Write(jsonBytes)
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	type Data struct {
		Foo struct {
			Bar string `json:"bar"`
		} `json:"foo"`
	}
	var data Data

	response, err := Request("GET", ts.URL, Options{Results: &data})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	if bytes.Compare(jsonBytes, response.JsonResult) != 0 {
		t.Fatalf("json returned \"%s\" is not \"%s\"", response.JsonResult, jsonBytes)
	}

	if data.Foo.Bar != "baz" {
		t.Fatalf("Results returned %v", data)
	}
}

func TestSetHeaders(t *testing.T) {
  var wasCalled bool
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("Hi"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

  _, err := Request("GET", ts.URL, Options{
    SetHeaders: func(r *http.Request) error {
      wasCalled = true
      return nil
    },
  })

  if err != nil {
    t.Fatal(err)
  }

  if !wasCalled {
    t.Fatal("I expected header setter callback to be called, but it wasn't")
  }

  myError := fmt.Errorf("boo")

  _, err = Request("GET", ts.URL, Options{
    SetHeaders: func(r *http.Request) error {
      return myError
    },
  })

  if err != myError {
    t.Fatal("I expected errors to propegate back to the caller.")
  }
}

<commit_msg>Adding test<commit_after>package perigee

import (
	"bytes"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

func TestNormal(t *testing.T) {
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	response, err := Request("GET", ts.URL, Options{})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}
	if response.StatusCode != 200 {
		t.Fatalf("response code %d is not 200", response.StatusCode)
	}
}

func TestOKCodes(t *testing.T) {
	expectCode := 201
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(expectCode)
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	options := Options{
		OkCodes: []int{expectCode},
	}
	results, err := Request("GET", ts.URL, options)
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}
	if results.StatusCode != expectCode {
		t.Fatalf("response code %d is not %d", results.StatusCode, expectCode)
	}
}

func TestLocation(t *testing.T) {
	newLocation := "http://www.example.com"
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Location", newLocation)
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	response, err := Request("GET", ts.URL, Options{})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	location, err := response.HttpResponse.Location()
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	if location.String() != newLocation {
		t.Fatalf("location returned \"%s\" is not \"%s\"", location.String(), newLocation)
	}
}

func TestHeaders(t *testing.T) {
	newLocation := "http://www.example.com"
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Location", newLocation)
			w.Write([]byte("testing"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	response, err := Request("GET", ts.URL, Options{})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	location := response.HttpResponse.Header.Get("Location")
	if location == "" {
		t.Fatalf("Location should not empty")
	}

	if location != newLocation {
		t.Fatalf("location returned \"%s\" is not \"%s\"", location, newLocation)
	}
}

func TestCustomHeaders(t *testing.T) {
	var contentType, accept, contentLength string

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		m := map[string][]string(r.Header)
		contentType = m["Content-Type"][0]
		accept = m["Accept"][0]
		contentLength = m["Content-Length"][0]
	})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	_, err := Request("GET", ts.URL, Options{
		ContentLength: 5,
		ContentType:   "x-application/vb",
		Accept:        "x-application/c",
		ReqBody:       strings.NewReader("Hello"),
	})
	if err != nil {
		t.Fatalf(err.Error())
	}

	if contentType != "x-application/vb" {
		t.Fatalf("I expected x-application/vb; got ", contentType)
	}

	if contentLength != "5" {
		t.Fatalf("I expected 5 byte content length; got ", contentLength)
	}

	if accept != "x-application/c" {
		t.Fatalf("I expected x-application/c; got ", accept)
	}
}

func TestJson(t *testing.T) {
	newLocation := "http://www.example.com"
	jsonBytes := []byte(`{"foo": {"bar": "baz"}}`)
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Location", newLocation)
			w.Write(jsonBytes)
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	type Data struct {
		Foo struct {
			Bar string `json:"bar"`
		} `json:"foo"`
	}
	var data Data

	response, err := Request("GET", ts.URL, Options{Results: &data})
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	if bytes.Compare(jsonBytes, response.JsonResult) != 0 {
		t.Fatalf("json returned \"%s\" is not \"%s\"", response.JsonResult, jsonBytes)
	}

	if data.Foo.Bar != "baz" {
		t.Fatalf("Results returned %v", data)
	}
}

func TestSetHeaders(t *testing.T) {
	var wasCalled bool
	handler := http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("Hi"))
		})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	_, err := Request("GET", ts.URL, Options{
		SetHeaders: func(r *http.Request) error {
			wasCalled = true
			return nil
		},
	})

	if err != nil {
		t.Fatal(err)
	}

	if !wasCalled {
		t.Fatal("I expected header setter callback to be called, but it wasn't")
	}

	myError := fmt.Errorf("boo")

	_, err = Request("GET", ts.URL, Options{
		SetHeaders: func(r *http.Request) error {
			return myError
		},
	})

	if err != myError {
		t.Fatal("I expected errors to propegate back to the caller.")
	}
}

func TestBodilessMethodsAreSentWithoutContentHeaders(t *testing.T) {
	var h map[string][]string

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h = r.Header
	})
	ts := httptest.NewServer(handler)
	defer ts.Close()

	_, err := Request("GET", ts.URL, Options{})
	if err != nil {
		t.Fatalf(err.Error())
	}

	if len(h["Content-Type"]) != 0 {
		t.Fatalf("I expected nothing for Content-Type but got ", h["Content-Type"])
	}

	if len(h["Content-Length"]) != 0 {
		t.Fatalf("I expected nothing for Content-Length but got ", h["Content-Type"])
	}
}
<|endoftext|>


<commit_before>package goxp

import (
	"bytes"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"
)
<commit_msg>recovery_test.go +func Test_Recovery<commit_after>package goxp

import (
	"bytes"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"
)

func Test_Recovery(t *testing.T) {
	buff := bytes.NewBufferString("")
	recorder := httptest.NewRecorder()

	setENV(Dev)
	m := New()
	// replace log for testing
	m.Map(log.New(buff, "[goxp] ", 0))
	m.Use(func(res http.ResponseWriter, req *http.Request) {
		res.Header().Set("Content-Type", "unpredictable")
	})
	m.Use(Recovery())
	m.Use(func(res http.ResponseWriter, req *http.Request) {
		panic("here is a panic!")
	})
	m.ServeHTTP(recorder, (*http.Request)(nil))
	expect(t, recorder.Code, http.StatusInternalServerError)
	expect(t, recorder.HeaderMap.Get("Content-Type"), "text/html")
	refute(t, recorder.Body.Len(), 0)
	refute(t, len(buff.String()), 0)
}
<|endoftext|>


<commit_before>package pngtile

/*
#include "pngtile.h"
*/
import "C"
import (
	"time"
)

func makeTime(t C.time_t) time.Time {
	return time.Unix(int64(t), 0)
}

func makeImageInfo(ci *C.struct_pt_image_info) ImageInfo {
	return ImageInfo{
		ImageWidth:        uint(ci.image_width),
		ImageHeight:       uint(ci.image_height),
		ImageBPP:          uint(ci.image_bpp),
		ImageModifiedTime: makeTime(ci.image_mtime),
		ImageBytes:        uint(ci.image_bytes),
		CacheVersion:      int(ci.cache_version),
		CacheModifiedTime: makeTime(ci.cache_mtime),
		CacheBytes:        uint(ci.cache_bytes),
		CacheBlocks:       uint(ci.cache_blocks),
	}
}

type ImageInfo struct {
	ImageWidth, ImageHeight, ImageBPP uint
	ImageModifiedTime                 time.Time
	ImageBytes                        uint
	CacheVersion                      int
	CacheModifiedTime                 time.Time
	CacheBytes                        uint
	CacheBlocks                       uint
}
<commit_msg>go pngtile ImageInfo JSON<commit_after>package pngtile

/*
#include "pngtile.h"
*/
import "C"
import (
	"time"
)

func makeTime(t C.time_t) time.Time {
	return time.Unix(int64(t), 0)
}

func makeImageInfo(ci *C.struct_pt_image_info) ImageInfo {
	return ImageInfo{
		ImageWidth:        uint(ci.image_width),
		ImageHeight:       uint(ci.image_height),
		ImageBPP:          uint(ci.image_bpp),
		ImageModifiedTime: makeTime(ci.image_mtime),
		ImageBytes:        uint(ci.image_bytes),
		CacheVersion:      int(ci.cache_version),
		CacheModifiedTime: makeTime(ci.cache_mtime),
		CacheBytes:        uint(ci.cache_bytes),
		CacheBlocks:       uint(ci.cache_blocks),
	}
}

type ImageInfo struct {
	ImageWidth        uint      `json:"image_width"`
	ImageHeight       uint      `json:"image_height"`
	ImageBPP          uint      `json:"image_bpp"`
	ImageModifiedTime time.Time `json:"image_mtime"`
	ImageBytes        uint      `json:"image_bytes"`
	CacheVersion      int       `json:"cache_version"`
	CacheModifiedTime time.Time `json:"cache_mtime"`
	CacheBytes        uint      `json:"cache_bytes"`
	CacheBlocks       uint      `json:"cache_blocks"`
}
<|endoftext|>


<commit_before>package main

// reads in a reddit comment archive URL
//  and just extracts the body field
import (
	"compress/bzip2"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sync"
)

// globals used for worker queues and global counts
var wg sync.WaitGroup

// Reddit is struct used to unmarshal the reddit comment
type Reddit struct {
	Body string `json:"body"`
}

// doit does the following
//   reads a URL
//   uncompresses it (bzip2)
//   json decodes
//   extracts comment body
//   writes to output file as mini-json
//
//
func doit(prefix, url string) {
	const maxLines = 1 << 31

	// generate outputfile name
	// blah/RC_2015-06.gz --> RC_2015-06-counts.csv
	base := path.Base(url)
	ext := filepath.Ext(base)
	base = base[0:len(base)-len(ext)] + "-body.json.gz"
	log.Printf("[%s] %s -> %s", prefix, url, base)

	resp, err := http.Get(url)
	if err != nil {
		log.Fatalf("[%s] url error: %s", prefix, err)
	}
	defer resp.Body.Close()
	file := bzip2.NewReader(resp.Body)

	// no need to buffer this since raw network and bzip2 will
	// naturally buffer the input
	jsonin := json.NewDecoder(file)

	// set up output file
	fo, err := os.Create(base)
	if err != nil {
		log.Fatalf("[%s] unable to write: %s", prefix, err)
	}
	// gzip output
	bufout := gzip.NewWriter(fo)
	// steam out json
	jsonout := json.NewEncoder(bufout)

	obj := Reddit{}
	lines := 0
	for jsonin.More() && lines < maxLines {
		lines++
		// decode an array value (Message)
		obj.Body = ""
		err := jsonin.Decode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to unmarshal object: %s", prefix, err)
		}
		if obj.Body == "[deleted]" || obj.Body == "" {
			continue
		}
		err = jsonout.Encode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to marshal object: %s", prefix, err)
		}
	}

	bufout.Close()
	fo.Close()
	log.Printf("[%s] done %d lines", prefix, lines)
}

func worker(id int, jobs <-chan string) {
	for j := range jobs {
		doit(fmt.Sprintf("%d:%s", id, j), j)
	}
	wg.Done()
}

func main() {

	args := []string{}
	year := 2015
	for month := 1; month <= 12; month++ {
		arg := fmt.Sprintf("http://files.pushshift.io/reddit/comments/RC_%d-%02d.bz2", year, month)
		args = append(args, arg)
	}

	jobs := make(chan string, len(args))

	numCPU := runtime.NumCPU()
	for w := 1; w <= numCPU; w++ {
		wg.Add(1)
		go worker(w, jobs)
	}

	for _, arg := range args {
		log.Printf("[MASTER]: adding %s", arg)
		jobs <- arg
	}
	close(jobs)
	wg.Wait()
	log.Printf("[MASTER]: done")
}
<commit_msg>iteratoe of year/months<commit_after>package main

// reads in a reddit comment archive URL
//  and just extracts the body field
import (
	"compress/bzip2"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sync"
)

// globals used for worker queues and global counts
var wg sync.WaitGroup

// Reddit is struct used to unmarshal the reddit comment
type Reddit struct {
	Body string `json:"body"`
}

// doit does the following
//   reads a URL
//   uncompresses it (bzip2)
//   json decodes
//   extracts comment body
//   writes to output file as mini-json
//
//
func doit(prefix, url string) {
	const maxLines = 1 << 31

	// generate outputfile name
	// blah/RC_2015-06.gz --> RC_2015-06-counts.csv
	base := path.Base(url)
	ext := filepath.Ext(base)
	base = base[0:len(base)-len(ext)] + "-body.json.gz"
	log.Printf("[%s] %s -> %s", prefix, url, base)

	resp, err := http.Get(url)
	if err != nil {
		log.Fatalf("[%s] url error: %s", prefix, err)
	}
	defer resp.Body.Close()
	file := bzip2.NewReader(resp.Body)

	// no need to buffer this since raw network and bzip2 will
	// naturally buffer the input
	jsonin := json.NewDecoder(file)

	// set up output file
	fo, err := os.Create(base)
	if err != nil {
		log.Fatalf("[%s] unable to write: %s", prefix, err)
	}
	// gzip output
	bufout := gzip.NewWriter(fo)
	// steam out json
	jsonout := json.NewEncoder(bufout)

	obj := Reddit{}
	lines := 0
	for jsonin.More() && lines < maxLines {
		lines++
		// decode an array value (Message)
		obj.Body = ""
		err := jsonin.Decode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to unmarshal object: %s", prefix, err)
		}
		if obj.Body == "[deleted]" || obj.Body == "" {
			continue
		}
		err = jsonout.Encode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to marshal object: %s", prefix, err)
		}
	}

	bufout.Close()
	fo.Close()
	log.Printf("[%s] done %d lines", prefix, lines)
}

func worker(id int, jobs <-chan string) {
	for j := range jobs {
		doit(fmt.Sprintf("%d:%s", id, j), j)
	}
	wg.Done()
}

type yearmonth struct {
	year  int
	month int
}

func yearmonthRange(start, end yearmonth) []yearmonth {
	out := []yearmonth{}
	m := start.month
	y := start.year
	for {
		out = append(out, yearmonth{y, m})
		m++
		if m == 13 {
			m = 1
			y++
		}
		if y > end.year {
			break
		}
		if y == end.year && m > end.month {
			break
		}
	}
	return out
}

func main() {
	dates := yearmonthRange(yearmonth{2010, 1}, yearmonth{2011, 12})
	for _, ym := range dates {
		arg := fmt.Sprintf("http://files.pushshift.io/reddit/comments/RC_%d-%02d.bz2", ym.year, ym.month)
		args = append(args, arg)
	}

	jobs := make(chan string, len(args))

	numCPU := runtime.NumCPU()
	for w := 1; w <= numCPU; w++ {
		wg.Add(1)
		go worker(w, jobs)
	}

	for _, arg := range args {
		log.Printf("[MASTER]: adding %s", arg)
		jobs <- arg
	}
	close(jobs)
	wg.Wait()
	log.Printf("[MASTER]: done")
}
<|endoftext|>


<commit_before>package libkb

import (
	"encoding/hex"
	"fmt"
	"net/url"
	"strconv"
)

type HTTPValue interface {
	String() string
}

type HTTPArgs map[string]HTTPValue

type S struct {
	Val string
}

func HexArg(b []byte) S {
	return S{Val: hex.EncodeToString(b)}
}

type I struct {
	Val int
}

type U struct {
	Val uint64
}

type UHex struct {
	Val uint64
}

type B struct {
	Val bool
}

func (a *HTTPArgs) Add(s string, v HTTPValue) {
	(*a)[s] = v
}

func NewHTTPArgs() HTTPArgs {
	return make(HTTPArgs)
}

func (s S) String() string    { return s.Val }
func (i I) String() string    { return strconv.Itoa(i.Val) }
func (u U) String() string    { return strconv.FormatUint(u.Val, 10) }
func (h UHex) String() string { return fmt.Sprintf("%016x", h.Val) }
func (b B) String() string {
	if b.Val {
		return "1"
	}
	return "0"
}

func (a HTTPArgs) ToValues() url.Values {
	ret := url.Values{}
	for k, v := range a {
		ret.Set(k, v.String())
	}
	return ret
}

func (a HTTPArgs) EncodeToString() string {
	return a.ToValues().Encode()
}

func HTTPArgsFromKeyValuePair(key string, val HTTPValue) HTTPArgs {
	ret := HTTPArgs{}
	ret[key] = val
	return ret
}
<commit_msg>Add base64 encoded arg<commit_after>package libkb

import (
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"net/url"
	"strconv"
)

type HTTPValue interface {
	String() string
}

type HTTPArgs map[string]HTTPValue

type S struct {
	Val string
}

func HexArg(b []byte) S {
	return S{Val: hex.EncodeToString(b)}
}

func B64Arg(b []byte) S {
	return S{Val: base64.StdEncoding.EncodeToString(b)}
}

type I struct {
	Val int
}

type U struct {
	Val uint64
}

type UHex struct {
	Val uint64
}

type B struct {
	Val bool
}

func (a *HTTPArgs) Add(s string, v HTTPValue) {
	(*a)[s] = v
}

func NewHTTPArgs() HTTPArgs {
	return make(HTTPArgs)
}

func (s S) String() string    { return s.Val }
func (i I) String() string    { return strconv.Itoa(i.Val) }
func (u U) String() string    { return strconv.FormatUint(u.Val, 10) }
func (h UHex) String() string { return fmt.Sprintf("%016x", h.Val) }
func (b B) String() string {
	if b.Val {
		return "1"
	}
	return "0"
}

func (a HTTPArgs) ToValues() url.Values {
	ret := url.Values{}
	for k, v := range a {
		ret.Set(k, v.String())
	}
	return ret
}

func (a HTTPArgs) EncodeToString() string {
	return a.ToValues().Encode()
}

func HTTPArgsFromKeyValuePair(key string, val HTTPValue) HTTPArgs {
	ret := HTTPArgs{}
	ret[key] = val
	return ret
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package key

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"sort"
	"strings"
)

var MinKey = KeyspaceId("")
var MaxKey = KeyspaceId(strings.Repeat("\xff", 64))

// NOTE(msolomon) not sure about all these types - feels like it will create
// hoops later.
type Uint64Key uint64

func (i Uint64Key) String() string {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, uint64(i))
	return buf.String()
}

func (i Uint64Key) KeyspaceId() KeyspaceId {
	return KeyspaceId(i.String())
}

type KeyspaceId string

func (kid KeyspaceId) Hex() HexKeyspaceId {
	return HexKeyspaceId(hex.EncodeToString([]byte(kid)))
}

func (kid KeyspaceId) MarshalJSON() ([]byte, error) {
	return []byte("\"" + string(kid.Hex()) + "\""), nil
}

func (kid *KeyspaceId) UnmarshalJSON(data []byte) error {
	*kid = HexKeyspaceId(data[1 : len(data)-1]).Unhex()
	return nil
}

type HexKeyspaceId string

func (hkid HexKeyspaceId) Unhex() KeyspaceId {
	b, err := hex.DecodeString(string(hkid))
	if err != nil {
		panic(err)
	}
	return KeyspaceId(string(b))
}

type KeyRange struct {
	Start KeyspaceId
	End   KeyspaceId
}

func (kr KeyRange) MapKey() string {
	return string(kr.Start) + "-" + string(kr.End)
}

func (kr KeyRange) Contains(i KeyspaceId) bool {
	return kr.Start < i && i <= kr.End
}

type KeyspaceRange struct {
	Keyspace string
	KeyRange
}

type KeyspaceIdArray []KeyspaceId

func (p KeyspaceIdArray) Len() int { return len(p) }

func (p KeyspaceIdArray) Less(i, j int) bool {
	return p[i] < p[j]
}

func (p KeyspaceIdArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func (p KeyspaceIdArray) Sort() { sort.Sort(p) }

type KeyRangeArray []KeyRange

func (p KeyRangeArray) Len() int { return len(p) }

func (p KeyRangeArray) Less(i, j int) bool {
	return p[i].Start < p[j].Start
}

func (p KeyRangeArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func (p KeyRangeArray) Sort() { sort.Sort(p) }
<commit_msg>match new range bounds following recent discussions<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package key

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"sort"
	"strings"
)

var MinKey = KeyspaceId("")
var MaxKey = KeyspaceId("")

// NOTE(msolomon) not sure about all these types - feels like it will create
// hoops later.
type Uint64Key uint64

func (i Uint64Key) String() string {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, uint64(i))
	return buf.String()
}

func (i Uint64Key) KeyspaceId() KeyspaceId {
	return KeyspaceId(i.String())
}

type KeyspaceId string

func (kid KeyspaceId) Hex() HexKeyspaceId {
	return HexKeyspaceId(hex.EncodeToString([]byte(kid)))
}

func (kid KeyspaceId) MarshalJSON() ([]byte, error) {
	return []byte("\"" + string(kid.Hex()) + "\""), nil
}

func (kid *KeyspaceId) UnmarshalJSON(data []byte) error {
	*kid = HexKeyspaceId(data[1 : len(data)-1]).Unhex()
	return nil
}

type HexKeyspaceId string

func (hkid HexKeyspaceId) Unhex() KeyspaceId {
	b, err := hex.DecodeString(string(hkid))
	if err != nil {
		panic(err)
	}
	return KeyspaceId(string(b))
}

type KeyRange struct {
	Start KeyspaceId
	End   KeyspaceId
}

func (kr KeyRange) MapKey() string {
	return string(kr.Start) + "-" + string(kr.End)
}

func (kr KeyRange) Contains(i KeyspaceId) bool {
	return kr.Start <= i && kr.End != MaxKey && i < kr.End
}

type KeyspaceRange struct {
	Keyspace string
	KeyRange
}

type KeyspaceIdArray []KeyspaceId

func (p KeyspaceIdArray) Len() int { return len(p) }

func (p KeyspaceIdArray) Less(i, j int) bool {
	return p[i] < p[j]
}

func (p KeyspaceIdArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func (p KeyspaceIdArray) Sort() { sort.Sort(p) }

type KeyRangeArray []KeyRange

func (p KeyRangeArray) Len() int { return len(p) }

func (p KeyRangeArray) Less(i, j int) bool {
	return p[i].Start < p[j].Start
}

func (p KeyRangeArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func (p KeyRangeArray) Sort() { sort.Sort(p) }
<|endoftext|>


<commit_before>package main

import "testing"

func TestGetBitsFromPacket(t *testing.T) {

	var bytePos int
	var bitPos int
	tests := []struct {
		packet []byte
		bpP    int
		ret    uint8
	}{
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 24, 255},
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 12, 240},
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 9, 14},
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 3, 1},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 24, 0},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 12, 0},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 9, 0},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 3, 0},
	}

	for _, test := range tests {
		res := getBitsFromPacket(test.packet, &bytePos, &bitPos, test.bpP)
		if res != test.ret {
			t.Errorf("Input: %d Expected: %d \t Got %d", test.packet, test.ret, res)
		}
	}
}
<commit_msg>Adjust test<commit_after>package main

import "testing"

func TestGetBitsFromPacket(t *testing.T) {

	var bytePos int
	var bitPos int
	tests := []struct {
		packet []byte
		bpP    uint
		ret    uint8
	}{
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 24, 255},
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 12, 240},
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 9, 14},
		{[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, 3, 1},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 24, 0},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 12, 0},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 9, 0},
		{[]byte{0x00, 0x00, 0x00, 0x00, 0x00}, 3, 0},
	}

	for _, test := range tests {
		res := getBitsFromPacket(test.packet, &bytePos, &bitPos, test.bpP)
		if res != test.ret {
			t.Errorf("Input: %d Expected: %d \t Got %d", test.packet, test.ret, res)
		}
	}
}
<|endoftext|>


<commit_before>package gocd

import (
	"context"
	"fmt"
	"github.com/pkg/errors"
)

// PipelinesService describes the HAL _link resource for the api response object for a pipelineconfig
type PipelinesService service

// PipelineRequest describes a pipeline request object
type PipelineRequest struct {
	Group    string    `json:"group"`
	Pipeline *Pipeline `json:"pipeline"`
}

// Pipeline describes a pipeline object
type Pipeline struct {
	Name                  string     `json:"name"`
	LabelTemplate         string     `json:"label_template,omitempty"`
	EnablePipelineLocking bool       `json:"enable_pipeline_locking,omitempty"`
	Template              string     `json:"template,omitempty"`
	Materials             []Material `json:"materials,omitempty"`
	Label                 string     `json:"label,omitempty"`
	Stages                []*Stage    `json:"stages"`
	Version               string     `json:"version,omitempty"`
}

// Material describes an artifact dependency for a pipeline object.
type Material struct {
	Type        string             `json:"type"`
	Fingerprint string             `json:"fingerprint,omitempty"`
	Description string             `json:"description,omitempty"`
	Attributes  MaterialAttributes `json:"attributes"`
}

// MaterialAttributes describes a material type
type MaterialAttributes struct {
	URL             string          `json:"url"`
	Destination     string          `json:"destination,omitempty"`
	Filter          *MaterialFilter `json:"filter,omitempty"`
	InvertFilter    bool            `json:"invert_filter"`
	Name            string          `json:"name,omitempty"`
	AutoUpdate      bool            `json:"auto_update,omitempty"`
	Branch          string          `json:"branch,omitempty"`
	SubmoduleFolder string          `json:"submodule_folder,omitempty"`
	ShallowClone    bool            `json:"shallow_clone,omitempty"`
}

// MaterialFilter describes which globs to ignore
type MaterialFilter struct {
	Ignore []string `json:"ignore"`
}

// PipelineHistory describes the history of runs for a pipeline
type PipelineHistory struct {
	Pipelines []*PipelineInstance `json:"pipelines"`
}

// PipelineInstance describes a single pipeline run
type PipelineInstance struct {
	BuildCause   BuildCause `json:"build_cause"`
	CanRun       bool       `json:"can_run"`
	Name         string     `json:"name"`
	NaturalOrder int        `json:"natural_order"`
	Comment      string     `json:"comment"`
	Stages       []*Stage    `json:"stages"`
}

// BuildCause describes the triggers which caused the build to start.
type BuildCause struct {
	Approver          string             `json:"approver,omitempty"`
	MaterialRevisions []MaterialRevision `json:"material_revisions"`
	TriggerForced     bool               `json:"trigger_forced"`
	TriggerMessage    string             `json:"trigger_message"`
}

// MaterialRevision describes the uniquely identifiable version for the material which was pulled for this build
type MaterialRevision struct {
	Modifications []Modification `json:"modifications"`
	Material      struct {
		Description string `json:"description"`
		Fingerprint string `json:"fingerprint"`
		Type        string `json:"type"`
		ID          int    `json:"id"`
	} `json:"material"`
	Changed bool `json:"changed"`
}

// Modification describes the commit/revision for the material which kicked off the build.
type Modification struct {
	EmailAddress string `json:"email_address"`
	ID           int    `json:"id"`
	ModifiedTime int    `json:"modified_time"`
	UserName     string `json:"user_name"`
	Comment      string `json:"comment"`
	Revision     string `json:"revision"`
}

// PipelineStatus describes whether a pipeline can be run or scheduled.
type PipelineStatus struct {
	Locked      bool `json:"locked"`
	Paused      bool `json:"paused"`
	Schedulable bool `json:"schedulable"`
}

// GetStatus returns a list of pipeline instanves describing the pipeline history.
func (pgs *PipelinesService) GetStatus(ctx context.Context, name string, offset int) (*PipelineStatus, *APIResponse, error) {
	ps := PipelineStatus{}
	_, resp, err := pgs.client.getAction(ctx, &APIClientRequest{
		Path:         fmt.Sprintf("pipelines/%s/status", name),
		ResponseBody: &ps,
	})

	return &ps, resp, err
}

// Pause allows a pipeline to handle new build events
func (pgs *PipelinesService) Pause(ctx context.Context, name string) (bool, *APIResponse, error) {
	return pgs.pipelineAction(ctx, name, "pause")
}

// Unpause allows a pipeline to handle new build events
func (pgs *PipelinesService) Unpause(ctx context.Context, name string) (bool, *APIResponse, error) {
	return pgs.pipelineAction(ctx, name, "unpause")
}

// ReleaseLock frees a pipeline to handle new build events
func (pgs *PipelinesService) ReleaseLock(ctx context.Context, name string) (bool, *APIResponse, error) {
	return pgs.pipelineAction(ctx, name, "releaseLock")
}

// Create a pipeline
func (pgs *PipelinesService) Create(ctx context.Context, p *Pipeline, group string) (*Pipeline, *APIResponse, error) {
	pt := Pipeline{}
	_, resp, err := pgs.client.postAction(ctx, &APIClientRequest{
		Path:       "admin/pipelines",
		APIVersion: apiV4,
		RequestBody: PipelineRequest{
			Group:    group,
			Pipeline: p,
		},
		ResponseBody: &pt,
	})

	return &pt, resp, err
}

// Get returns a list of pipeline instanves describing the pipeline history.
func (pgs *PipelinesService) GetInstance(ctx context.Context, name string, offset int) (*PipelineInstance, *APIResponse, error) {
	stub := pgs.buildPaginatedStub("admin/pipelines/%s/instance", name, offset)

	pt := PipelineInstance{}
	_, resp, err := pgs.client.getAction(ctx, &APIClientRequest{
		Path:         stub,
		ResponseBody: &pt,
	})

	return &pt, resp, err
}

// GetHistory returns a list of pipeline instances describing the pipeline history.
func (pgs *PipelinesService) GetHistory(ctx context.Context, name string, offset int) (*PipelineHistory, *APIResponse, error) {
	stub := pgs.buildPaginatedStub("pipelines/%s/history", name, offset)

	pt := PipelineHistory{}
	_, resp, err := pgs.client.getAction(ctx, &APIClientRequest{
		Path:         stub,
		ResponseBody: &pt,
	})

	return &pt, resp, err
}

func (pgs *PipelinesService) pipelineAction(ctx context.Context, name string, action string) (bool, *APIResponse, error) {

	_, resp, err := pgs.client.postAction(ctx, &APIClientRequest{
		Path:         fmt.Sprintf("pipelines/%s/%s", name, action),
		ResponseType: responseTypeJSON,
		Headers: map[string]string{
			"Confirm": "true",
		},
	})

	return resp.HTTP.StatusCode == 200, resp, err
}

func (pgs *PipelinesService) buildPaginatedStub(format string, name string, offset int) string {
	stub := fmt.Sprintf(format, name)
	if offset > 0 {
		stub = fmt.Sprintf("%s/%d", stub, offset)
	}
	return stub
}
<commit_msg>Removed unused import<commit_after>package gocd

import (
	"context"
	"fmt"
)

// PipelinesService describes the HAL _link resource for the api response object for a pipelineconfig
type PipelinesService service

// PipelineRequest describes a pipeline request object
type PipelineRequest struct {
	Group    string    `json:"group"`
	Pipeline *Pipeline `json:"pipeline"`
}

// Pipeline describes a pipeline object
type Pipeline struct {
	Name                  string     `json:"name"`
	LabelTemplate         string     `json:"label_template,omitempty"`
	EnablePipelineLocking bool       `json:"enable_pipeline_locking,omitempty"`
	Template              string     `json:"template,omitempty"`
	Materials             []Material `json:"materials,omitempty"`
	Label                 string     `json:"label,omitempty"`
	Stages                []*Stage    `json:"stages"`
	Version               string     `json:"version,omitempty"`
}

// Material describes an artifact dependency for a pipeline object.
type Material struct {
	Type        string             `json:"type"`
	Fingerprint string             `json:"fingerprint,omitempty"`
	Description string             `json:"description,omitempty"`
	Attributes  MaterialAttributes `json:"attributes"`
}

// MaterialAttributes describes a material type
type MaterialAttributes struct {
	URL             string          `json:"url"`
	Destination     string          `json:"destination,omitempty"`
	Filter          *MaterialFilter `json:"filter,omitempty"`
	InvertFilter    bool            `json:"invert_filter"`
	Name            string          `json:"name,omitempty"`
	AutoUpdate      bool            `json:"auto_update,omitempty"`
	Branch          string          `json:"branch,omitempty"`
	SubmoduleFolder string          `json:"submodule_folder,omitempty"`
	ShallowClone    bool            `json:"shallow_clone,omitempty"`
}

// MaterialFilter describes which globs to ignore
type MaterialFilter struct {
	Ignore []string `json:"ignore"`
}

// PipelineHistory describes the history of runs for a pipeline
type PipelineHistory struct {
	Pipelines []*PipelineInstance `json:"pipelines"`
}

// PipelineInstance describes a single pipeline run
type PipelineInstance struct {
	BuildCause   BuildCause `json:"build_cause"`
	CanRun       bool       `json:"can_run"`
	Name         string     `json:"name"`
	NaturalOrder int        `json:"natural_order"`
	Comment      string     `json:"comment"`
	Stages       []*Stage    `json:"stages"`
}

// BuildCause describes the triggers which caused the build to start.
type BuildCause struct {
	Approver          string             `json:"approver,omitempty"`
	MaterialRevisions []MaterialRevision `json:"material_revisions"`
	TriggerForced     bool               `json:"trigger_forced"`
	TriggerMessage    string             `json:"trigger_message"`
}

// MaterialRevision describes the uniquely identifiable version for the material which was pulled for this build
type MaterialRevision struct {
	Modifications []Modification `json:"modifications"`
	Material      struct {
		Description string `json:"description"`
		Fingerprint string `json:"fingerprint"`
		Type        string `json:"type"`
		ID          int    `json:"id"`
	} `json:"material"`
	Changed bool `json:"changed"`
}

// Modification describes the commit/revision for the material which kicked off the build.
type Modification struct {
	EmailAddress string `json:"email_address"`
	ID           int    `json:"id"`
	ModifiedTime int    `json:"modified_time"`
	UserName     string `json:"user_name"`
	Comment      string `json:"comment"`
	Revision     string `json:"revision"`
}

// PipelineStatus describes whether a pipeline can be run or scheduled.
type PipelineStatus struct {
	Locked      bool `json:"locked"`
	Paused      bool `json:"paused"`
	Schedulable bool `json:"schedulable"`
}

// GetStatus returns a list of pipeline instanves describing the pipeline history.
func (pgs *PipelinesService) GetStatus(ctx context.Context, name string, offset int) (*PipelineStatus, *APIResponse, error) {
	ps := PipelineStatus{}
	_, resp, err := pgs.client.getAction(ctx, &APIClientRequest{
		Path:         fmt.Sprintf("pipelines/%s/status", name),
		ResponseBody: &ps,
	})

	return &ps, resp, err
}

// Pause allows a pipeline to handle new build events
func (pgs *PipelinesService) Pause(ctx context.Context, name string) (bool, *APIResponse, error) {
	return pgs.pipelineAction(ctx, name, "pause")
}

// Unpause allows a pipeline to handle new build events
func (pgs *PipelinesService) Unpause(ctx context.Context, name string) (bool, *APIResponse, error) {
	return pgs.pipelineAction(ctx, name, "unpause")
}

// ReleaseLock frees a pipeline to handle new build events
func (pgs *PipelinesService) ReleaseLock(ctx context.Context, name string) (bool, *APIResponse, error) {
	return pgs.pipelineAction(ctx, name, "releaseLock")
}

// Create a pipeline
func (pgs *PipelinesService) Create(ctx context.Context, p *Pipeline, group string) (*Pipeline, *APIResponse, error) {
	pt := Pipeline{}
	_, resp, err := pgs.client.postAction(ctx, &APIClientRequest{
		Path:       "admin/pipelines",
		APIVersion: apiV4,
		RequestBody: PipelineRequest{
			Group:    group,
			Pipeline: p,
		},
		ResponseBody: &pt,
	})

	return &pt, resp, err
}

// Get returns a list of pipeline instanves describing the pipeline history.
func (pgs *PipelinesService) GetInstance(ctx context.Context, name string, offset int) (*PipelineInstance, *APIResponse, error) {
	stub := pgs.buildPaginatedStub("admin/pipelines/%s/instance", name, offset)

	pt := PipelineInstance{}
	_, resp, err := pgs.client.getAction(ctx, &APIClientRequest{
		Path:         stub,
		ResponseBody: &pt,
	})

	return &pt, resp, err
}

// GetHistory returns a list of pipeline instances describing the pipeline history.
func (pgs *PipelinesService) GetHistory(ctx context.Context, name string, offset int) (*PipelineHistory, *APIResponse, error) {
	stub := pgs.buildPaginatedStub("pipelines/%s/history", name, offset)

	pt := PipelineHistory{}
	_, resp, err := pgs.client.getAction(ctx, &APIClientRequest{
		Path:         stub,
		ResponseBody: &pt,
	})

	return &pt, resp, err
}

func (pgs *PipelinesService) pipelineAction(ctx context.Context, name string, action string) (bool, *APIResponse, error) {

	_, resp, err := pgs.client.postAction(ctx, &APIClientRequest{
		Path:         fmt.Sprintf("pipelines/%s/%s", name, action),
		ResponseType: responseTypeJSON,
		Headers: map[string]string{
			"Confirm": "true",
		},
	})

	return resp.HTTP.StatusCode == 200, resp, err
}

func (pgs *PipelinesService) buildPaginatedStub(format string, name string, offset int) string {
	stub := fmt.Sprintf(format, name)
	if offset > 0 {
		stub = fmt.Sprintf("%s/%d", stub, offset)
	}
	return stub
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// +build appengine

package dl

// TODO(adg): refactor this to use the tools/godoc/static template.

const templateHTML = `
{{define "root"}}
<!DOCTYPE html>

        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
        <title>Downloads - The Go Programming Language</title>
        <link type="text/css" rel="stylesheet" href="/lib/godoc/style.css">

The Go Programming Language

<form method="GET" action="/search">

Documents
Packages
The Project
Help
Blog
<input type="text" id="search" name="q" class="inactive" value="Search" placeholder="Search">

</form>

Downloads

After downloading a binary release suitable for your system,
please follow the installation instructions.

If you are building from source,
follow the source installation instructions.

See the release history for more
information about Go releases.

{{with .Featured}}

Featured downloads

{{range .}}
{{template "download" .}}
{{end}}
{{end}}

{{with .Stable}}

Stable versions

{{template "releases" .}}
{{end}}

{{with .Unstable}}

Unstable version

{{template "releases" .}}
{{end}}

Older versions

Older releases of Go are available at Google Code.

        Except as
        noted,
        the content of this page is licensed under the Creative Commons
        Attribution 3.0 License,

        and code is licensed under a BSD license.

        Terms of Service |
        Privacy Policy

{{end}}

{{define "releases"}}
{{range .}}

{{.Version}} ▹

{{.Version}} ▾

		{{if .Stable}}{{else}}

This is an unstable version of Go. Use with caution.

		{{end}}
		{{template "files" .Files}}

{{end}}
{{end}}

{{define "files"}}

  File name
  Kind
  OS
  Arch
  Size
  {{/* Use the checksum type of the first file for the column heading. */}}
  {{(index . 0).ChecksumType}} Checksum

{{range .}}

  {{.Filename}}
  {{pretty .Kind}}
  {{.PrettyOS}}
  {{pretty .Arch}}
  {{.PrettySize}}
  <tt>{{.PrettyChecksum}}</tt>

{{else}}

  No downloads available.

{{end}}

{{end}}

{{define "download"}}

{{.Platform}}

{{with .Requirements}}
{{.}}
{{end}}

  {{.Filename}}
  {{if .Size}}({{.PrettySize}}){{end}}

{{.ChecksumType}}: {{.PrettyChecksum}}

{{end}}
`
<commit_msg>godoc/dl: remove broken link to old releases<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// +build appengine

package dl

// TODO(adg): refactor this to use the tools/godoc/static template.

const templateHTML = `
{{define "root"}}
<!DOCTYPE html>

        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
        <title>Downloads - The Go Programming Language</title>
        <link type="text/css" rel="stylesheet" href="/lib/godoc/style.css">

The Go Programming Language

<form method="GET" action="/search">

Documents
Packages
The Project
Help
Blog
<input type="text" id="search" name="q" class="inactive" value="Search" placeholder="Search">

</form>

Downloads

After downloading a binary release suitable for your system,
please follow the installation instructions.

If you are building from source,
follow the source installation instructions.

See the release history for more
information about Go releases.

{{with .Featured}}

Featured downloads

{{range .}}
{{template "download" .}}
{{end}}
{{end}}

{{with .Stable}}

Stable versions

{{template "releases" .}}
{{end}}

{{with .Unstable}}

Unstable version

{{template "releases" .}}
{{end}}

        Except as
        noted,
        the content of this page is licensed under the Creative Commons
        Attribution 3.0 License,

        and code is licensed under a BSD license.

        Terms of Service |
        Privacy Policy

{{end}}

{{define "releases"}}
{{range .}}

{{.Version}} ▹

{{.Version}} ▾

		{{if .Stable}}{{else}}

This is an unstable version of Go. Use with caution.

		{{end}}
		{{template "files" .Files}}

{{end}}
{{end}}

{{define "files"}}

  File name
  Kind
  OS
  Arch
  Size
  {{/* Use the checksum type of the first file for the column heading. */}}
  {{(index . 0).ChecksumType}} Checksum

{{range .}}

  {{.Filename}}
  {{pretty .Kind}}
  {{.PrettyOS}}
  {{pretty .Arch}}
  {{.PrettySize}}
  <tt>{{.PrettyChecksum}}</tt>

{{else}}

  No downloads available.

{{end}}

{{end}}

{{define "download"}}

{{.Platform}}

{{with .Requirements}}
{{.}}
{{end}}

  {{.Filename}}
  {{if .Size}}({{.PrettySize}}){{end}}

{{.ChecksumType}}: {{.PrettyChecksum}}

{{end}}
`
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"log"
	"net/http"
)

func init() {
	http.HandleFunc("/compile", Compile)
}

type Response struct {
	Output string `json:"output"`
	Errors string `json:"compile_errors"`
}

func Compile(w http.ResponseWriter, req *http.Request) {
	resp := new(Response)
	out, err := compile(req)
	if err != nil {
		if out != nil {
			resp.Errors = string(out)
		} else {
			resp.Errors = err.Error()
		}
	} else {
		resp.Output = string(out)
	}
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		log.Println(err)
	}
}
<commit_msg>go-tour: show "go.exe is not found" message<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"log"
	"net/http"
)

func init() {
	http.HandleFunc("/compile", Compile)
}

type Response struct {
	Output string `json:"output"`
	Errors string `json:"compile_errors"`
}

func Compile(w http.ResponseWriter, req *http.Request) {
	resp := new(Response)
	out, err := compile(req)
	if err != nil {
		if len(out) > 0 {
			resp.Errors = string(out) + "\n" + err.Error()
		} else {
			resp.Errors = err.Error()
		}
	} else {
		resp.Output = string(out)
	}
	if err := json.NewEncoder(w).Encode(resp); err != nil {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"text/template"

	"github.com/cgansen/elastigo/api"
	"github.com/cgansen/health-near-me/healthnearme"
	geo "github.com/kellydunn/golang-geo"
)

var tmplPath string

func init() {
	flag.StringVar(&tmplPath, "tmpl", "../tmpl/", "path to templates")
	flag.Parse()
}

// Perform a search for a SMS user.
func SMSSearchHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("%s %s %s %s", req.Method, req.RequestURI, req.URL.RawQuery, req.Header.Get("User-Agent"))

	if err := req.ParseForm(); err != nil {
		log.Printf("error parsing form: %s", err)
		http.Error(w, "error parsing form body", 500)
		return
	}

	// TODO(cgansen):
	// support sessions

	search := req.FormValue("Body")
	log.Printf("sms search: %s", search)

	cmd := strings.TrimSpace(strings.ToLower(search))
	switch cmd {
	case "list", "list services":
		t, err := template.ParseFiles(tmplPath + "help.txt")
		if err != nil {
			log.Printf("error loading template: %s", err)
			http.Error(w, "error loading template", 500)
			return
		}

		if err := t.Execute(w, nil); err != nil {
			log.Printf("error executing template: %s", err)
			http.Error(w, "error executing template", 500)
			return
		}

		return
	default:
		// split query
		pieces := strings.Split(cmd, "near")

		term := strings.TrimSpace(pieces[0])
		location := strings.TrimSpace(pieces[1])

		// geocode
		geocoder := &geo.GoogleGeocoder{}
		point, err := geocoder.Geocode(pieces[1])
		if err != nil {
			// handle
			log.Printf("error geocoding: %s, location is: %s", err, location)
			http.Error(w, "error geocoding", 500)
			return
		}

		log.Printf("geocoded %s to %#v", location, point)

		// map term to searchType
		searchType, err := healthnearme.SearchType(term)
		if err != nil {
			// couldn't map it, so send a message asking user to retry

			t, err := template.New("problem.txt").ParseFiles(tmplPath + "problem.txt")
			if err != nil {
				log.Print("template error: ", err)
				http.Error(w, "error loading template", 500)
				return
			}

			ctxt := map[string]string{
				"Term": term,
			}

			if err := t.Execute(w, ctxt); err != nil {
				log.Print(err)
				http.Error(w, "error writing results", 500)
				return
			}

			w.Header().Add("Content-type", "text/xml")
			return
		}

		// lookup
		result, err := healthnearme.DoSearch(point.Lat(), point.Lng(), 1609, strconv.Itoa(int(searchType)))

		// respond
		hits, err := healthnearme.LoadResults(result, point)
		if err != nil {
			log.Print(err)
			http.Error(w, "error processing search results", 500)
			return
		}
		log.Printf("%d results for %s", len(hits), cmd)

		t, err := template.New("nearby_providers.txt").Funcs(template.FuncMap{"round": strconv.FormatFloat}).ParseFiles(tmplPath + "nearby_providers.txt")
		if err != nil {
			log.Print("template error: ", err)
			http.Error(w, "error loading template", 500)
			return
		}

		ctxt := map[string]interface{}{
			"Count":    len(hits),
			"Location": location,
			"Results":  hits,
		}

		if err := t.Execute(w, ctxt); err != nil {
			log.Print(err)
			http.Error(w, "error writing results", 500)
			return
		}

		w.Header().Add("Content-type", "text/xml")
		return
	}

	return
}

func SearchHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("%s %s %s %s %s", req.Method, req.RequestURI, req.RemoteAddr, req.Header.Get("X-Real-IP"), req.Header.Get("User-Agent"))

	slat, slon, sdist, styp := req.FormValue("lat"), req.FormValue("lon"), req.FormValue("dist"), req.FormValue("searchType")

	lat, err := strconv.ParseFloat(slat, 64)
	if err != nil {
		http.Error(w, "lat is required and must be a float, e.g. 41.42", 400)
		return
	}

	lon, err := strconv.ParseFloat(slon, 64)
	if err != nil {
		http.Error(w, "lon is required and must be a float, e.g. -87.88", 400)
		return
	}

	dist, err := strconv.ParseInt(sdist, 10, 64)
	if err != nil {
		http.Error(w, "dist is required and must be an integer", 400)
		return
	}

	if styp == "" {
		http.Error(w, "searchType is required and must be an integer or 'all'", 400)
		return
	}

	log.Printf("http search: %f,%f %d %s", lat, lon, dist, styp)

	result, err := healthnearme.DoSearch(lat, lon, dist, styp)
	if err != nil {
		log.Printf("error searching: %s", err)
		http.Error(w, "error searching index", 503)
		return
	}

	hits, err := healthnearme.LoadResults(result, geo.NewPoint(lat, lon))
	if err != nil {
		log.Print(err)
		http.Error(w, "error processing search results", 500)
	}

	jsn, err := json.MarshalIndent(hits, "", "  ")
	if err != nil {
		log.Print(err)
		http.Error(w, "error dumping search results to json", 500)
		return
	}

	w.Header().Add("Content-type", "application/json")
	// delim := ")]}',\n"
	delim := ""
	resp := fmt.Sprintf("%s%s(%s);", delim, req.FormValue("callback"), string(jsn))

	_, err = w.Write([]byte(resp))
	return

}

func HealthCheckHandler(w http.ResponseWriter, req *http.Request) {
	w.Write([]byte("OK"))
	return
}

func main() {
	api.Domain = "localhost"

	http.HandleFunc("/sms_search", SMSSearchHandler)
	http.HandleFunc("/search", SearchHandler)
	http.HandleFunc("/healthcheck", HealthCheckHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))
}
<commit_msg>add help as a sms keyword<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"text/template"

	"github.com/cgansen/elastigo/api"
	"github.com/cgansen/health-near-me/healthnearme"
	geo "github.com/kellydunn/golang-geo"
)

var tmplPath string

func init() {
	flag.StringVar(&tmplPath, "tmpl", "../tmpl/", "path to templates")
	flag.Parse()
}

// Perform a search for a SMS user.
func SMSSearchHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("%s %s %s %s", req.Method, req.RequestURI, req.URL.RawQuery, req.Header.Get("User-Agent"))

	if err := req.ParseForm(); err != nil {
		log.Printf("error parsing form: %s", err)
		http.Error(w, "error parsing form body", 500)
		return
	}

	// TODO(cgansen):
	// support sessions

	search := req.FormValue("Body")
	log.Printf("sms search: %s", search)

	cmd := strings.TrimSpace(strings.ToLower(search))
	switch cmd {
	case "help", "list", "list services":
		t, err := template.ParseFiles(tmplPath + "help.txt")
		if err != nil {
			log.Printf("error loading template: %s", err)
			http.Error(w, "error loading template", 500)
			return
		}

		if err := t.Execute(w, nil); err != nil {
			log.Printf("error executing template: %s", err)
			http.Error(w, "error executing template", 500)
			return
		}

		return
	default:
		// split query
		pieces := strings.Split(cmd, "near")

		term := strings.TrimSpace(pieces[0])
		location := strings.TrimSpace(pieces[1])

		// geocode
		geocoder := &geo.GoogleGeocoder{}
		point, err := geocoder.Geocode(pieces[1])
		if err != nil {
			// handle
			log.Printf("error geocoding: %s, location is: %s", err, location)
			http.Error(w, "error geocoding", 500)
			return
		}

		log.Printf("geocoded %s to %#v", location, point)

		// map term to searchType
		searchType, err := healthnearme.SearchType(term)
		if err != nil {
			// couldn't map it, so send a message asking user to retry

			t, err := template.New("problem.txt").ParseFiles(tmplPath + "problem.txt")
			if err != nil {
				log.Print("template error: ", err)
				http.Error(w, "error loading template", 500)
				return
			}

			ctxt := map[string]string{
				"Term": term,
			}

			if err := t.Execute(w, ctxt); err != nil {
				log.Print(err)
				http.Error(w, "error writing results", 500)
				return
			}

			w.Header().Add("Content-type", "text/xml")
			return
		}

		// lookup
		result, err := healthnearme.DoSearch(point.Lat(), point.Lng(), 1609, strconv.Itoa(int(searchType)))

		// respond
		hits, err := healthnearme.LoadResults(result, point)
		if err != nil {
			log.Print(err)
			http.Error(w, "error processing search results", 500)
			return
		}
		log.Printf("%d results for %s", len(hits), cmd)

		t, err := template.New("nearby_providers.txt").Funcs(template.FuncMap{"round": strconv.FormatFloat}).ParseFiles(tmplPath + "nearby_providers.txt")
		if err != nil {
			log.Print("template error: ", err)
			http.Error(w, "error loading template", 500)
			return
		}

		ctxt := map[string]interface{}{
			"Count":    len(hits),
			"Location": location,
			"Results":  hits,
		}

		if err := t.Execute(w, ctxt); err != nil {
			log.Print(err)
			http.Error(w, "error writing results", 500)
			return
		}

		w.Header().Add("Content-type", "text/xml")
		return
	}

	return
}

func SearchHandler(w http.ResponseWriter, req *http.Request) {
	log.Printf("%s %s %s %s %s", req.Method, req.RequestURI, req.RemoteAddr, req.Header.Get("X-Real-IP"), req.Header.Get("User-Agent"))

	slat, slon, sdist, styp := req.FormValue("lat"), req.FormValue("lon"), req.FormValue("dist"), req.FormValue("searchType")

	lat, err := strconv.ParseFloat(slat, 64)
	if err != nil {
		http.Error(w, "lat is required and must be a float, e.g. 41.42", 400)
		return
	}

	lon, err := strconv.ParseFloat(slon, 64)
	if err != nil {
		http.Error(w, "lon is required and must be a float, e.g. -87.88", 400)
		return
	}

	dist, err := strconv.ParseInt(sdist, 10, 64)
	if err != nil {
		http.Error(w, "dist is required and must be an integer", 400)
		return
	}

	if styp == "" {
		http.Error(w, "searchType is required and must be an integer or 'all'", 400)
		return
	}

	log.Printf("http search: %f,%f %d %s", lat, lon, dist, styp)

	result, err := healthnearme.DoSearch(lat, lon, dist, styp)
	if err != nil {
		log.Printf("error searching: %s", err)
		http.Error(w, "error searching index", 503)
		return
	}

	hits, err := healthnearme.LoadResults(result, geo.NewPoint(lat, lon))
	if err != nil {
		log.Print(err)
		http.Error(w, "error processing search results", 500)
	}

	jsn, err := json.MarshalIndent(hits, "", "  ")
	if err != nil {
		log.Print(err)
		http.Error(w, "error dumping search results to json", 500)
		return
	}

	w.Header().Add("Content-type", "application/json")
	// delim := ")]}',\n"
	delim := ""
	resp := fmt.Sprintf("%s%s(%s);", delim, req.FormValue("callback"), string(jsn))

	_, err = w.Write([]byte(resp))
	return

}

func HealthCheckHandler(w http.ResponseWriter, req *http.Request) {
	w.Write([]byte("OK"))
	return
}

func main() {
	api.Domain = "localhost"

	http.HandleFunc("/sms_search", SMSSearchHandler)
	http.HandleFunc("/search", SearchHandler)
	http.HandleFunc("/healthcheck", HealthCheckHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !copybara

// Package main is a client to a Swarming server.
//
// The reference server python implementation documentation can be found at
// https://github.com/luci/luci-py/tree/master/appengine/swarming/doc
package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"os"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/auth/client/authcli"
	"go.chromium.org/luci/client/casclient"
	"go.chromium.org/luci/client/cmd/swarming/lib"
	"go.chromium.org/luci/client/versioncli"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

type authFlags struct {
	flags       authcli.Flags
	defaultOpts auth.Options
	parsedOpts  *auth.Options
}

func (af *authFlags) Register(f *flag.FlagSet) {
	af.flags.Register(f, af.defaultOpts)
}

func (af *authFlags) Parse() error {
	opts, err := af.flags.Options()
	if err != nil {
		return err
	}
	af.parsedOpts = &opts
	return nil
}

func (af *authFlags) NewHTTPClient(ctx context.Context) (*http.Client, error) {
	if af.parsedOpts == nil {
		return nil, errors.Reason("AuthFlags.Parse() must be called").Err()
	}
	return auth.NewAuthenticator(ctx, auth.OptionalLogin, *af.parsedOpts).Client()
}

func (af *authFlags) NewRBEClient(ctx context.Context, addr string, instance string) (*client.Client, error) {
	if af.parsedOpts == nil {
		return nil, errors.Reason("AuthFlags.Parse() must be called").Err()
	}
	return casclient.NewLegacy(ctx, addr, instance, *af.parsedOpts, true)
}

func getApplication() *subcommands.DefaultApplication {
	authOpts := chromeinfra.DefaultAuthOptions()
	af := &authFlags{defaultOpts: authOpts}

	return &subcommands.DefaultApplication{
		Name:  "swarming",
		Title: "Client tool to access a swarming server.",
		// Keep in alphabetical order of their name.
		Commands: []*subcommands.Command{
			lib.CmdBots(af),
			lib.CmdCancelTask(af),
			lib.CmdCollect(af),
			lib.CmdDeleteBots(af),
			lib.CmdReproduce(af),
			lib.CmdRequestShow(af),
			lib.CmdSpawnTasks(af),
			lib.CmdTasks(af),
			lib.CmdTerminateBot(af),
			lib.CmdTrigger(af),
			subcommands.CmdHelp,
			authcli.SubcommandInfo(authOpts, "whoami", false),
			authcli.SubcommandLogin(authOpts, "login", false),
			authcli.SubcommandLogout(authOpts, "logout", false),
			versioncli.CmdVersion(lib.SwarmingVersion),
		},

		EnvVars: map[string]subcommands.EnvVarDefinition{
			lib.TaskIDEnvVar: {
				Advanced: true,
				ShortDesc: ("Used when processing new triggered tasks. Is used as the " +
					"parent task ID for the newly triggered tasks."),
			},
		},
	}
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	mathrand.SeedRandomly()
	os.Exit(subcommands.Run(getApplication(), nil))
}
<commit_msg>swarming: use subcommands.Section<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !copybara

// Package main is a client to a Swarming server.
//
// The reference server python implementation documentation can be found at
// https://github.com/luci/luci-py/tree/master/appengine/swarming/doc
package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"os"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/client"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/auth/client/authcli"
	"go.chromium.org/luci/client/casclient"
	"go.chromium.org/luci/client/cmd/swarming/lib"
	"go.chromium.org/luci/client/versioncli"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

type authFlags struct {
	flags       authcli.Flags
	defaultOpts auth.Options
	parsedOpts  *auth.Options
}

func (af *authFlags) Register(f *flag.FlagSet) {
	af.flags.Register(f, af.defaultOpts)
}

func (af *authFlags) Parse() error {
	opts, err := af.flags.Options()
	if err != nil {
		return err
	}
	af.parsedOpts = &opts
	return nil
}

func (af *authFlags) NewHTTPClient(ctx context.Context) (*http.Client, error) {
	if af.parsedOpts == nil {
		return nil, errors.Reason("AuthFlags.Parse() must be called").Err()
	}
	return auth.NewAuthenticator(ctx, auth.OptionalLogin, *af.parsedOpts).Client()
}

func (af *authFlags) NewRBEClient(ctx context.Context, addr string, instance string) (*client.Client, error) {
	if af.parsedOpts == nil {
		return nil, errors.Reason("AuthFlags.Parse() must be called").Err()
	}
	return casclient.NewLegacy(ctx, addr, instance, *af.parsedOpts, true)
}

func getApplication() *subcommands.DefaultApplication {
	authOpts := chromeinfra.DefaultAuthOptions()
	af := &authFlags{defaultOpts: authOpts}

	return &subcommands.DefaultApplication{
		Name:  "swarming",
		Title: "Client tool to access a swarming server.",
		// Keep in alphabetical order of their name.
		Commands: []*subcommands.Command{
			subcommands.Section("task related commands\n"),
			lib.CmdCancelTask(af),
			lib.CmdCollect(af),
			lib.CmdReproduce(af),
			lib.CmdRequestShow(af),
			lib.CmdSpawnTasks(af),
			lib.CmdTasks(af),
			lib.CmdTrigger(af),
			subcommands.Section("bot related commands\n"),
			lib.CmdBots(af),
			lib.CmdDeleteBots(af),
			lib.CmdTerminateBot(af),
			subcommands.Section("other commands\n"),
			subcommands.CmdHelp,
			authcli.SubcommandInfo(authOpts, "whoami", false),
			authcli.SubcommandLogin(authOpts, "login", false),
			authcli.SubcommandLogout(authOpts, "logout", false),
			versioncli.CmdVersion(lib.SwarmingVersion),
		},

		EnvVars: map[string]subcommands.EnvVarDefinition{
			lib.TaskIDEnvVar: {
				Advanced: true,
				ShortDesc: ("Used when processing new triggered tasks. Is used as the " +
					"parent task ID for the newly triggered tasks."),
			},
		},
	}
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	mathrand.SeedRandomly()
	os.Exit(subcommands.Run(getApplication(), nil))
}
<|endoftext|>


<commit_before>package x86_16

import (
	"fmt"
	"syscall"

	co "github.com/lunixbochs/usercorn/go/kernel/common"
	"github.com/lunixbochs/usercorn/go/models"
	uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn"
)

const (
	STACK_BASE = 0x8000
	STACK_SIZE = 0x1000
)

var dosSysNum = map[int]string{
	0x00: "terminate",
	0x01: "char_in",
	0x02: "char_out",
	0x09: "display",
	0x4C: "terminate_with_code",
}

// TODO: Create a reverse map of this for conciseness
var abiMap = map[int][]int{
	0x00: {},
	0x01: {uc.X86_REG_DX},
	0x02: {uc.X86_REG_DX}, // Actually DL
	0x09: {uc.X86_REG_DX, uc.X86_REG_DS},
	0x30: {},
	0x3C: {uc.X86_REG_DX, uc.X86_REG_DS, uc.X86_REG_AL},
	0x3D: {uc.X86_REG_DX, uc.X86_REG_DS, uc.X86_REG_AL},
	0x4C: {uc.X86_REG_AL},
}

type PSP struct {
	CPMExit                     [2]uint8
	FirstFreeSegment            uint16
	Reserved1                   uint8
	CPMCall5Compat              [5]uint8
	OldTSRAddress               uint32
	OldBreakAddress             uint32
	CriticalErrorHandlerAddress uint32
	CallerPSPSegment            uint16
	JobFileTable                [20]uint8
	EnvironmentSegment          uint16
	INT21SSSP                   uint32
	JobFileTableSize            uint16
	JobFileTablePointer         uint32
	PreviousPSP                 uint32
	Reserved2                   uint32
	DOSVersion                  uint16
	Reserved3                   [14]uint8
	DOSFarCall                  [3]uint8
	Reserved4                   uint16
	ExtendedFCB1                [7]uint8
	FCB1                        [16]uint8
	FCB2                        [20]uint8
	CommandLineLength           uint8
	CommandLine                 [127]byte
}

type DosKernel struct {
	*co.KernelBase
}

func (k *DosKernel) Terminate() {
	k.U.Exit(models.ExitStatus(0))
}

func (k *DosKernel) CharIn(buf co.Buf) {
}

func (k *DosKernel) CharOut(char uint16) {
	fmt.Printf("%c", uint8(char&0xFF))
}

func (k *DosKernel) Display(buf co.Buf) {
	// TODO: Read ahead? This'll be slow
	var i uint64
	var mem []uint8
	char := uint8(0)

	for i = 1; char != '$'; i++ {
		mem, _ = k.U.MemRead(buf.Addr, i)
		char = mem[i-1]
	}

	syscall.Write(1, mem[:i-2])
}

func (k *DosKernel) GetDosVersion() {
	k.U.RegWrite(uc.X86_REG_AX, 0x7)
}

func (k *DosKernel) TerminateWithCode(code int) {
	k.U.Exit(models.ExitStatus(code))
}

func NewKernel() *DosKernel {
	return &DosKernel{&co.KernelBase{}}
}

var regNames = []string{
	"ip", "sp", "bp", "ax", "bx", "cx", "dx",
	"si", "di", "flags", "cs", "ds", "es", "ss",
}

func DosInit(u models.Usercorn, args, env []string) error {
	u.RegWrite(u.Arch().SP, STACK_BASE)
	u.SetStackBase(STACK_BASE)
	u.SetStackSize(STACK_SIZE)
	u.SetEntry(0x100)
	return nil
}

func DosSyscall(u models.Usercorn) {
	num, _ := u.RegRead(uc.X86_REG_AH)
	name, _ := dosSysNum[int(num)]
	// TODO: How are registers numbered from here?
	u.Syscall(int(num), name, dosArgs(u, int(num)))
	// TODO: Set error
}

func dosArgs(u models.Usercorn, num int) func(n int) ([]uint64, error) {
	return co.RegArgs(u, abiMap[num])
}

func DosInterrupt(u models.Usercorn, cause uint32) {
	intno := cause & 0xFF
	if intno == 0x21 {
		DosSyscall(u)
	} else if intno == 0x20 {
		u.Syscall(0, "terminate", func(int) ([]uint64, error) { return []uint64{}, nil })
	} else {
		panic(fmt.Sprintf("unhandled X86 interrupt %d", intno))
	}
}
func DosKernels(u models.Usercorn) []interface{} {
	kernel := &DosKernel{&co.KernelBase{}}
	return []interface{}{kernel}
}

func init() {
	Arch.RegisterOS(&models.OS{
		Name:      "DOS",
		Init:      DosInit,
		Interrupt: DosInterrupt,
		Kernels:   DosKernels,
	})
}
<commit_msg>DOS: Init SP to BOTTOM of stack<commit_after>package x86_16

import (
	"fmt"
	"syscall"

	co "github.com/lunixbochs/usercorn/go/kernel/common"
	"github.com/lunixbochs/usercorn/go/models"
	uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn"
)

const (
	STACK_BASE = 0x8000
	STACK_SIZE = 0x1000
)

var dosSysNum = map[int]string{
	0x00: "terminate",
	0x01: "char_in",
	0x02: "char_out",
	0x09: "display",
	0x4C: "terminate_with_code",
}

// TODO: Create a reverse map of this for conciseness
var abiMap = map[int][]int{
	0x00: {},
	0x01: {uc.X86_REG_DX},
	0x02: {uc.X86_REG_DX}, // Actually DL
	0x09: {uc.X86_REG_DX, uc.X86_REG_DS},
	0x30: {},
	0x3C: {uc.X86_REG_DX, uc.X86_REG_DS, uc.X86_REG_AL},
	0x3D: {uc.X86_REG_DX, uc.X86_REG_DS, uc.X86_REG_AL},
	0x4C: {uc.X86_REG_AL},
}

type PSP struct {
	CPMExit                     [2]uint8
	FirstFreeSegment            uint16
	Reserved1                   uint8
	CPMCall5Compat              [5]uint8
	OldTSRAddress               uint32
	OldBreakAddress             uint32
	CriticalErrorHandlerAddress uint32
	CallerPSPSegment            uint16
	JobFileTable                [20]uint8
	EnvironmentSegment          uint16
	INT21SSSP                   uint32
	JobFileTableSize            uint16
	JobFileTablePointer         uint32
	PreviousPSP                 uint32
	Reserved2                   uint32
	DOSVersion                  uint16
	Reserved3                   [14]uint8
	DOSFarCall                  [3]uint8
	Reserved4                   uint16
	ExtendedFCB1                [7]uint8
	FCB1                        [16]uint8
	FCB2                        [20]uint8
	CommandLineLength           uint8
	CommandLine                 [127]byte
}

type DosKernel struct {
	*co.KernelBase
}

func (k *DosKernel) Terminate() {
	k.U.Exit(models.ExitStatus(0))
}

func (k *DosKernel) CharIn(buf co.Buf) {
}

func (k *DosKernel) CharOut(char uint16) {
	fmt.Printf("%c", uint8(char&0xFF))
}

func (k *DosKernel) Display(buf co.Buf) {
	// TODO: Read ahead? This'll be slow
	var i uint64
	var mem []uint8
	char := uint8(0)

	for i = 1; char != '$'; i++ {
		mem, _ = k.U.MemRead(buf.Addr, i)
		char = mem[i-1]
	}

	syscall.Write(1, mem[:i-2])
}

func (k *DosKernel) GetDosVersion() {
	k.U.RegWrite(uc.X86_REG_AX, 0x7)
}

func (k *DosKernel) TerminateWithCode(code int) {
	k.U.Exit(models.ExitStatus(code))
}

func NewKernel() *DosKernel {
	return &DosKernel{&co.KernelBase{}}
}

var regNames = []string{
	"ip", "sp", "bp", "ax", "bx", "cx", "dx",
	"si", "di", "flags", "cs", "ds", "es", "ss",
}

func DosInit(u models.Usercorn, args, env []string) error {
	u.RegWrite(u.Arch().SP, STACK_BASE+STACK_SIZE)
	u.SetStackBase(STACK_BASE)
	u.SetStackSize(STACK_SIZE)
	u.SetEntry(0x100)
	return nil
}

func DosSyscall(u models.Usercorn) {
	num, _ := u.RegRead(uc.X86_REG_AH)
	name, _ := dosSysNum[int(num)]
	// TODO: How are registers numbered from here?
	u.Syscall(int(num), name, dosArgs(u, int(num)))
	// TODO: Set error
}

func dosArgs(u models.Usercorn, num int) func(n int) ([]uint64, error) {
	return co.RegArgs(u, abiMap[num])
}

func DosInterrupt(u models.Usercorn, cause uint32) {
	intno := cause & 0xFF
	if intno == 0x21 {
		DosSyscall(u)
	} else if intno == 0x20 {
		u.Syscall(0, "terminate", func(int) ([]uint64, error) { return []uint64{}, nil })
	} else {
		panic(fmt.Sprintf("unhandled X86 interrupt %d", intno))
	}
}
func DosKernels(u models.Usercorn) []interface{} {
	kernel := &DosKernel{&co.KernelBase{}}
	return []interface{}{kernel}
}

func init() {
	Arch.RegisterOS(&models.OS{
		Name:      "DOS",
		Init:      DosInit,
		Interrupt: DosInterrupt,
		Kernels:   DosKernels,
	})
}
<|endoftext|>


<commit_before>/*
   Copyright 2016 GitHub Inc.
	 See https://github.com/github/gh-ost/blob/master/LICENSE
*/

package logic

import (
	"fmt"
	"net/http"
	"sync/atomic"
	"time"

	"github.com/github/gh-ost/go/base"
	"github.com/github/gh-ost/go/mysql"
	"github.com/github/gh-ost/go/sql"
	"github.com/outbrain/golib/log"
	"github.com/outbrain/golib/sqlutils"
)

// Throttler collects metrics related to throttling and makes informed decisison
// whether throttling should take place.
type Throttler struct {
	migrationContext *base.MigrationContext
	applier          *Applier
	inspector        *Inspector
}

func NewThrottler(applier *Applier, inspector *Inspector) *Throttler {
	return &Throttler{
		migrationContext: base.GetMigrationContext(),
		applier:          applier,
		inspector:        inspector,
	}
}

// shouldThrottle performs checks to see whether we should currently be throttling.
// It merely observes the metrics collected by other components, it does not issue
// its own metric collection.
func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint base.ThrottleReasonHint) {
	generalCheckResult := this.migrationContext.GetThrottleGeneralCheckResult()
	if generalCheckResult.ShouldThrottle {
		return generalCheckResult.ShouldThrottle, generalCheckResult.Reason, generalCheckResult.ReasonHint
	}
	// HTTP throttle
	statusCode := atomic.LoadInt64(&this.migrationContext.ThrottleHTTPStatusCode)
	if statusCode != 0 && statusCode != http.StatusOK {
		return true, fmt.Sprintf("http=%d", statusCode), base.NoThrottleReasonHint
	}
	// Replication lag throttle
	maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)
	lag := atomic.LoadInt64(&this.migrationContext.CurrentLag)
	if time.Duration(lag) > time.Duration(maxLagMillisecondsThrottleThreshold)*time.Millisecond {
		return true, fmt.Sprintf("lag=%fs", time.Duration(lag).Seconds()), base.NoThrottleReasonHint
	}
	checkThrottleControlReplicas := true
	if (this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica) && (atomic.LoadInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag) > 0) {
		checkThrottleControlReplicas = false
	}
	if checkThrottleControlReplicas {
		lagResult := this.migrationContext.GetControlReplicasLagResult()
		if lagResult.Err != nil {
			return true, fmt.Sprintf("%+v %+v", lagResult.Key, lagResult.Err), base.NoThrottleReasonHint
		}
		if lagResult.Lag > time.Duration(maxLagMillisecondsThrottleThreshold)*time.Millisecond {
			return true, fmt.Sprintf("%+v replica-lag=%fs", lagResult.Key, lagResult.Lag.Seconds()), base.NoThrottleReasonHint
		}
	}
	// Got here? No metrics indicates we need throttling.
	return false, "", base.NoThrottleReasonHint
}

// parseChangelogHeartbeat parses a string timestamp and deduces replication lag
func parseChangelogHeartbeat(heartbeatValue string) (lag time.Duration, err error) {
	heartbeatTime, err := time.Parse(time.RFC3339Nano, heartbeatValue)
	if err != nil {
		return lag, err
	}
	lag = time.Since(heartbeatTime)
	return lag, nil
}

// parseChangelogHeartbeat parses a string timestamp and deduces replication lag
func (this *Throttler) parseChangelogHeartbeat(heartbeatValue string) (err error) {
	if lag, err := parseChangelogHeartbeat(heartbeatValue); err != nil {
		return log.Errore(err)
	} else {
		atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag))
		return nil
	}
}

// collectReplicationLag reads the latest changelog heartbeat value
func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- bool) {
	collectFunc := func() error {
		if atomic.LoadInt64(&this.migrationContext.CleanupImminentFlag) > 0 {
			return nil
		}

		if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica {
			// when running on replica, the heartbeat injection is also done on the replica.
			// This means we will always get a good heartbeat value.
			// When runnign on replica, we should instead check the `SHOW SLAVE STATUS` output.
			if lag, err := mysql.GetReplicationLag(this.inspector.connectionConfig); err != nil {
				return log.Errore(err)
			} else {
				atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag))
			}
		} else {
			if heartbeatValue, err := this.inspector.readChangelogState("heartbeat"); err != nil {
				return log.Errore(err)
			} else {
				this.parseChangelogHeartbeat(heartbeatValue)
			}
		}
		return nil
	}

	collectFunc()
	firstThrottlingCollected <- true

	ticker := time.Tick(time.Duration(this.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond)
	for range ticker {
		go collectFunc()
	}
}

// collectControlReplicasLag polls all the control replicas to get maximum lag value
func (this *Throttler) collectControlReplicasLag() {

	replicationLagQuery := fmt.Sprintf(`
		select value from %s.%s where hint = 'heartbeat' and id <= 255
		`,
		sql.EscapeName(this.migrationContext.DatabaseName),
		sql.EscapeName(this.migrationContext.GetChangelogTableName()),
	)

	readReplicaLag := func(connectionConfig *mysql.ConnectionConfig) (lag time.Duration, err error) {
		dbUri := connectionConfig.GetDBUri("information_schema")

		var heartbeatValue string
		if db, _, err := sqlutils.GetDB(dbUri); err != nil {
			return lag, err
		} else if err = db.QueryRow(replicationLagQuery).Scan(&heartbeatValue); err != nil {
			return lag, err
		}
		lag, err = parseChangelogHeartbeat(heartbeatValue)
		return lag, err
	}

	readControlReplicasLag := func() (result *mysql.ReplicationLagResult) {
		instanceKeyMap := this.migrationContext.GetThrottleControlReplicaKeys()
		if instanceKeyMap.Len() == 0 {
			return result
		}
		lagResults := make(chan *mysql.ReplicationLagResult, instanceKeyMap.Len())
		for replicaKey := range *instanceKeyMap {
			connectionConfig := this.migrationContext.InspectorConnectionConfig.Duplicate()
			connectionConfig.Key = replicaKey

			lagResult := &mysql.ReplicationLagResult{Key: connectionConfig.Key}
			go func() {
				lagResult.Lag, lagResult.Err = readReplicaLag(connectionConfig)
				lagResults <- lagResult
			}()
		}
		for range *instanceKeyMap {
			lagResult := <-lagResults
			if result == nil {
				result = lagResult
			} else if lagResult.Err != nil {
				result = lagResult
			} else if lagResult.Lag.Nanoseconds() > result.Lag.Nanoseconds() {
				result = lagResult
			}
		}
		return result
	}

	checkControlReplicasLag := func() {
		if (this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica) && (atomic.LoadInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag) > 0) {
			// No need to read lag
			return
		}
		this.migrationContext.SetControlReplicasLagResult(readControlReplicasLag())
	}
	aggressiveTicker := time.Tick(100 * time.Millisecond)
	relaxedFactor := 10
	counter := 0
	shouldReadLagAggressively := false

	for range aggressiveTicker {
		if counter%relaxedFactor == 0 {
			// we only check if we wish to be aggressive once per second. The parameters for being aggressive
			// do not typically change at all throughout the migration, but nonetheless we check them.
			counter = 0
			maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)
			shouldReadLagAggressively = (maxLagMillisecondsThrottleThreshold < 1000)
		}
		if counter == 0 || shouldReadLagAggressively {
			// We check replication lag every so often, or if we wish to be aggressive
			checkControlReplicasLag()
		}
		counter++
	}
}

func (this *Throttler) criticalLoadIsMet() (met bool, variableName string, value int64, threshold int64, err error) {
	criticalLoad := this.migrationContext.GetCriticalLoad()
	for variableName, threshold = range criticalLoad {
		value, err = this.applier.ShowStatusVariable(variableName)
		if err != nil {
			return false, variableName, value, threshold, err
		}
		if value >= threshold {
			return true, variableName, value, threshold, nil
		}
	}
	return false, variableName, value, threshold, nil
}

// collectReplicationLag reads the latest changelog heartbeat value
func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- bool) {
	collectFunc := func() (sleep bool, err error) {
		url := this.migrationContext.GetThrottleHTTP()
		if url == "" {
			return true, nil
		}
		resp, err := http.Get(url)
		if err != nil {
			return false, err
		}
		atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(resp.StatusCode))
		return false, nil
	}

	collectFunc()
	firstThrottlingCollected <- true

	ticker := time.Tick(100 * time.Millisecond)
	for range ticker {
		if sleep, _ := collectFunc(); sleep {
			time.Sleep(1 * time.Second)
		}
	}
}

// collectGeneralThrottleMetrics reads the once-per-sec metrics, and stores them onto this.migrationContext
func (this *Throttler) collectGeneralThrottleMetrics() error {

	setThrottle := func(throttle bool, reason string, reasonHint base.ThrottleReasonHint) error {
		this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(throttle, reason, reasonHint))
		return nil
	}

	// Regardless of throttle, we take opportunity to check for panic-abort
	if this.migrationContext.PanicFlagFile != "" {
		if base.FileExists(this.migrationContext.PanicFlagFile) {
			this.migrationContext.PanicAbort <- fmt.Errorf("Found panic-file %s. Aborting without cleanup", this.migrationContext.PanicFlagFile)
		}
	}

	criticalLoadMet, variableName, value, threshold, err := this.criticalLoadIsMet()
	if err != nil {
		return setThrottle(true, fmt.Sprintf("%s %s", variableName, err), base.NoThrottleReasonHint)
	}
	if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds == 0 {
		this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold)
	}
	if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 {
		log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds)
		go func() {
			timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds))
			<-timer.C
			if criticalLoadMetAgain, variableName, value, threshold, _ := this.criticalLoadIsMet(); criticalLoadMetAgain {
				this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met again after %d millis: %s=%d, >=%d", this.migrationContext.CriticalLoadIntervalMilliseconds, variableName, value, threshold)
			}
		}()
	}

	// Back to throttle considerations

	// User-based throttle
	if atomic.LoadInt64(&this.migrationContext.ThrottleCommandedByUser) > 0 {
		return setThrottle(true, "commanded by user", base.UserCommandThrottleReasonHint)
	}
	if this.migrationContext.ThrottleFlagFile != "" {
		if base.FileExists(this.migrationContext.ThrottleFlagFile) {
			// Throttle file defined and exists!
			return setThrottle(true, "flag-file", base.NoThrottleReasonHint)
		}
	}
	if this.migrationContext.ThrottleAdditionalFlagFile != "" {
		if base.FileExists(this.migrationContext.ThrottleAdditionalFlagFile) {
			// 2nd Throttle file defined and exists!
			return setThrottle(true, "flag-file", base.NoThrottleReasonHint)
		}
	}

	maxLoad := this.migrationContext.GetMaxLoad()
	for variableName, threshold := range maxLoad {
		value, err := this.applier.ShowStatusVariable(variableName)
		if err != nil {
			return setThrottle(true, fmt.Sprintf("%s %s", variableName, err), base.NoThrottleReasonHint)
		}
		if value >= threshold {
			return setThrottle(true, fmt.Sprintf("max-load %s=%d >= %d", variableName, value, threshold), base.NoThrottleReasonHint)
		}
	}
	if this.migrationContext.GetThrottleQuery() != "" {
		if res, _ := this.applier.ExecuteThrottleQuery(); res > 0 {
			return setThrottle(true, "throttle-query", base.NoThrottleReasonHint)
		}
	}

	return setThrottle(false, "", base.NoThrottleReasonHint)
}

// initiateThrottlerMetrics initiates the various processes that collect measurements
// that may affect throttling. There are several components, all running independently,
// that collect such metrics.
func (this *Throttler) initiateThrottlerCollection(firstThrottlingCollected chan<- bool) {
	go this.collectReplicationLag(firstThrottlingCollected)
	go this.collectControlReplicasLag()
	go this.collectThrottleHTTPStatus(firstThrottlingCollected)

	go func() {
		this.collectGeneralThrottleMetrics()
		firstThrottlingCollected <- true

		throttlerMetricsTick := time.Tick(1 * time.Second)
		for range throttlerMetricsTick {
			this.collectGeneralThrottleMetrics()
		}
	}()
}

// initiateThrottlerChecks initiates the throttle ticker and sets the basic behavior of throttling.
func (this *Throttler) initiateThrottlerChecks() error {
	throttlerTick := time.Tick(100 * time.Millisecond)

	throttlerFunction := func() {
		alreadyThrottling, currentReason, _ := this.migrationContext.IsThrottled()
		shouldThrottle, throttleReason, throttleReasonHint := this.shouldThrottle()
		if shouldThrottle && !alreadyThrottling {
			// New throttling
			this.applier.WriteAndLogChangelog("throttle", throttleReason)
		} else if shouldThrottle && alreadyThrottling && (currentReason != throttleReason) {
			// Change of reason
			this.applier.WriteAndLogChangelog("throttle", throttleReason)
		} else if alreadyThrottling && !shouldThrottle {
			// End of throttling
			this.applier.WriteAndLogChangelog("throttle", "done throttling")
		}
		this.migrationContext.SetThrottled(shouldThrottle, throttleReason, throttleReasonHint)
	}
	throttlerFunction()
	for range throttlerTick {
		throttlerFunction()
	}

	return nil
}

// throttle sees if throttling needs take place, and if so, continuously sleeps (blocks)
// until throttling reasons are gone
func (this *Throttler) throttle(onThrottled func()) {
	for {
		// IsThrottled() is non-blocking; the throttling decision making takes place asynchronously.
		// Therefore calling IsThrottled() is cheap
		if shouldThrottle, _, _ := this.migrationContext.IsThrottled(); !shouldThrottle {
			return
		}
		if onThrottled != nil {
			onThrottled()
		}
		time.Sleep(250 * time.Millisecond)
	}
}
<commit_msg>HEAD instead of GET<commit_after>/*
   Copyright 2016 GitHub Inc.
	 See https://github.com/github/gh-ost/blob/master/LICENSE
*/

package logic

import (
	"fmt"
	"net/http"
	"sync/atomic"
	"time"

	"github.com/github/gh-ost/go/base"
	"github.com/github/gh-ost/go/mysql"
	"github.com/github/gh-ost/go/sql"
	"github.com/outbrain/golib/log"
	"github.com/outbrain/golib/sqlutils"
)

// Throttler collects metrics related to throttling and makes informed decisison
// whether throttling should take place.
type Throttler struct {
	migrationContext *base.MigrationContext
	applier          *Applier
	inspector        *Inspector
}

func NewThrottler(applier *Applier, inspector *Inspector) *Throttler {
	return &Throttler{
		migrationContext: base.GetMigrationContext(),
		applier:          applier,
		inspector:        inspector,
	}
}

// shouldThrottle performs checks to see whether we should currently be throttling.
// It merely observes the metrics collected by other components, it does not issue
// its own metric collection.
func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint base.ThrottleReasonHint) {
	generalCheckResult := this.migrationContext.GetThrottleGeneralCheckResult()
	if generalCheckResult.ShouldThrottle {
		return generalCheckResult.ShouldThrottle, generalCheckResult.Reason, generalCheckResult.ReasonHint
	}
	// HTTP throttle
	statusCode := atomic.LoadInt64(&this.migrationContext.ThrottleHTTPStatusCode)
	if statusCode != 0 && statusCode != http.StatusOK {
		return true, fmt.Sprintf("http=%d", statusCode), base.NoThrottleReasonHint
	}
	// Replication lag throttle
	maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)
	lag := atomic.LoadInt64(&this.migrationContext.CurrentLag)
	if time.Duration(lag) > time.Duration(maxLagMillisecondsThrottleThreshold)*time.Millisecond {
		return true, fmt.Sprintf("lag=%fs", time.Duration(lag).Seconds()), base.NoThrottleReasonHint
	}
	checkThrottleControlReplicas := true
	if (this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica) && (atomic.LoadInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag) > 0) {
		checkThrottleControlReplicas = false
	}
	if checkThrottleControlReplicas {
		lagResult := this.migrationContext.GetControlReplicasLagResult()
		if lagResult.Err != nil {
			return true, fmt.Sprintf("%+v %+v", lagResult.Key, lagResult.Err), base.NoThrottleReasonHint
		}
		if lagResult.Lag > time.Duration(maxLagMillisecondsThrottleThreshold)*time.Millisecond {
			return true, fmt.Sprintf("%+v replica-lag=%fs", lagResult.Key, lagResult.Lag.Seconds()), base.NoThrottleReasonHint
		}
	}
	// Got here? No metrics indicates we need throttling.
	return false, "", base.NoThrottleReasonHint
}

// parseChangelogHeartbeat parses a string timestamp and deduces replication lag
func parseChangelogHeartbeat(heartbeatValue string) (lag time.Duration, err error) {
	heartbeatTime, err := time.Parse(time.RFC3339Nano, heartbeatValue)
	if err != nil {
		return lag, err
	}
	lag = time.Since(heartbeatTime)
	return lag, nil
}

// parseChangelogHeartbeat parses a string timestamp and deduces replication lag
func (this *Throttler) parseChangelogHeartbeat(heartbeatValue string) (err error) {
	if lag, err := parseChangelogHeartbeat(heartbeatValue); err != nil {
		return log.Errore(err)
	} else {
		atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag))
		return nil
	}
}

// collectReplicationLag reads the latest changelog heartbeat value
func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- bool) {
	collectFunc := func() error {
		if atomic.LoadInt64(&this.migrationContext.CleanupImminentFlag) > 0 {
			return nil
		}

		if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica {
			// when running on replica, the heartbeat injection is also done on the replica.
			// This means we will always get a good heartbeat value.
			// When runnign on replica, we should instead check the `SHOW SLAVE STATUS` output.
			if lag, err := mysql.GetReplicationLag(this.inspector.connectionConfig); err != nil {
				return log.Errore(err)
			} else {
				atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag))
			}
		} else {
			if heartbeatValue, err := this.inspector.readChangelogState("heartbeat"); err != nil {
				return log.Errore(err)
			} else {
				this.parseChangelogHeartbeat(heartbeatValue)
			}
		}
		return nil
	}

	collectFunc()
	firstThrottlingCollected <- true

	ticker := time.Tick(time.Duration(this.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond)
	for range ticker {
		go collectFunc()
	}
}

// collectControlReplicasLag polls all the control replicas to get maximum lag value
func (this *Throttler) collectControlReplicasLag() {

	replicationLagQuery := fmt.Sprintf(`
		select value from %s.%s where hint = 'heartbeat' and id <= 255
		`,
		sql.EscapeName(this.migrationContext.DatabaseName),
		sql.EscapeName(this.migrationContext.GetChangelogTableName()),
	)

	readReplicaLag := func(connectionConfig *mysql.ConnectionConfig) (lag time.Duration, err error) {
		dbUri := connectionConfig.GetDBUri("information_schema")

		var heartbeatValue string
		if db, _, err := sqlutils.GetDB(dbUri); err != nil {
			return lag, err
		} else if err = db.QueryRow(replicationLagQuery).Scan(&heartbeatValue); err != nil {
			return lag, err
		}
		lag, err = parseChangelogHeartbeat(heartbeatValue)
		return lag, err
	}

	readControlReplicasLag := func() (result *mysql.ReplicationLagResult) {
		instanceKeyMap := this.migrationContext.GetThrottleControlReplicaKeys()
		if instanceKeyMap.Len() == 0 {
			return result
		}
		lagResults := make(chan *mysql.ReplicationLagResult, instanceKeyMap.Len())
		for replicaKey := range *instanceKeyMap {
			connectionConfig := this.migrationContext.InspectorConnectionConfig.Duplicate()
			connectionConfig.Key = replicaKey

			lagResult := &mysql.ReplicationLagResult{Key: connectionConfig.Key}
			go func() {
				lagResult.Lag, lagResult.Err = readReplicaLag(connectionConfig)
				lagResults <- lagResult
			}()
		}
		for range *instanceKeyMap {
			lagResult := <-lagResults
			if result == nil {
				result = lagResult
			} else if lagResult.Err != nil {
				result = lagResult
			} else if lagResult.Lag.Nanoseconds() > result.Lag.Nanoseconds() {
				result = lagResult
			}
		}
		return result
	}

	checkControlReplicasLag := func() {
		if (this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica) && (atomic.LoadInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag) > 0) {
			// No need to read lag
			return
		}
		this.migrationContext.SetControlReplicasLagResult(readControlReplicasLag())
	}
	aggressiveTicker := time.Tick(100 * time.Millisecond)
	relaxedFactor := 10
	counter := 0
	shouldReadLagAggressively := false

	for range aggressiveTicker {
		if counter%relaxedFactor == 0 {
			// we only check if we wish to be aggressive once per second. The parameters for being aggressive
			// do not typically change at all throughout the migration, but nonetheless we check them.
			counter = 0
			maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)
			shouldReadLagAggressively = (maxLagMillisecondsThrottleThreshold < 1000)
		}
		if counter == 0 || shouldReadLagAggressively {
			// We check replication lag every so often, or if we wish to be aggressive
			checkControlReplicasLag()
		}
		counter++
	}
}

func (this *Throttler) criticalLoadIsMet() (met bool, variableName string, value int64, threshold int64, err error) {
	criticalLoad := this.migrationContext.GetCriticalLoad()
	for variableName, threshold = range criticalLoad {
		value, err = this.applier.ShowStatusVariable(variableName)
		if err != nil {
			return false, variableName, value, threshold, err
		}
		if value >= threshold {
			return true, variableName, value, threshold, nil
		}
	}
	return false, variableName, value, threshold, nil
}

// collectReplicationLag reads the latest changelog heartbeat value
func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- bool) {
	collectFunc := func() (sleep bool, err error) {
		url := this.migrationContext.GetThrottleHTTP()
		if url == "" {
			return true, nil
		}
		resp, err := http.Head(url)
		if err != nil {
			return false, err
		}
		atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(resp.StatusCode))
		return false, nil
	}

	collectFunc()
	firstThrottlingCollected <- true

	ticker := time.Tick(100 * time.Millisecond)
	for range ticker {
		if sleep, _ := collectFunc(); sleep {
			time.Sleep(1 * time.Second)
		}
	}
}

// collectGeneralThrottleMetrics reads the once-per-sec metrics, and stores them onto this.migrationContext
func (this *Throttler) collectGeneralThrottleMetrics() error {

	setThrottle := func(throttle bool, reason string, reasonHint base.ThrottleReasonHint) error {
		this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(throttle, reason, reasonHint))
		return nil
	}

	// Regardless of throttle, we take opportunity to check for panic-abort
	if this.migrationContext.PanicFlagFile != "" {
		if base.FileExists(this.migrationContext.PanicFlagFile) {
			this.migrationContext.PanicAbort <- fmt.Errorf("Found panic-file %s. Aborting without cleanup", this.migrationContext.PanicFlagFile)
		}
	}

	criticalLoadMet, variableName, value, threshold, err := this.criticalLoadIsMet()
	if err != nil {
		return setThrottle(true, fmt.Sprintf("%s %s", variableName, err), base.NoThrottleReasonHint)
	}
	if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds == 0 {
		this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold)
	}
	if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 {
		log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds)
		go func() {
			timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds))
			<-timer.C
			if criticalLoadMetAgain, variableName, value, threshold, _ := this.criticalLoadIsMet(); criticalLoadMetAgain {
				this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met again after %d millis: %s=%d, >=%d", this.migrationContext.CriticalLoadIntervalMilliseconds, variableName, value, threshold)
			}
		}()
	}

	// Back to throttle considerations

	// User-based throttle
	if atomic.LoadInt64(&this.migrationContext.ThrottleCommandedByUser) > 0 {
		return setThrottle(true, "commanded by user", base.UserCommandThrottleReasonHint)
	}
	if this.migrationContext.ThrottleFlagFile != "" {
		if base.FileExists(this.migrationContext.ThrottleFlagFile) {
			// Throttle file defined and exists!
			return setThrottle(true, "flag-file", base.NoThrottleReasonHint)
		}
	}
	if this.migrationContext.ThrottleAdditionalFlagFile != "" {
		if base.FileExists(this.migrationContext.ThrottleAdditionalFlagFile) {
			// 2nd Throttle file defined and exists!
			return setThrottle(true, "flag-file", base.NoThrottleReasonHint)
		}
	}

	maxLoad := this.migrationContext.GetMaxLoad()
	for variableName, threshold := range maxLoad {
		value, err := this.applier.ShowStatusVariable(variableName)
		if err != nil {
			return setThrottle(true, fmt.Sprintf("%s %s", variableName, err), base.NoThrottleReasonHint)
		}
		if value >= threshold {
			return setThrottle(true, fmt.Sprintf("max-load %s=%d >= %d", variableName, value, threshold), base.NoThrottleReasonHint)
		}
	}
	if this.migrationContext.GetThrottleQuery() != "" {
		if res, _ := this.applier.ExecuteThrottleQuery(); res > 0 {
			return setThrottle(true, "throttle-query", base.NoThrottleReasonHint)
		}
	}

	return setThrottle(false, "", base.NoThrottleReasonHint)
}

// initiateThrottlerMetrics initiates the various processes that collect measurements
// that may affect throttling. There are several components, all running independently,
// that collect such metrics.
func (this *Throttler) initiateThrottlerCollection(firstThrottlingCollected chan<- bool) {
	go this.collectReplicationLag(firstThrottlingCollected)
	go this.collectControlReplicasLag()
	go this.collectThrottleHTTPStatus(firstThrottlingCollected)

	go func() {
		this.collectGeneralThrottleMetrics()
		firstThrottlingCollected <- true

		throttlerMetricsTick := time.Tick(1 * time.Second)
		for range throttlerMetricsTick {
			this.collectGeneralThrottleMetrics()
		}
	}()
}

// initiateThrottlerChecks initiates the throttle ticker and sets the basic behavior of throttling.
func (this *Throttler) initiateThrottlerChecks() error {
	throttlerTick := time.Tick(100 * time.Millisecond)

	throttlerFunction := func() {
		alreadyThrottling, currentReason, _ := this.migrationContext.IsThrottled()
		shouldThrottle, throttleReason, throttleReasonHint := this.shouldThrottle()
		if shouldThrottle && !alreadyThrottling {
			// New throttling
			this.applier.WriteAndLogChangelog("throttle", throttleReason)
		} else if shouldThrottle && alreadyThrottling && (currentReason != throttleReason) {
			// Change of reason
			this.applier.WriteAndLogChangelog("throttle", throttleReason)
		} else if alreadyThrottling && !shouldThrottle {
			// End of throttling
			this.applier.WriteAndLogChangelog("throttle", "done throttling")
		}
		this.migrationContext.SetThrottled(shouldThrottle, throttleReason, throttleReasonHint)
	}
	throttlerFunction()
	for range throttlerTick {
		throttlerFunction()
	}

	return nil
}

// throttle sees if throttling needs take place, and if so, continuously sleeps (blocks)
// until throttling reasons are gone
func (this *Throttler) throttle(onThrottled func()) {
	for {
		// IsThrottled() is non-blocking; the throttling decision making takes place asynchronously.
		// Therefore calling IsThrottled() is cheap
		if shouldThrottle, _, _ := this.migrationContext.IsThrottled(); !shouldThrottle {
			return
		}
		if onThrottled != nil {
			onThrottled()
		}
		time.Sleep(250 * time.Millisecond)
	}
}
<|endoftext|>