<commit_before>package goluago

import (
	"github.com/Shopify/go-lua"
	"github.com/Shopify/goluago/encoding/json"
	"github.com/Shopify/goluago/regexp"
	"github.com/Shopify/goluago/strings"
)

func Open(l *lua.State) {
	regexp.Open(l)
	strings.Open(l)
	json.Open(l)
}
<commit_msg>Add time to the catchall Open.<commit_after>package goluago

import (
	"github.com/Shopify/go-lua"
	"github.com/Shopify/goluago/encoding/json"
	"github.com/Shopify/goluago/regexp"
	"github.com/Shopify/goluago/strings"
	"github.com/Shopify/goluago/time"
)

func Open(l *lua.State) {
	regexp.Open(l)
	strings.Open(l)
	json.Open(l)
	time.Open(l)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"
)

// type MediaResponse {
// 	MediaURL string
// }

var (
	bucketName string
	baseURL    string
)

func init() {
	flag.StringVar(&bucketName, "b", "", "Bucket Name")
	flag.StringVar(&baseURL, "u", "", "Base URL")
}

func rootHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Nothing to see here!")
}

func tweetbot(c web.C, w http.ResponseWriter, r *http.Request) {
	r.ParseMultipartForm(r.ContentLength)

	// message := r.Form["message"]
	// source := r.Form["source"]
	file, header, err := r.FormFile("media")
	defer file.Close()

	if err != nil {
		panic(err.Error())
	}

	timeStamp := time.Now().Unix()
	random := rand.Intn(999999)
	filename := fmt.Sprintf("%x-%x-%s", timeStamp, random, header.Filename)

	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}

	// Open Bucket
	s := s3.New(auth, aws.EUWest)
	bucket := s.Bucket(bucketName)

	path := fmt.Sprintf("tweetbot/%s", filename)

	contentLength, err := strconv.Atoi(header.Header.Get("Content-Length"))
	if err != nil {
		panic(err.Error())
	}

	buffer := make([]byte, contentLength)
	cBytes, err := file.Read(buffer)

	err = bucket.Put(path, buffer[0:cBytes], header.Header.Get("Content-Type"), s3.PublicRead)
	if err != nil {
		panic(err.Error())
	}

	url := fmt.Sprintf("%s/%s", baseURL, path)

	fmt.Printf("\nFile %s (%s) uploaded successfully.\n", header.Filename, header.Header)

	responseMap := map[string]string{"url": url}
	jsonResponse, _ := json.Marshal(responseMap)
	fmt.Println(string(jsonResponse))

	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, string(jsonResponse))
}

func main() {
	goji.Get("/", rootHandler)
	re := regexp.MustCompile("/tweetbot")
	goji.Post(re, tweetbot)
	goji.Serve()
}
<commit_msg>Set the right headers<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"
)

// type MediaResponse {
// 	MediaURL string
// }

var (
	bucketName string
	baseURL    string
)

func init() {
	flag.StringVar(&bucketName, "b", "", "Bucket Name")
	flag.StringVar(&baseURL, "u", "", "Base URL")
}

func rootHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Nothing to see here!")
}

func tweetbot(c web.C, w http.ResponseWriter, r *http.Request) {
	r.ParseMultipartForm(r.ContentLength)

	// message := r.Form["message"]
	// source := r.Form["source"]
	file, header, err := r.FormFile("media")
	defer file.Close()

	if err != nil {
		panic(err.Error())
	}

	timeStamp := time.Now().Unix()
	random := rand.Intn(999999)
	filename := fmt.Sprintf("%x-%x-%s", timeStamp, random, header.Filename)

	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}

	// Open Bucket
	s := s3.New(auth, aws.EUWest)
	bucket := s.Bucket(bucketName)

	path := fmt.Sprintf("tweetbot/%s", filename)

	contentLength, err := strconv.Atoi(header.Header.Get("Content-Length"))
	if err != nil {
		panic(err.Error())
	}

	buffer := make([]byte, contentLength)
	cBytes, err := file.Read(buffer)

	s3Headers := map[string][]string{"Content-Type": {header.Header.Get("Content-Type")}, "Cache-Control": {"public, max-age=315360000"}}

	err = bucket.PutHeader(path, buffer[0:cBytes], s3Headers, s3.PublicRead)
	if err != nil {
		panic(err.Error())
	}

	url := fmt.Sprintf("%s/%s", baseURL, path)

	fmt.Printf("\nFile %s (%s) uploaded successfully.\n", header.Filename, header.Header)

	responseMap := map[string]string{"url": url}
	jsonResponse, _ := json.Marshal(responseMap)
	fmt.Println(string(jsonResponse))

	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, string(jsonResponse))
}

func main() {
	goji.Get("/", rootHandler)
	re := regexp.MustCompile("/tweetbot")
	goji.Post(re, tweetbot)
	goji.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"path/filepath"

	"github.com/samherrmann/gowrap/gotools"
)

// runGoBuildChain executes the Go build tool-chain per configuration.
// If no errors are encountered, it returns the paths to the resulting
// executables.
func runGoBuildChain(platforms *[]gotools.Platform) (*[]string, error) {
	paths := &[]string{}

	for _, p := range *platforms {
		log.Println("Building " + buildName(appName, appVersion, &p) + "...")
		err := goGenerate()
		if err != nil {
			return nil, err
		}

		path, err := goBuild(appName, appVersion, &p)
		if err != nil {
			return nil, err
		}
		*paths = append(*paths, path)
	}
	return paths, nil
}

// goGenerate executes the command "go generate"
func goGenerate() error {
	return gotools.Generate()
}

// goBuild executes the command "go build" for the desired target OS and architecture.
// If no errors are encountered, it returns the path to the resulting executable.
func goBuild(name string, version string, p *gotools.Platform) (string, error) {
	err := gotools.SetGoOS(p.GOOS)
	if err != nil {
		return "", err
	}

	err = gotools.SetGoArch(p.GOARCH)
	if err != nil {
		return "", err
	}

	path, err := buildPath(name, version, p)
	if err != nil {
		return "", err
	}

	return path, gotools.Build("-o", path, "-ldflags", "-X main.version="+version)
}

// buildPath constructs a file path for a given target
func buildPath(name string, version string, p *gotools.Platform) (string, error) {
	ext, err := gotools.ExeSuffix()
	if err != nil {
		return "", err
	}
	return filepath.Join(outputRoot, buildName(name, version, p), name+ext), err
}

// buildName returns a build-name in the form of appname-version-os-arch
// ex: myapp-v1.0.0-linux-amd64
func buildName(name string, version string, p *gotools.Platform) string {
	return name + "-" + version + "-" + p.GOOS + "-" + p.GOARCH
}
<commit_msg>Call go generate only once<commit_after>package main

import (
	"log"
	"path/filepath"

	"github.com/samherrmann/gowrap/gotools"
)

// runGoBuildChain executes the Go build tool-chain per configuration.
// If no errors are encountered, it returns the paths to the resulting
// executables.
func runGoBuildChain(platforms *[]gotools.Platform) (*[]string, error) {
	paths := &[]string{}

	log.Println("Running go generate...")
	err := goGenerate()
	if err != nil {
		return nil, err
	}

	for _, p := range *platforms {
		log.Println("Building " + buildName(appName, appVersion, &p) + "...")
		path, err := goBuild(appName, appVersion, &p)
		if err != nil {
			return nil, err
		}
		*paths = append(*paths, path)
	}
	return paths, nil
}

// goGenerate executes the command "go generate"
func goGenerate() error {
	return gotools.Generate()
}

// goBuild executes the command "go build" for the desired target OS and architecture.
// If no errors are encountered, it returns the path to the resulting executable.
func goBuild(name string, version string, p *gotools.Platform) (string, error) {
	err := gotools.SetGoOS(p.GOOS)
	if err != nil {
		return "", err
	}

	err = gotools.SetGoArch(p.GOARCH)
	if err != nil {
		return "", err
	}

	path, err := buildPath(name, version, p)
	if err != nil {
		return "", err
	}

	return path, gotools.Build("-o", path, "-ldflags", "-X main.version="+version)
}

// buildPath constructs a file path for a given target
func buildPath(name string, version string, p *gotools.Platform) (string, error) {
	ext, err := gotools.ExeSuffix()
	if err != nil {
		return "", err
	}
	return filepath.Join(outputRoot, buildName(name, version, p), name+ext), err
}

// buildName returns a build-name in the form of appname-version-os-arch
// ex: myapp-v1.0.0-linux-amd64
func buildName(name string, version string, p *gotools.Platform) string {
	return name + "-" + version + "-" + p.GOOS + "-" + p.GOARCH
}
<|endoftext|>


<commit_before>package gps

import (
	"bytes"
	"context"
	"fmt"
	"os/exec"
	"sync"
	"time"

	"github.com/Masterminds/vcs"
)

// monitoredCmd wraps a cmd and will keep monitoring the process until it
// finishes, the provided context is canceled, or a certain amount of time has
// passed and the command showed no signs of activity.
type monitoredCmd struct {
	cmd     *exec.Cmd
	timeout time.Duration
	stdout  *activityBuffer
	stderr  *activityBuffer
}

func newMonitoredCmd(cmd *exec.Cmd, timeout time.Duration) *monitoredCmd {
	stdout, stderr := newActivityBuffer(), newActivityBuffer()
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return &monitoredCmd{
		cmd:     cmd,
		timeout: timeout,
		stdout:  stdout,
		stderr:  stderr,
	}
}

// run will wait for the command to finish and return the error, if any. If the
// command does not show any activity for more than the specified timeout the
// process will be killed.
func (c *monitoredCmd) run(ctx context.Context) error {
	// Check for cancellation before even starting
	if ctx.Err() != nil {
		return ctx.Err()
	}

	ticker := time.NewTicker(c.timeout)
	done := make(chan error, 1)
	defer ticker.Stop()
	go func() { done <- c.cmd.Run() }()

	for {
		select {
		case <-ticker.C:
			if c.hasTimedOut() {
				// On windows it is apparently (?) possible for the process
				// pointer to become nil without Run() having returned (and
				// thus, passing through the done channel). Guard against this.
				if c.cmd.Process != nil {
					if err := c.cmd.Process.Kill(); err != nil {
						return &killCmdError{err}
					}
				}

				return &timeoutError{c.timeout}
			}
		case <-ctx.Done():
			if c.cmd.Process != nil {
				if err := c.cmd.Process.Kill(); err != nil {
					return &killCmdError{err}
				}
			}
			return ctx.Err()
		case err := <-done:
			return err
		}
	}
}

func (c *monitoredCmd) hasTimedOut() bool {
	t := time.Now().Add(-c.timeout)
	return c.stderr.lastActivity().Before(t) &&
		c.stdout.lastActivity().Before(t)
}

func (c *monitoredCmd) combinedOutput(ctx context.Context) ([]byte, error) {
	if err := c.run(ctx); err != nil {
		return c.stderr.buf.Bytes(), err
	}

	return c.stdout.buf.Bytes(), nil
}

// activityBuffer is a buffer that keeps track of the last time a Write
// operation was performed on it.
type activityBuffer struct {
	sync.Mutex
	buf               *bytes.Buffer
	lastActivityStamp time.Time
}

func newActivityBuffer() *activityBuffer {
	return &activityBuffer{
		buf: bytes.NewBuffer(nil),
	}
}

func (b *activityBuffer) Write(p []byte) (int, error) {
	b.Lock()
	b.lastActivityStamp = time.Now()
	defer b.Unlock()
	return b.buf.Write(p)
}

func (b *activityBuffer) lastActivity() time.Time {
	b.Lock()
	defer b.Unlock()
	return b.lastActivityStamp
}

type timeoutError struct {
	timeout time.Duration
}

func (e timeoutError) Error() string {
	return fmt.Sprintf("command killed after %s of no activity", e.timeout)
}

type killCmdError struct {
	err error
}

func (e killCmdError) Error() string {
	return fmt.Sprintf("error killing command: %s", e.err)
}

func runFromCwd(ctx context.Context, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(exec.Command(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}

func runFromRepoDir(ctx context.Context, repo vcs.Repo, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(repo.CmdFromDir(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}
<commit_msg>Removing process check and kill to prevent race<commit_after>package gps

import (
	"bytes"
	"context"
	"fmt"
	"os/exec"
	"sync"
	"time"

	"github.com/Masterminds/vcs"
)

// monitoredCmd wraps a cmd and will keep monitoring the process until it
// finishes, the provided context is canceled, or a certain amount of time has
// passed and the command showed no signs of activity.
type monitoredCmd struct {
	cmd     *exec.Cmd
	timeout time.Duration
	stdout  *activityBuffer
	stderr  *activityBuffer
}

func newMonitoredCmd(cmd *exec.Cmd, timeout time.Duration) *monitoredCmd {
	stdout, stderr := newActivityBuffer(), newActivityBuffer()
	cmd.Stdout, cmd.Stderr = stdout, stderr
	return &monitoredCmd{
		cmd:     cmd,
		timeout: timeout,
		stdout:  stdout,
		stderr:  stderr,
	}
}

// run will wait for the command to finish and return the error, if any. If the
// command does not show any activity for more than the specified timeout the
// process will be killed.
func (c *monitoredCmd) run(ctx context.Context) error {
	// Check for cancellation before even starting
	if ctx.Err() != nil {
		return ctx.Err()
	}

	ticker := time.NewTicker(c.timeout)
	done := make(chan error, 1)
	defer ticker.Stop()
	go func() { done <- c.cmd.Run() }()

	for {
		select {
		case <-ticker.C:
			if c.hasTimedOut() {
				// On windows it is apparently (?) possible for the process
				// pointer to become nil without Run() having returned (and
				// thus, passing through the done channel). Guard against this.
				if c.cmd.Process != nil {
					if err := c.cmd.Process.Kill(); err != nil {
						return &killCmdError{err}
					}
				}

				return &timeoutError{c.timeout}
			}
		case <-ctx.Done():
			return ctx.Err()
		case err := <-done:
			return err
		}
	}
}

func (c *monitoredCmd) hasTimedOut() bool {
	t := time.Now().Add(-c.timeout)
	return c.stderr.lastActivity().Before(t) &&
		c.stdout.lastActivity().Before(t)
}

func (c *monitoredCmd) combinedOutput(ctx context.Context) ([]byte, error) {
	if err := c.run(ctx); err != nil {
		return c.stderr.buf.Bytes(), err
	}

	return c.stdout.buf.Bytes(), nil
}

// activityBuffer is a buffer that keeps track of the last time a Write
// operation was performed on it.
type activityBuffer struct {
	sync.Mutex
	buf               *bytes.Buffer
	lastActivityStamp time.Time
}

func newActivityBuffer() *activityBuffer {
	return &activityBuffer{
		buf: bytes.NewBuffer(nil),
	}
}

func (b *activityBuffer) Write(p []byte) (int, error) {
	b.Lock()
	b.lastActivityStamp = time.Now()
	defer b.Unlock()
	return b.buf.Write(p)
}

func (b *activityBuffer) lastActivity() time.Time {
	b.Lock()
	defer b.Unlock()
	return b.lastActivityStamp
}

type timeoutError struct {
	timeout time.Duration
}

func (e timeoutError) Error() string {
	return fmt.Sprintf("command killed after %s of no activity", e.timeout)
}

type killCmdError struct {
	err error
}

func (e killCmdError) Error() string {
	return fmt.Sprintf("error killing command: %s", e.err)
}

func runFromCwd(ctx context.Context, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(exec.Command(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}

func runFromRepoDir(ctx context.Context, repo vcs.Repo, cmd string, args ...string) ([]byte, error) {
	c := newMonitoredCmd(repo.CmdFromDir(cmd, args...), 2*time.Minute)
	return c.combinedOutput(ctx)
}
<|endoftext|>


<commit_before>// errorcheck -0 -d=nil
// Fails on ppc64x because of incomplete optimization.
// See issues 9058.
// Same reason for mips64x.
// +build !ppc64,!ppc64le,!mips64,!mips64le,!amd64

// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that nil checks are removed.
// Optimization is enabled.

package p

type Struct struct {
	X int
	Y float64
}

type BigStruct struct {
	X int
	Y float64
	A [1 << 20]int
	Z string
}

type Empty struct {
}

type Empty1 struct {
	Empty
}

var (
	intp       *int
	arrayp     *[10]int
	array0p    *[0]int
	bigarrayp  *[1 << 26]int
	structp    *Struct
	bigstructp *BigStruct
	emptyp     *Empty
	empty1p    *Empty1
)

func f1() {
	_ = *intp // ERROR "generated nil check"

	// This one should be removed but the block copy needs
	// to be turned into its own pseudo-op in order to see
	// the indirect.
	_ = *arrayp // ERROR "generated nil check"

	// 0-byte indirect doesn't suffice.
	// we don't registerize globals, so there are no removed repeated nil checks.
	_ = *array0p // ERROR "generated nil check"
	_ = *array0p // ERROR "generated nil check"

	_ = *intp    // ERROR "generated nil check"
	_ = *arrayp  // ERROR "generated nil check"
	_ = *structp // ERROR "generated nil check"
	_ = *emptyp  // ERROR "generated nil check"
	_ = *arrayp  // ERROR "generated nil check"
}

func f2() {
	var (
		intp       *int
		arrayp     *[10]int
		array0p    *[0]int
		bigarrayp  *[1 << 20]int
		structp    *Struct
		bigstructp *BigStruct
		emptyp     *Empty
		empty1p    *Empty1
	)

	_ = *intp       // ERROR "generated nil check"
	_ = *arrayp     // ERROR "generated nil check"
	_ = *array0p    // ERROR "generated nil check"
	_ = *array0p    // ERROR "removed repeated nil check"
	_ = *intp       // ERROR "removed repeated nil check"
	_ = *arrayp     // ERROR "removed repeated nil check"
	_ = *structp    // ERROR "generated nil check"
	_ = *emptyp     // ERROR "generated nil check"
	_ = *arrayp     // ERROR "removed repeated nil check"
	_ = *bigarrayp  // ERROR "generated nil check" ARM removed nil check before indirect!!
	_ = *bigstructp // ERROR "generated nil check"
	_ = *empty1p    // ERROR "generated nil check"
}

func fx10k() *[10000]int

var b bool

func f3(x *[10000]int) {
	// Using a huge type and huge offsets so the compiler
	// does not expect the memory hardware to fault.
	_ = x[9999] // ERROR "generated nil check"

	for {
		if x[9999] != 0 { // ERROR "generated nil check"
			break
		}
	}

	x = fx10k()
	_ = x[9999] // ERROR "generated nil check"
	if b {
		_ = x[9999] // ERROR "removed repeated nil check"
	} else {
		_ = x[9999] // ERROR "removed repeated nil check"
	}
	_ = x[9999] // ERROR "generated nil check"

	x = fx10k()
	if b {
		_ = x[9999] // ERROR "generated nil check"
	} else {
		_ = x[9999] // ERROR "generated nil check"
	}
	_ = x[9999] // ERROR "generated nil check"

	fx10k()
	// This one is a bit redundant, if we figured out that
	// x wasn't going to change across the function call.
	// But it's a little complex to do and in practice doesn't
	// matter enough.
	_ = x[9999] // ERROR "generated nil check"
}

func f3a() {
	x := fx10k()
	y := fx10k()
	z := fx10k()
	_ = &x[9] // ERROR "generated nil check"
	y = z
	_ = &x[9] // ERROR "removed repeated nil check"
	x = y
	_ = &x[9] // ERROR "generated nil check"
}

func f3b() {
	x := fx10k()
	y := fx10k()
	_ = &x[9] // ERROR "generated nil check"
	y = x
	_ = &x[9] // ERROR "removed repeated nil check"
	x = y
	_ = &x[9] // ERROR "removed repeated nil check"
}

func fx10() *[10]int

func f4(x *[10]int) {
	// Most of these have no checks because a real memory reference follows,
	// and the offset is small enough that if x is nil, the address will still be
	// in the first unmapped page of memory.

	_ = x[9] // ERROR "removed nil check before indirect"

	for {
		if x[9] != 0 { // ERROR "removed nil check before indirect"
			break
		}
	}

	x = fx10()
	_ = x[9] // ERROR "removed nil check before indirect"
	if b {
		_ = x[9] // ERROR "removed nil check before indirect"
	} else {
		_ = x[9] // ERROR "removed nil check before indirect"
	}
	_ = x[9] // ERROR "removed nil check before indirect"

	x = fx10()
	if b {
		_ = x[9] // ERROR "removed nil check before indirect"
	} else {
		_ = &x[9] // ERROR "generated nil check"
	}
	_ = x[9] // ERROR "removed nil check before indirect"

	fx10()
	_ = x[9] // ERROR "removed nil check before indirect"

	x = fx10()
	y := fx10()
	_ = &x[9] // ERROR "generated nil check"
	y = x
	_ = &x[9] // ERROR "removed repeated nil check"
	x = y
	_ = &x[9] // ERROR "removed repeated nil check"
}

func f5(p *float32, q *float64, r *float32, s *float64) float64 {
	x := float64(*p) // ERROR "removed nil check"
	y := *q          // ERROR "removed nil check"
	*r = 7           // ERROR "removed nil check"
	*s = 9           // ERROR "removed nil check"
	return x + y
}

type T [29]byte

func f6(p, q *T) {
	x := *p // ERROR "generated nil check"
	// On ARM, the nil check on this store gets removed.  On other archs,
	// it doesn't.  Makes this hard to test.  SSA will always remove it.
	//*q = x
	_ = x
}
<commit_msg>[dev.ssa] test: remove extra tests from non-SSA builds<commit_after>// errorcheck -0 -d=nil
// Fails on ppc64x because of incomplete optimization.
// See issues 9058.
// Same reason for mips64x.
// +build !ppc64,!ppc64le,!mips64,!mips64le,!amd64

// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that nil checks are removed.
// Optimization is enabled.

package p

type Struct struct {
	X int
	Y float64
}

type BigStruct struct {
	X int
	Y float64
	A [1 << 20]int
	Z string
}

type Empty struct {
}

type Empty1 struct {
	Empty
}

var (
	intp       *int
	arrayp     *[10]int
	array0p    *[0]int
	bigarrayp  *[1 << 26]int
	structp    *Struct
	bigstructp *BigStruct
	emptyp     *Empty
	empty1p    *Empty1
)

func f1() {
	_ = *intp // ERROR "generated nil check"

	// This one should be removed but the block copy needs
	// to be turned into its own pseudo-op in order to see
	// the indirect.
	_ = *arrayp // ERROR "generated nil check"

	// 0-byte indirect doesn't suffice.
	// we don't registerize globals, so there are no removed repeated nil checks.
	_ = *array0p // ERROR "generated nil check"
	_ = *array0p // ERROR "generated nil check"

	_ = *intp    // ERROR "generated nil check"
	_ = *arrayp  // ERROR "generated nil check"
	_ = *structp // ERROR "generated nil check"
	_ = *emptyp  // ERROR "generated nil check"
	_ = *arrayp  // ERROR "generated nil check"
}

func f2() {
	var (
		intp       *int
		arrayp     *[10]int
		array0p    *[0]int
		bigarrayp  *[1 << 20]int
		structp    *Struct
		bigstructp *BigStruct
		emptyp     *Empty
		empty1p    *Empty1
	)

	_ = *intp       // ERROR "generated nil check"
	_ = *arrayp     // ERROR "generated nil check"
	_ = *array0p    // ERROR "generated nil check"
	_ = *array0p    // ERROR "removed repeated nil check"
	_ = *intp       // ERROR "removed repeated nil check"
	_ = *arrayp     // ERROR "removed repeated nil check"
	_ = *structp    // ERROR "generated nil check"
	_ = *emptyp     // ERROR "generated nil check"
	_ = *arrayp     // ERROR "removed repeated nil check"
	_ = *bigarrayp  // ERROR "generated nil check" ARM removed nil check before indirect!!
	_ = *bigstructp // ERROR "generated nil check"
	_ = *empty1p    // ERROR "generated nil check"
}

func fx10k() *[10000]int

var b bool

func f3(x *[10000]int) {
	// Using a huge type and huge offsets so the compiler
	// does not expect the memory hardware to fault.
	_ = x[9999] // ERROR "generated nil check"

	for {
		if x[9999] != 0 { // ERROR "generated nil check"
			break
		}
	}

	x = fx10k()
	_ = x[9999] // ERROR "generated nil check"
	if b {
		_ = x[9999] // ERROR "removed repeated nil check"
	} else {
		_ = x[9999] // ERROR "removed repeated nil check"
	}
	_ = x[9999] // ERROR "generated nil check"

	x = fx10k()
	if b {
		_ = x[9999] // ERROR "generated nil check"
	} else {
		_ = x[9999] // ERROR "generated nil check"
	}
	_ = x[9999] // ERROR "generated nil check"

	fx10k()
	// This one is a bit redundant, if we figured out that
	// x wasn't going to change across the function call.
	// But it's a little complex to do and in practice doesn't
	// matter enough.
	_ = x[9999] // ERROR "generated nil check"
}

func f3a() {
	x := fx10k()
	y := fx10k()
	z := fx10k()
	_ = &x[9] // ERROR "generated nil check"
	y = z
	_ = &x[9] // ERROR "removed repeated nil check"
	x = y
	_ = &x[9] // ERROR "generated nil check"
}

func f3b() {
	x := fx10k()
	y := fx10k()
	_ = &x[9] // ERROR "generated nil check"
	y = x
	_ = &x[9] // ERROR "removed repeated nil check"
	x = y
	_ = &x[9] // ERROR "removed repeated nil check"
}

func fx10() *[10]int

func f4(x *[10]int) {
	// Most of these have no checks because a real memory reference follows,
	// and the offset is small enough that if x is nil, the address will still be
	// in the first unmapped page of memory.

	_ = x[9] // ERROR "removed nil check before indirect"

	for {
		if x[9] != 0 { // ERROR "removed nil check before indirect"
			break
		}
	}

	x = fx10()
	_ = x[9] // ERROR "removed nil check before indirect"
	if b {
		_ = x[9] // ERROR "removed nil check before indirect"
	} else {
		_ = x[9] // ERROR "removed nil check before indirect"
	}
	_ = x[9] // ERROR "removed nil check before indirect"

	x = fx10()
	if b {
		_ = x[9] // ERROR "removed nil check before indirect"
	} else {
		_ = &x[9] // ERROR "generated nil check"
	}
	_ = x[9] // ERROR "removed nil check before indirect"

	fx10()
	_ = x[9] // ERROR "removed nil check before indirect"

	x = fx10()
	y := fx10()
	_ = &x[9] // ERROR "generated nil check"
	y = x
	_ = &x[9] // ERROR "removed repeated nil check"
	x = y
	_ = &x[9] // ERROR "removed repeated nil check"
}
<|endoftext|>


<commit_before>package libdupes

import (
	md5 "crypto/md5"
	"github.com/cenkalti/log"
	"io"
	"os"
	filepath "path/filepath"
	"sort"
)

// In our first pass, only hash the first initialBlocksize bytes of a file.
const initialBlocksize = 4096

type filesWithHashes struct {
	Unhashed        string
	FirstPassHashes map[[md5.Size]byte]string
	FullHashes      map[[md5.Size]byte][]string
}

// Info contains information on duplicate file sets--specifically, the per-file size in bytes and the file paths.
type Info struct {
	Size  int64
	Names []string
}

type bySize []Info

func (a bySize) Len() int      { return len(a) }
func (a bySize) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a bySize) Less(i, j int) bool {
	return a[i].Size*int64(len(a[i].Names)) < a[j].Size*int64(len(a[j].Names))
}

// Hash a file at path. If blocksize is >0, only hash up to the first blocksize bytes.
func hash(path string, blocksize int64) ([md5.Size]byte, error) {
	var sum [md5.Size]byte
	f, err := os.Open(path)
	if err != nil {
		log.Warningln(err)
		return sum, err
	}
	defer f.Close()
	h := md5.New()
	if blocksize > 0 {
		_, err = io.CopyN(h, f, blocksize)
	} else {
		_, err = io.Copy(h, f)
	}
	if err != nil {
		log.Warningln(err)
		return sum, err
	}
	s := h.Sum(nil)
	if len(s) != len(sum) {
		panic("Unexpected checksum length")
	}
	for i, v := range s {
		sum[i] = v
	}
	return sum, nil
}

func min(x, y int64) int64 {
	if x < y {
		return x
	}
	return y
}

// Dupes finds all duplicate files starting at the directory specified by "root". If specified, progressCb will be called to update the file processing progress.
func Dupes(roots []string, progressCb func(cur int, outof int)) ([]Info, error) {
	// Get files.
	// In order to enable the progress callback, we first list all the files (which should be relatively cheap) and then reiterate through the index to actually detect duplicates.
	pending := make(map[string]int64)
        for _, root := range roots {
	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		pending[path] = int64(info.Size())
		return nil
	})
      }
	files := make(map[int64]*filesWithHashes)
	i := 0
	for path, size := range pending {
		if progressCb != nil {
			progressCb(i, len(pending))
		}
		i++
		hs, ok := files[size]
		if !ok {
			// If we've never seen another file of this size, we don't have to do the md5 sum.
			files[size] = &filesWithHashes{
				Unhashed:        path,
				FirstPassHashes: make(map[[md5.Size]byte]string),
				FullHashes:      make(map[[md5.Size]byte][]string),
			}
			continue
		}
		// If there's an unhashed file, we have to compute the first-pass hashes.
		if hs.Unhashed != "" {
			// This should never ever happen, so we don't handle errors properly.
			if len(hs.FirstPassHashes) > 0 || len(hs.FullHashes) > 0 {
				panic("logic error!")
			}
			// First-pass hash of the unhashed.
			sum, err := hash(hs.Unhashed, min(initialBlocksize, size))
			if err != nil {
				log.Warningln(err)
				continue
			}
			hs.FirstPassHashes[sum] = hs.Unhashed
			hs.Unhashed = ""
		}
		// Now we compute the first-pass hash of the current file.
		sum, err := hash(path, min(initialBlocksize, size))
		if err != nil {
			log.Warningln(err)
			continue
		}
		collision, ok := hs.FirstPassHashes[sum]
		if ok {
			// Second-pass hashes required.
			if collision != "" {
				// Also have to do a second-pass hash of the previous.
				hs.FirstPassHashes[sum] = ""
				sum, err := hash(collision, -1)
				if err != nil {
					log.Warningln(err)
					continue
				}
				fs, _ := hs.FullHashes[sum]
				hs.FullHashes[sum] = append(fs, collision)
			}
			// And of the current file.
			sum, err := hash(path, -1)
			if err != nil {
				log.Warningln(err)
				continue
			}
			fs, _ := hs.FullHashes[sum]
			hs.FullHashes[sum] = append(fs, path)

		} else {
			hs.FirstPassHashes[sum] = path
		}
	}
	dupes := []Info{}
	for size, hs := range files {
		for _, files := range hs.FullHashes {
			if len(files) > 1 {
				dupes = append(dupes, Info{Size: size, Names: files})
			}
		}
	}

	sort.Sort(sort.Reverse(bySize(dupes)))
	return dupes, nil
}
<commit_msg>Properly error if path does not exist.<commit_after>package libdupes

import (
        "fmt"
	md5 "crypto/md5"
	"github.com/cenkalti/log"
	"io"
	"os"
	filepath "path/filepath"
	"sort"
)

// In our first pass, only hash the first initialBlocksize bytes of a file.
const initialBlocksize = 4096

type filesWithHashes struct {
	Unhashed        string
	FirstPassHashes map[[md5.Size]byte]string
	FullHashes      map[[md5.Size]byte][]string
}

// Info contains information on duplicate file sets--specifically, the per-file size in bytes and the file paths.
type Info struct {
	Size  int64
	Names []string
}

type bySize []Info

func (a bySize) Len() int      { return len(a) }
func (a bySize) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a bySize) Less(i, j int) bool {
	return a[i].Size*int64(len(a[i].Names)) < a[j].Size*int64(len(a[j].Names))
}

// Hash a file at path. If blocksize is >0, only hash up to the first blocksize bytes.
func hash(path string, blocksize int64) ([md5.Size]byte, error) {
	var sum [md5.Size]byte
	f, err := os.Open(path)
	if err != nil {
		log.Warningln(err)
		return sum, err
	}
	defer f.Close()
	h := md5.New()
	if blocksize > 0 {
		_, err = io.CopyN(h, f, blocksize)
	} else {
		_, err = io.Copy(h, f)
	}
	if err != nil {
		log.Warningln(err)
		return sum, err
	}
	s := h.Sum(nil)
	if len(s) != len(sum) {
		panic("Unexpected checksum length")
	}
	for i, v := range s {
		sum[i] = v
	}
	return sum, nil
}

func min(x, y int64) int64 {
	if x < y {
		return x
	}
	return y
}

// Dupes finds all duplicate files starting at the directory specified by "root". If specified, progressCb will be called to update the file processing progress.
func Dupes(roots []string, progressCb func(cur int, outof int)) ([]Info, error) {
	// Get files.
	// In order to enable the progress callback, we first list all the files (which should be relatively cheap) and then reiterate through the index to actually detect duplicates.
	pending := make(map[string]int64)
        for _, root := range roots {
	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
                if info == nil {
                        return fmt.Errorf("%s does not exist", path)
                }
		if info.IsDir() {
			return nil
		}
		pending[path] = int64(info.Size())
		return nil
	})
      }
	files := make(map[int64]*filesWithHashes)
	i := 0
	for path, size := range pending {
		if progressCb != nil {
			progressCb(i, len(pending))
		}
		i++
		hs, ok := files[size]
		if !ok {
			// If we've never seen another file of this size, we don't have to do the md5 sum.
			files[size] = &filesWithHashes{
				Unhashed:        path,
				FirstPassHashes: make(map[[md5.Size]byte]string),
				FullHashes:      make(map[[md5.Size]byte][]string),
			}
			continue
		}
		// If there's an unhashed file, we have to compute the first-pass hashes.
		if hs.Unhashed != "" {
			// This should never ever happen, so we don't handle errors properly.
			if len(hs.FirstPassHashes) > 0 || len(hs.FullHashes) > 0 {
				panic("logic error!")
			}
			// First-pass hash of the unhashed.
			sum, err := hash(hs.Unhashed, min(initialBlocksize, size))
			if err != nil {
				log.Warningln(err)
				continue
			}
			hs.FirstPassHashes[sum] = hs.Unhashed
			hs.Unhashed = ""
		}
		// Now we compute the first-pass hash of the current file.
		sum, err := hash(path, min(initialBlocksize, size))
		if err != nil {
			log.Warningln(err)
			continue
		}
		collision, ok := hs.FirstPassHashes[sum]
		if ok {
			// Second-pass hashes required.
			if collision != "" {
				// Also have to do a second-pass hash of the previous.
				hs.FirstPassHashes[sum] = ""
				sum, err := hash(collision, -1)
				if err != nil {
					log.Warningln(err)
					continue
				}
				fs, _ := hs.FullHashes[sum]
				hs.FullHashes[sum] = append(fs, collision)
			}
			// And of the current file.
			sum, err := hash(path, -1)
			if err != nil {
				log.Warningln(err)
				continue
			}
			fs, _ := hs.FullHashes[sum]
			hs.FullHashes[sum] = append(fs, path)

		} else {
			hs.FirstPassHashes[sum] = path
		}
	}
	dupes := []Info{}
	for size, hs := range files {
		for _, files := range hs.FullHashes {
			if len(files) > 1 {
				dupes = append(dupes, Info{Size: size, Names: files})
			}
		}
	}

	sort.Sort(sort.Reverse(bySize(dupes)))
	return dupes, nil
}
<|endoftext|>


<commit_before>package mmu

import (
	"github.com/djhworld/gomeboycolor/types"
	"github.com/stretchrcom/testify/assert"
	"testing"
)

func TestWriteByteToExternalRAM(t *testing.T) {
	//boundary tests

	//low
	var address types.Word = 0xA000
	var value byte = 0x83
	var normalisedLoc int = 0

	t.Logf("Writing %X to %X", value, address)
	gbc := new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.externalRAM[normalisedLoc], value)

	//middle
	address = 0xAFFF
	value = 0x33
	normalisedLoc = 4095

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.externalRAM[normalisedLoc], value)

	//high
	address = 0xBFFF
	value = 0xA2
	normalisedLoc = 8191

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.externalRAM[normalisedLoc], value)
}

func TestWriteByteToWorkingRAM(t *testing.T) {
	//boundary tests

	//low
	var address types.Word = 0xC000
	var value byte = 0x83
	var normalisedLoc int = 0
	var normalisedShadowLoc int = 0

	t.Logf("Writing %X to %X", value, address)
	gbc := new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.workingRAM[normalisedLoc], value)
	//check shadow
	assert.Equal(t, gbc.workingRAMShadow[normalisedShadowLoc], value)

	//middle
	address = 0xCFFF
	value = 0x31
	normalisedLoc = 4095
	normalisedShadowLoc = 3583

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.workingRAM[normalisedLoc], value)
	//check shadow
	assert.Equal(t, gbc.workingRAMShadow[normalisedShadowLoc], value)

	//high
	address = 0xDFFF
	value = 0x87
	normalisedLoc = 8191

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.workingRAM[normalisedLoc], value)
	//no shadow available as working ram shadow shaves off that last 512 bytes
}

func TestWriteByteToZeroPageRAM(t *testing.T) {
	//boundary tests

	//low
	var address types.Word = 0xFF80
	var value byte = 0x83
	var normalisedLoc int = 0

	t.Logf("Writing %X to %X", value, address)
	gbc := new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.zeroPageRAM[normalisedLoc], value)

	//middle
	address = 0xFFBF
	value = 0x33
	normalisedLoc = 63

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.zeroPageRAM[normalisedLoc], value)

	//high
	address = 0xFFFF
	value = 0xA2
	normalisedLoc = 127

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.zeroPageRAM[normalisedLoc], value)
}

func TestWriteByteToBootRegion(t *testing.T) {
	gbc := new(GbcMMU)
	gbc.inBootMode = true

	//should panic as you can't write to ROM!
	assert.Panics(t, func() {
		gbc.WriteByte(0x0001, 0xFE)
	}, "Should have panicked!")
}

func TestWriteByteToROMRegion(t *testing.T) {
	gbc := new(GbcMMU)
	gbc.inBootMode = false

	//should panic as you can't write to ROM!
	assert.Panics(t, func() {
		gbc.WriteByte(0x3FFE, 0xFE)
	}, "Should have paniciked!")
}

func TestRegionBoundaries(t *testing.T) {
	gbc := new(GbcMMU)
	gbc.boot[0] = 1
	gbc.boot[255] = 1

	gbc.cartrom[0] = 1
	gbc.cartrom[32767] = 1

	gbc.externalRAM[0] = 1
	gbc.externalRAM[8191] = 1

	gbc.workingRAM[0] = 1
	gbc.workingRAM[8191] = 1

	gbc.workingRAMShadow[0] = 1
	gbc.workingRAMShadow[7679] = 1

	gbc.zeroPageRAM[0] = 1
	gbc.zeroPageRAM[127] = 1

}

func TestReadByteFromBoot(t *testing.T) {
	var ROM []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3}
	gbc := new(GbcMMU)
	gbc.SetInBootMode(true)
	gbc.LoadROM(0, BOOT, ROM)
	assert.Equal(t, gbc.ReadByte(0x0002), ROM[2])
}

func TestReadByteFromCart(t *testing.T) {
	var ROM []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3}
	gbc := new(GbcMMU)
	gbc.SetInBootMode(false)
	gbc.LoadROM(0x1000, CARTROM, ROM)
	assert.Equal(t, gbc.ReadByte(0x1002), ROM[2])
}

func TestReadWriteByte(t *testing.T) {
	var value byte = 0xFC
	var addr types.Word = 0xC476
	gbc := new(GbcMMU)
	gbc.WriteByte(addr, value)
	assert.Equal(t, gbc.ReadByte(addr), value)
}

func TestLoadBootROM(t *testing.T) {
	var startAddr types.Word = 0
	var ROM []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3}
	gbc := new(GbcMMU)
	gbc.LoadROM(startAddr, BOOT, ROM)
	//check whether start address -> end of ROM is equal to ROM
	assert.Equal(t, gbc.boot[startAddr:len(ROM)], ROM)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 253
	ok, err := gbc.LoadROM(startAddr, BOOT, ROM)
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMWillOverextendAddressableRegion, err)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 0
	ok, err = gbc.LoadROM(startAddr, BOOT, make([]byte, 3000))
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMIsBiggerThanRegion, err)
}

func TestLoadCartROM(t *testing.T) {
	var startAddr types.Word = 0
	var rom []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3, 0xFF, 0x2C}
	gbc := new(GbcMMU)
	gbc.LoadROM(startAddr, CARTROM, rom)
	//check whether start address -> end of ROM is equal to ROM
	assert.Equal(t, gbc.cartrom[startAddr:len(rom)], rom)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 32765
	ok, err := gbc.LoadROM(startAddr, CARTROM, rom)
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMWillOverextendAddressableRegion, err)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 0
	ok, err = gbc.LoadROM(startAddr, CARTROM, make([]byte, 42765))
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMIsBiggerThanRegion, err)
}

func TestImplementsInterface(t *testing.T) {
	gbc := new(GbcMMU)
	assert.Implements(t, (*MemoryMappedUnit)(nil), gbc)
}
<commit_msg>Added extra read test to make sure all regions can be read<commit_after>package mmu

import (
	"github.com/djhworld/gomeboycolor/types"
	"github.com/stretchrcom/testify/assert"
	"testing"
)

func TestWriteByteToExternalRAM(t *testing.T) {
	//boundary tests

	//low
	var address types.Word = 0xA000
	var value byte = 0x83
	var normalisedLoc int = 0

	t.Logf("Writing %X to %X", value, address)
	gbc := new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.externalRAM[normalisedLoc], value)

	//middle
	address = 0xAFFF
	value = 0x33
	normalisedLoc = 4095

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.externalRAM[normalisedLoc], value)

	//high
	address = 0xBFFF
	value = 0xA2
	normalisedLoc = 8191

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.externalRAM[normalisedLoc], value)
}

func TestWriteByteToWorkingRAM(t *testing.T) {
	//boundary tests

	//low
	var address types.Word = 0xC000
	var value byte = 0x83
	var normalisedLoc int = 0
	var normalisedShadowLoc int = 0

	t.Logf("Writing %X to %X", value, address)
	gbc := new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.workingRAM[normalisedLoc], value)
	//check shadow
	assert.Equal(t, gbc.workingRAMShadow[normalisedShadowLoc], value)

	//middle
	address = 0xCFFF
	value = 0x31
	normalisedLoc = 4095
	normalisedShadowLoc = 3583

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.workingRAM[normalisedLoc], value)
	//check shadow
	assert.Equal(t, gbc.workingRAMShadow[normalisedShadowLoc], value)

	//high
	address = 0xDFFF
	value = 0x87
	normalisedLoc = 8191

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.workingRAM[normalisedLoc], value)
	//no shadow available as working ram shadow shaves off that last 512 bytes
}

func TestWriteByteToZeroPageRAM(t *testing.T) {
	//boundary tests

	//low
	var address types.Word = 0xFF80
	var value byte = 0x83
	var normalisedLoc int = 0

	t.Logf("Writing %X to %X", value, address)
	gbc := new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.zeroPageRAM[normalisedLoc], value)

	//middle
	address = 0xFFBF
	value = 0x33
	normalisedLoc = 63

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.zeroPageRAM[normalisedLoc], value)

	//high
	address = 0xFFFF
	value = 0xA2
	normalisedLoc = 127

	t.Logf("Writing %X to %X", value, address)
	gbc = new(GbcMMU)
	gbc.inBootMode = false
	gbc.WriteByte(address, value)
	assert.Equal(t, gbc.zeroPageRAM[normalisedLoc], value)
}

func TestWriteByteToBootRegion(t *testing.T) {
	gbc := new(GbcMMU)
	gbc.inBootMode = true

	//should panic as you can't write to ROM!
	assert.Panics(t, func() {
		gbc.WriteByte(0x0001, 0xFE)
	}, "Should have panicked!")
}

func TestWriteByteToROMRegion(t *testing.T) {
	gbc := new(GbcMMU)
	gbc.inBootMode = false

	//should panic as you can't write to ROM!
	assert.Panics(t, func() {
		gbc.WriteByte(0x3FFE, 0xFE)
	}, "Should have paniciked!")
}

func TestRegionBoundaries(t *testing.T) {
	gbc := new(GbcMMU)
	gbc.boot[0] = 1
	gbc.boot[255] = 1

	gbc.cartrom[0] = 1
	gbc.cartrom[32767] = 1

	gbc.externalRAM[0] = 1
	gbc.externalRAM[8191] = 1

	gbc.workingRAM[0] = 1
	gbc.workingRAM[8191] = 1

	gbc.workingRAMShadow[0] = 1
	gbc.workingRAMShadow[7679] = 1

	gbc.zeroPageRAM[0] = 1
	gbc.zeroPageRAM[127] = 1

}

func TestReadByteFromBoot(t *testing.T) {
	var ROM []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3}
	gbc := new(GbcMMU)
	gbc.SetInBootMode(true)
	gbc.LoadROM(0, BOOT, ROM)
	assert.Equal(t, gbc.ReadByte(0x0002), ROM[2])
}

func TestReadByteFromCart(t *testing.T) {
	var ROM []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3}
	gbc := new(GbcMMU)
	gbc.SetInBootMode(false)
	gbc.LoadROM(0x1000, CARTROM, ROM)
	assert.Equal(t, gbc.ReadByte(0x1002), ROM[2])
}

func TestReadWriteByte(t *testing.T) {
	var value byte = 0xFC
	var addr types.Word = 0xC476
	gbc := new(GbcMMU)
	gbc.WriteByte(addr, value)
	assert.Equal(t, gbc.ReadByte(addr), value)
}

func TestLoadBootROM(t *testing.T) {
	var startAddr types.Word = 0
	var ROM []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3}
	gbc := new(GbcMMU)
	gbc.LoadROM(startAddr, BOOT, ROM)
	//check whether start address -> end of ROM is equal to ROM
	assert.Equal(t, gbc.boot[startAddr:len(ROM)], ROM)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 253
	ok, err := gbc.LoadROM(startAddr, BOOT, ROM)
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMWillOverextendAddressableRegion, err)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 0
	ok, err = gbc.LoadROM(startAddr, BOOT, make([]byte, 3000))
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMIsBiggerThanRegion, err)
}

func TestLoadCartROM(t *testing.T) {
	var startAddr types.Word = 0
	var rom []byte = []byte{0x03, 0x77, 0x04, 0xFF, 0xA3, 0xA2, 0xB3, 0xFF, 0x2C}
	gbc := new(GbcMMU)
	gbc.LoadROM(startAddr, CARTROM, rom)
	//check whether start address -> end of ROM is equal to ROM
	assert.Equal(t, gbc.cartrom[startAddr:len(rom)], rom)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 32765
	ok, err := gbc.LoadROM(startAddr, CARTROM, rom)
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMWillOverextendAddressableRegion, err)

	//check that error is returned if ROM is loaded that will over extend BOOT region
	gbc = new(GbcMMU)
	startAddr = 0
	ok, err = gbc.LoadROM(startAddr, CARTROM, make([]byte, 42765))
	assert.False(t, ok)
	assert.NotNil(t, err)
	assert.Equal(t, ROMIsBiggerThanRegion, err)
}

func TestImplementsInterface(t *testing.T) {
	gbc := new(GbcMMU)
	assert.Implements(t, (*MemoryMappedUnit)(nil), gbc)
}

func TestReadByte(t *testing.T) {
	rom := make([]byte, 32768,32768)
	for i := 32767; i >= 0; i-- {
		rom[i] = byte(i)
	}
	gbc := new(GbcMMU)
	gbc.SetInBootMode(false)
	gbc.LoadROM(0x0000, CARTROM, rom)

	var i types.Word = 0x0000
	for ; i < 0x8000; i++ {
		f := gbc.ReadByte(i)
		assert.Equal(t, f, rom[i])
	}
}
<|endoftext|>


<commit_before>package internal

import (
	"fmt"
	"strconv"
)

// HandlerType are the handler targets for commands and events.
type HandlerType string

// HandlerType values.
const (
	HandlerTypeBrowser  HandlerType = "browser"
	HandlerTypeRenderer HandlerType = "renderer"
)

// String satisfies stringer.
func (ht HandlerType) String() string {
	return string(ht)
}

// MarshalJSON satisfies json.Marshaler.
func (ht HandlerType) MarshalJSON() ([]byte, error) {
	return []byte(`"` + ht + `"`), nil
}

// UnmarshalJSON satisfies json.Unmarshaler.
func (ht *HandlerType) UnmarshalJSON(buf []byte) error {
	s, err := strconv.Unquote(string(buf))
	if err != nil {
		return err
	}

	switch HandlerType(s) {
	case HandlerTypeBrowser:
		*ht = HandlerTypeBrowser
	case HandlerTypeRenderer:
		*ht = HandlerTypeRenderer

	default:
		return fmt.Errorf("unknown handler type %s", string(buf))
	}

	return nil
}

// TypeEnum is the Chrome domain type enum.
type TypeEnum string

// TypeEnum values.
const (
	TypeAny       TypeEnum = "any"
	TypeArray     TypeEnum = "array"
	TypeBoolean   TypeEnum = "boolean"
	TypeInteger   TypeEnum = "integer"
	TypeNumber    TypeEnum = "number"
	TypeObject    TypeEnum = "object"
	TypeString    TypeEnum = "string"
	TypeTimestamp TypeEnum = "timestamp"
)

// String satisfies stringer.
func (te TypeEnum) String() string {
	return string(te)
}

// MarshalJSON satisfies json.Marshaler.
func (te TypeEnum) MarshalJSON() ([]byte, error) {
	return []byte(`"` + te + `"`), nil
}

// UnmarshalJSON satisfies json.Unmarshaler.
func (te *TypeEnum) UnmarshalJSON(buf []byte) error {
	s, err := strconv.Unquote(string(buf))
	if err != nil {
		return err
	}

	switch TypeEnum(s) {
	case TypeAny:
		*te = TypeAny
	case TypeArray:
		*te = TypeArray
	case TypeBoolean:
		*te = TypeBoolean
	case TypeInteger:
		*te = TypeInteger
	case TypeNumber:
		*te = TypeNumber
	case TypeObject:
		*te = TypeObject
	case TypeString:
		*te = TypeString

	default:
		return fmt.Errorf("unknown type enum %s", string(buf))
	}

	return nil
}

// GoType returns the Go type for the TypeEnum.
func (te TypeEnum) GoType() string {
	switch te {
	case TypeAny:
		return "easyjson.RawMessage"

	case TypeBoolean:
		return "bool"

	case TypeInteger:
		return "int64"

	case TypeNumber:
		return "float64"

	case TypeString:
		return "string"

	case TypeTimestamp:
		return "time.Time"

	default:
		panic(fmt.Sprintf("called GoType on non primitive type %s", te.String()))
	}

	return ""
}

// GoEmptyValue returns the Go empty value for the TypeEnum.
func (te TypeEnum) GoEmptyValue() string {
	switch te {
	case TypeBoolean:
		return `false`

	case TypeInteger:
		return `0`

	case TypeNumber:
		return `0`

	case TypeString:
		return `""`

	case TypeTimestamp:
		return `time.Time{}`
	}

	return `nil`
}

// TimestampType are the various timestamp subtypes.
type TimestampType int

const (
	TimestampTypeMillisecond TimestampType = 1 + iota
	TimestampTypeSecond
	TimestampTypeBootstamp
)
<commit_msg>Adding missing comment to TimestampType const declarations<commit_after>package internal

import (
	"fmt"
	"strconv"
)

// HandlerType are the handler targets for commands and events.
type HandlerType string

// HandlerType values.
const (
	HandlerTypeBrowser  HandlerType = "browser"
	HandlerTypeRenderer HandlerType = "renderer"
)

// String satisfies stringer.
func (ht HandlerType) String() string {
	return string(ht)
}

// MarshalJSON satisfies json.Marshaler.
func (ht HandlerType) MarshalJSON() ([]byte, error) {
	return []byte(`"` + ht + `"`), nil
}

// UnmarshalJSON satisfies json.Unmarshaler.
func (ht *HandlerType) UnmarshalJSON(buf []byte) error {
	s, err := strconv.Unquote(string(buf))
	if err != nil {
		return err
	}

	switch HandlerType(s) {
	case HandlerTypeBrowser:
		*ht = HandlerTypeBrowser
	case HandlerTypeRenderer:
		*ht = HandlerTypeRenderer

	default:
		return fmt.Errorf("unknown handler type %s", string(buf))
	}

	return nil
}

// TypeEnum is the Chrome domain type enum.
type TypeEnum string

// TypeEnum values.
const (
	TypeAny       TypeEnum = "any"
	TypeArray     TypeEnum = "array"
	TypeBoolean   TypeEnum = "boolean"
	TypeInteger   TypeEnum = "integer"
	TypeNumber    TypeEnum = "number"
	TypeObject    TypeEnum = "object"
	TypeString    TypeEnum = "string"
	TypeTimestamp TypeEnum = "timestamp"
)

// String satisfies stringer.
func (te TypeEnum) String() string {
	return string(te)
}

// MarshalJSON satisfies json.Marshaler.
func (te TypeEnum) MarshalJSON() ([]byte, error) {
	return []byte(`"` + te + `"`), nil
}

// UnmarshalJSON satisfies json.Unmarshaler.
func (te *TypeEnum) UnmarshalJSON(buf []byte) error {
	s, err := strconv.Unquote(string(buf))
	if err != nil {
		return err
	}

	switch TypeEnum(s) {
	case TypeAny:
		*te = TypeAny
	case TypeArray:
		*te = TypeArray
	case TypeBoolean:
		*te = TypeBoolean
	case TypeInteger:
		*te = TypeInteger
	case TypeNumber:
		*te = TypeNumber
	case TypeObject:
		*te = TypeObject
	case TypeString:
		*te = TypeString

	default:
		return fmt.Errorf("unknown type enum %s", string(buf))
	}

	return nil
}

// GoType returns the Go type for the TypeEnum.
func (te TypeEnum) GoType() string {
	switch te {
	case TypeAny:
		return "easyjson.RawMessage"

	case TypeBoolean:
		return "bool"

	case TypeInteger:
		return "int64"

	case TypeNumber:
		return "float64"

	case TypeString:
		return "string"

	case TypeTimestamp:
		return "time.Time"

	default:
		panic(fmt.Sprintf("called GoType on non primitive type %s", te.String()))
	}

	return ""
}

// GoEmptyValue returns the Go empty value for the TypeEnum.
func (te TypeEnum) GoEmptyValue() string {
	switch te {
	case TypeBoolean:
		return `false`

	case TypeInteger:
		return `0`

	case TypeNumber:
		return `0`

	case TypeString:
		return `""`

	case TypeTimestamp:
		return `time.Time{}`
	}

	return `nil`
}

// TimestampType are the various timestamp subtypes.
type TimestampType int

// TimestampType values.
const (
	TimestampTypeMillisecond TimestampType = 1 + iota
	TimestampTypeSecond
	TimestampTypeBootstamp
)
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/spf13/cobra"

	"github.com/weaveworks/flux/api/v6"
	"github.com/weaveworks/flux/policy"
)

type workloadListOpts struct {
	*rootOpts
	namespace     string
	allNamespaces bool
}

func newWorkloadList(parent *rootOpts) *workloadListOpts {
	return &workloadListOpts{rootOpts: parent}
}

func (opts *workloadListOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "list-workloads",
		Aliases: []string{"list-controllers"}, // Transient backwards compatibility after replacing controller by workload
		Short:   "List workloads currently running in the cluster.",
		Example: makeExample("fluxctl list-workloads"),
		RunE:    opts.RunE,
	}
	cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "default", "Confine query to namespace")
	cmd.Flags().BoolVarP(&opts.allNamespaces, "all-namespaces", "a", false, "Query across all namespaces")
	return cmd
}

func (opts *workloadListOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return errorWantedNoArgs
	}

	if opts.allNamespaces {
		opts.namespace = ""
	}

	ctx := context.Background()

	workloads, err := opts.API.ListServices(ctx, opts.namespace)
	if err != nil {
		return err
	}

	sort.Sort(workloadStatusByName(workloads))

	w := newTabwriter()
	fmt.Fprintf(w, "WORKLOAD\tCONTAINER\tIMAGE\tRELEASE\tPOLICY\n")
	for _, workload := range workloads {
		if len(workload.Containers) > 0 {
			c := workload.Containers[0]
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", workload.ID, c.Name, c.Current.ID, workload.Status, policies(workload))
			for _, c := range workload.Containers[1:] {
				fmt.Fprintf(w, "\t%s\t%s\t\t\n", c.Name, c.Current.ID)
			}
		} else {
			fmt.Fprintf(w, "%s\t\t\t\t\n", workload.ID)
		}
	}
	w.Flush()
	return nil
}

type workloadStatusByName []v6.ControllerStatus

func (s workloadStatusByName) Len() int {
	return len(s)
}

func (s workloadStatusByName) Less(a, b int) bool {
	return s[a].ID.String() < s[b].ID.String()
}

func (s workloadStatusByName) Swap(a, b int) {
	s[a], s[b] = s[b], s[a]
}

func policies(s v6.ControllerStatus) string {
	var ps []string
	if s.Automated {
		ps = append(ps, string(policy.Automated))
	}
	if s.Locked {
		ps = append(ps, string(policy.Locked))
	}
	if s.Ignore {
		ps = append(ps, string(policy.Ignore))
	}
	sort.Strings(ps)
	return strings.Join(ps, ",")
}
<commit_msg>Always list the status of a workload in fluxctl<commit_after>package main

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/spf13/cobra"

	"github.com/weaveworks/flux/api/v6"
	"github.com/weaveworks/flux/policy"
)

type workloadListOpts struct {
	*rootOpts
	namespace     string
	allNamespaces bool
}

func newWorkloadList(parent *rootOpts) *workloadListOpts {
	return &workloadListOpts{rootOpts: parent}
}

func (opts *workloadListOpts) Command() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "list-workloads",
		Aliases: []string{"list-controllers"}, // Transient backwards compatibility after replacing controller by workload
		Short:   "List workloads currently running in the cluster.",
		Example: makeExample("fluxctl list-workloads"),
		RunE:    opts.RunE,
	}
	cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "default", "Confine query to namespace")
	cmd.Flags().BoolVarP(&opts.allNamespaces, "all-namespaces", "a", false, "Query across all namespaces")
	return cmd
}

func (opts *workloadListOpts) RunE(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return errorWantedNoArgs
	}

	if opts.allNamespaces {
		opts.namespace = ""
	}

	ctx := context.Background()

	workloads, err := opts.API.ListServices(ctx, opts.namespace)
	if err != nil {
		return err
	}

	sort.Sort(workloadStatusByName(workloads))

	w := newTabwriter()
	fmt.Fprintf(w, "WORKLOAD\tCONTAINER\tIMAGE\tRELEASE\tPOLICY\n")
	for _, workload := range workloads {
		if len(workload.Containers) > 0 {
			c := workload.Containers[0]
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", workload.ID, c.Name, c.Current.ID, workload.Status, policies(workload))
			for _, c := range workload.Containers[1:] {
				fmt.Fprintf(w, "\t%s\t%s\t\t\n", c.Name, c.Current.ID)
			}
		} else {
			fmt.Fprintf(w, "%s\t\t\t%s\t%s\n", workload.ID, workload.Status, policies(workload))
		}
	}
	w.Flush()
	return nil
}

type workloadStatusByName []v6.ControllerStatus

func (s workloadStatusByName) Len() int {
	return len(s)
}

func (s workloadStatusByName) Less(a, b int) bool {
	return s[a].ID.String() < s[b].ID.String()
}

func (s workloadStatusByName) Swap(a, b int) {
	s[a], s[b] = s[b], s[a]
}

func policies(s v6.ControllerStatus) string {
	var ps []string
	if s.Automated {
		ps = append(ps, string(policy.Automated))
	}
	if s.Locked {
		ps = append(ps, string(policy.Locked))
	}
	if s.Ignore {
		ps = append(ps, string(policy.Ignore))
	}
	sort.Strings(ps)
	return strings.Join(ps, ",")
}
<|endoftext|>


<commit_before>package gateway

import (
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/golib/ratelimiter"
	log "github.com/funkygao/log4go"
)

type subServer struct {
	*webServer

	idleConnsWg   sync.WaitGroup // wait for all inflight http connections done
	closedConnCh  chan string    // channel of remote addr
	idleConns     map[net.Conn]struct{}
	idleConnsLock sync.Mutex

	auditor log.Logger

	// websocket heartbeat configuration
	wsReadLimit int64
	wsPongWait  time.Duration

	shutdownOnce sync.Once
	ackShutdown  int32                                          // sync shutdown with ack handlers goroutines
	ackCh        chan ackOffsets                                // client ack'ed offsets
	ackedOffsets map[string]map[string]map[string]map[int]int64 // [cluster][topic][group][partition]: offset

	subMetrics        *subMetrics
	throttleSubStatus *ratelimiter.LeakyBuckets
}

func newSubServer(httpAddr, httpsAddr string, maxClients int, gw *Gateway) *subServer {
	this := &subServer{
		webServer:         newWebServer("sub", httpAddr, httpsAddr, maxClients, gw),
		closedConnCh:      make(chan string, 1<<10),
		idleConns:         make(map[net.Conn]struct{}, 200),
		wsReadLimit:       8 << 10,
		wsPongWait:        time.Minute,
		throttleSubStatus: ratelimiter.NewLeakyBuckets(60, time.Minute),
		ackShutdown:       0,
		ackCh:             make(chan ackOffsets, 100),
		ackedOffsets:      make(map[string]map[string]map[string]map[int]int64),
	}
	this.subMetrics = NewSubMetrics(this.gw)
	this.waitExitFunc = this.waitExit
	this.connStateFunc = this.connStateHandler

	if this.httpsServer != nil {
		this.httpsServer.ConnState = this.connStateFunc
	}

	if this.httpServer != nil {
		this.httpServer.ConnState = this.connStateFunc
	}

	this.auditor = log.NewDefaultLogger(log.TRACE)
	this.auditor.DeleteFilter("stdout")

	_ = os.Mkdir("audit", os.ModePerm)
	rotateEnabled, discardWhenDiskFull := true, false
	filer := log.NewFileLogWriter("audit/sub_audit.log", rotateEnabled, discardWhenDiskFull, 0644)
	if filer == nil {
		panic("failed to open sub audit log")
	}
	filer.SetFormat("[%d %T] [%L] (%S) %M")
	if Options.LogRotateSize > 0 {
		filer.SetRotateSize(Options.LogRotateSize)
	}
	filer.SetRotateLines(0)
	filer.SetRotateDaily(true)
	this.auditor.AddFilter("file", logLevel, filer)

	return this
}

func (this *subServer) Start() {
	this.gw.wg.Add(1)
	go this.ackCommitter()

	this.subMetrics.Load()
	this.webServer.Start()
}

func (this *subServer) connStateHandler(c net.Conn, cs http.ConnState) {
	switch cs {
	case http.StateNew:
		// Connections begin at StateNew and then
		// transition to either StateActive or StateClosed
		this.idleConnsWg.Add(1)

		if this.gw != nil && !Options.DisableMetrics {
			this.gw.svrMetrics.ConcurrentSub.Inc(1)
		}

	case http.StateActive:
		// StateActive fires before the request has entered a handler
		// and doesn't fire again until the request has been
		// handled.
		// After the request is handled, the state
		// transitions to StateClosed, StateHijacked, or StateIdle.
		this.idleConnsLock.Lock()
		delete(this.idleConns, c)
		this.idleConnsLock.Unlock()

	case http.StateIdle:
		// StateIdle represents a connection that has finished
		// handling a request and is in the keep-alive state, waiting
		// for a new request. Connections transition from StateIdle
		// to either StateActive or StateClosed.
		select {
		case <-this.gw.shutdownCh:
			// actively close the client safely because IO is all done
			c.Close()

		default:
			this.idleConnsLock.Lock()
			this.idleConns[c] = struct{}{}
			this.idleConnsLock.Unlock()
		}

	case http.StateHijacked:
		// websocket steals the socket
		if this.gw != nil && !Options.DisableMetrics {
			this.gw.svrMetrics.ConcurrentSub.Dec(1)

			this.gw.svrMetrics.ConcurrentSubWs.Inc(1)
		}

	case http.StateClosed:
		if this.gw != nil && !Options.DisableMetrics {
			this.gw.svrMetrics.ConcurrentSub.Dec(1)
		}

		remoteAddr := c.RemoteAddr().String()
		if Options.EnableClientStats {
			this.gw.clientStates.UnregisterSubClient(remoteAddr)
		}

		this.closedConnCh <- remoteAddr
		this.idleConnsWg.Done()
	}
}

func (this *subServer) waitExit(server *http.Server, listener net.Listener, exit <-chan struct{}) {
	<-exit

	// HTTP response will have "Connection: close"
	server.SetKeepAlivesEnabled(false)

	// avoid new connections
	if err := listener.Close(); err != nil {
		log.Error(err.Error())
	}

	this.idleConnsLock.Lock()
	t := time.Now().Add(time.Millisecond * 100)
	for c := range this.idleConns {
		c.SetReadDeadline(t)
	}
	this.idleConnsLock.Unlock()

	log.Trace("%s waiting for all connected client close...", this.name)
	if waitTimeout(&this.idleConnsWg, Options.SubTimeout) {
		log.Warn("%s waiting for all connected client close timeout: %s",
			this.name, Options.SubTimeout)
	}

	this.subMetrics.Flush()

	this.gw.wg.Done()
}

func (this *subServer) ackCommitter() {
	ticker := time.NewTicker(time.Second * 30)
	defer func() {
		ticker.Stop()
		log.Debug("ack committer done")
		this.gw.wg.Done()
	}()

	for {
		select {
		case <-this.gw.shutdownCh:
			this.shutdownOnce.Do(func() {
				atomic.AddInt32(&this.ackShutdown, -1)

				for {
					// waiting for all ack handlers finish
					if atomic.LoadInt32(&this.ackShutdown) <= -1 {
						break
					}

					time.Sleep(time.Millisecond * 10)
				}
				close(this.ackCh)
			})

		case acks, ok := <-this.ackCh:
			if ok {
				for _, ack := range acks {
					if _, present := this.ackedOffsets[ack.cluster]; !present {
						this.ackedOffsets[ack.cluster] = make(map[string]map[string]map[int]int64)
					}
					if _, present := this.ackedOffsets[ack.cluster][ack.topic]; !present {
						this.ackedOffsets[ack.cluster][ack.topic] = make(map[string]map[int]int64)
					}
					if _, present := this.ackedOffsets[ack.topic][ack.group]; !present {
						this.ackedOffsets[ack.cluster][ack.topic][ack.group] = make(map[int]int64)
					}

					// TODO validation
					this.ackedOffsets[ack.cluster][ack.topic][ack.group][ack.Partition] = ack.Offset
				}
			} else {
				// channel buffer drained, flush all offsets
				// zk is still alive, safe to commit offsets
				this.commitOffsets()
				return
			}

		case <-ticker.C:
			this.commitOffsets()
		}
	}

}

func (this *subServer) commitOffsets() {
	for cluster, clusterTopic := range this.ackedOffsets {
		zkcluster := meta.Default.ZkCluster(cluster)

		for topic, groupPartition := range clusterTopic {
			for group, partitionOffset := range groupPartition {
				for partition, offset := range partitionOffset {
					if offset == -1 {
						// this slot is empty
						continue
					}

					log.Debug("commit offset {C:%s T:%s G:%s P:%d O:%d}", cluster, topic, group, partition, offset)

					if err := zkcluster.ResetConsumerGroupOffset(topic, group,
						strconv.Itoa(partition), offset); err != nil {
						log.Error("commitOffsets: %v", err)
					} else {
						// mark this slot empty
						this.ackedOffsets[cluster][topic][group][partition] = -1
					}
				}
			}
		}
	}

}
<commit_msg>when conn state switch to StateHijacked|StateClosed, rm from idleConns<commit_after>package gateway

import (
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/golib/ratelimiter"
	log "github.com/funkygao/log4go"
)

type subServer struct {
	*webServer

	idleConnsWg   sync.WaitGroup // wait for all inflight http connections done
	closedConnCh  chan string    // channel of remote addr
	idleConns     map[net.Conn]struct{}
	idleConnsLock sync.Mutex

	auditor log.Logger

	// websocket heartbeat configuration
	wsReadLimit int64
	wsPongWait  time.Duration

	shutdownOnce sync.Once
	ackShutdown  int32                                          // sync shutdown with ack handlers goroutines
	ackCh        chan ackOffsets                                // client ack'ed offsets
	ackedOffsets map[string]map[string]map[string]map[int]int64 // [cluster][topic][group][partition]: offset

	subMetrics        *subMetrics
	throttleSubStatus *ratelimiter.LeakyBuckets
}

func newSubServer(httpAddr, httpsAddr string, maxClients int, gw *Gateway) *subServer {
	this := &subServer{
		webServer:         newWebServer("sub", httpAddr, httpsAddr, maxClients, gw),
		closedConnCh:      make(chan string, 1<<10),
		idleConns:         make(map[net.Conn]struct{}, 200),
		wsReadLimit:       8 << 10,
		wsPongWait:        time.Minute,
		throttleSubStatus: ratelimiter.NewLeakyBuckets(60, time.Minute),
		ackShutdown:       0,
		ackCh:             make(chan ackOffsets, 100),
		ackedOffsets:      make(map[string]map[string]map[string]map[int]int64),
	}
	this.subMetrics = NewSubMetrics(this.gw)
	this.waitExitFunc = this.waitExit
	this.connStateFunc = this.connStateHandler

	if this.httpsServer != nil {
		this.httpsServer.ConnState = this.connStateFunc
	}

	if this.httpServer != nil {
		this.httpServer.ConnState = this.connStateFunc
	}

	this.auditor = log.NewDefaultLogger(log.TRACE)
	this.auditor.DeleteFilter("stdout")

	_ = os.Mkdir("audit", os.ModePerm)
	rotateEnabled, discardWhenDiskFull := true, false
	filer := log.NewFileLogWriter("audit/sub_audit.log", rotateEnabled, discardWhenDiskFull, 0644)
	if filer == nil {
		panic("failed to open sub audit log")
	}
	filer.SetFormat("[%d %T] [%L] (%S) %M")
	if Options.LogRotateSize > 0 {
		filer.SetRotateSize(Options.LogRotateSize)
	}
	filer.SetRotateLines(0)
	filer.SetRotateDaily(true)
	this.auditor.AddFilter("file", logLevel, filer)

	return this
}

func (this *subServer) Start() {
	this.gw.wg.Add(1)
	go this.ackCommitter()

	this.subMetrics.Load()
	this.webServer.Start()
}

func (this *subServer) connStateHandler(c net.Conn, cs http.ConnState) {
	switch cs {
	case http.StateNew:
		// Connections begin at StateNew and then
		// transition to either StateActive or StateClosed
		this.idleConnsWg.Add(1)

		if this.gw != nil && !Options.DisableMetrics {
			this.gw.svrMetrics.ConcurrentSub.Inc(1)
		}

	case http.StateActive:
		// StateActive fires before the request has entered a handler
		// and doesn't fire again until the request has been
		// handled.
		// After the request is handled, the state
		// transitions to StateClosed, StateHijacked, or StateIdle.
		this.idleConnsLock.Lock()
		delete(this.idleConns, c)
		this.idleConnsLock.Unlock()

	case http.StateIdle:
		// StateIdle represents a connection that has finished
		// handling a request and is in the keep-alive state, waiting
		// for a new request. Connections transition from StateIdle
		// to either StateActive or StateClosed.
		select {
		case <-this.gw.shutdownCh:
			// actively close the client safely because IO is all done
			c.Close()

		default:
			this.idleConnsLock.Lock()
			this.idleConns[c] = struct{}{}
			this.idleConnsLock.Unlock()
		}

	case http.StateHijacked:
		// websocket steals the socket
		this.idleConnsLock.Lock()
		delete(this.idleConns, c)
		this.idleConnsLock.Unlock()

		if this.gw != nil && !Options.DisableMetrics {
			this.gw.svrMetrics.ConcurrentSub.Dec(1)

			this.gw.svrMetrics.ConcurrentSubWs.Inc(1)
		}

	case http.StateClosed:
		if this.gw != nil && !Options.DisableMetrics {
			this.gw.svrMetrics.ConcurrentSub.Dec(1)
		}

		remoteAddr := c.RemoteAddr().String()
		if Options.EnableClientStats {
			this.gw.clientStates.UnregisterSubClient(remoteAddr)
		}

		this.closedConnCh <- remoteAddr
		this.idleConnsWg.Done()

		this.idleConnsLock.Lock()
		delete(this.idleConns, c)
		this.idleConnsLock.Unlock()
	}
}

func (this *subServer) waitExit(server *http.Server, listener net.Listener, exit <-chan struct{}) {
	<-exit

	// HTTP response will have "Connection: close"
	server.SetKeepAlivesEnabled(false)

	// avoid new connections
	if err := listener.Close(); err != nil {
		log.Error(err.Error())
	}

	this.idleConnsLock.Lock()
	t := time.Now().Add(time.Millisecond * 100)
	for c := range this.idleConns {
		c.SetReadDeadline(t)
	}
	this.idleConnsLock.Unlock()

	log.Trace("%s waiting for all connected client close...", this.name)
	if waitTimeout(&this.idleConnsWg, Options.SubTimeout) {
		log.Warn("%s waiting for all connected client close timeout: %s",
			this.name, Options.SubTimeout)
	}

	this.subMetrics.Flush()

	this.gw.wg.Done()
}

func (this *subServer) ackCommitter() {
	ticker := time.NewTicker(time.Second * 30)
	defer func() {
		ticker.Stop()
		log.Debug("ack committer done")
		this.gw.wg.Done()
	}()

	for {
		select {
		case <-this.gw.shutdownCh:
			this.shutdownOnce.Do(func() {
				atomic.AddInt32(&this.ackShutdown, -1)

				for {
					// waiting for all ack handlers finish
					if atomic.LoadInt32(&this.ackShutdown) <= -1 {
						break
					}

					time.Sleep(time.Millisecond * 10)
				}
				close(this.ackCh)
			})

		case acks, ok := <-this.ackCh:
			if ok {
				for _, ack := range acks {
					if _, present := this.ackedOffsets[ack.cluster]; !present {
						this.ackedOffsets[ack.cluster] = make(map[string]map[string]map[int]int64)
					}
					if _, present := this.ackedOffsets[ack.cluster][ack.topic]; !present {
						this.ackedOffsets[ack.cluster][ack.topic] = make(map[string]map[int]int64)
					}
					if _, present := this.ackedOffsets[ack.topic][ack.group]; !present {
						this.ackedOffsets[ack.cluster][ack.topic][ack.group] = make(map[int]int64)
					}

					// TODO validation
					this.ackedOffsets[ack.cluster][ack.topic][ack.group][ack.Partition] = ack.Offset
				}
			} else {
				// channel buffer drained, flush all offsets
				// zk is still alive, safe to commit offsets
				this.commitOffsets()
				return
			}

		case <-ticker.C:
			this.commitOffsets()
		}
	}

}

func (this *subServer) commitOffsets() {
	for cluster, clusterTopic := range this.ackedOffsets {
		zkcluster := meta.Default.ZkCluster(cluster)

		for topic, groupPartition := range clusterTopic {
			for group, partitionOffset := range groupPartition {
				for partition, offset := range partitionOffset {
					if offset == -1 {
						// this slot is empty
						continue
					}

					log.Debug("commit offset {C:%s T:%s G:%s P:%d O:%d}", cluster, topic, group, partition, offset)

					if err := zkcluster.ResetConsumerGroupOffset(topic, group,
						strconv.Itoa(partition), offset); err != nil {
						log.Error("commitOffsets: %v", err)
					} else {
						// mark this slot empty
						this.ackedOffsets[cluster][topic][group][partition] = -1
					}
				}
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"regexp"
	"strings"
)

type count map[string]uint

func (c *count) RandomKey() string {
	var arr []string
	for key, val := range *c {
		for ; val > 0; val-- {
			arr = append(arr, key)
		}
	}
	if len(arr) == 0 {
		return ""
	}
	if len(arr) == 1 {
		return arr[0]
	}
	return arr[rand.Intn(len(arr)-1)]
}

func NGram(name string, size int) (ret []string) {
	tmp := ""
	for index, chr := range []rune(name) {
		tmp = tmp + string(chr)
		if index > 0 && (index+1)%size == 0 {
			ret = append(ret, tmp)
			tmp = ""
		}
	}
	return
}

func FillGrams(gramList []string, gramMap map[string]count) {
	start := ""
	for _, gram := range gramList {
		if _, ok := gramMap[start]; !ok {
			gramMap[start] = make(count)
		}
		if _, ok := gramMap[start][gram]; !ok {
			gramMap[start][gram] = 0
		}
		gramMap[start][gram] = gramMap[start][gram] + 1
		start = gram
	}
}

func SplitOnVowelGroups(name string) (ret []string) {
	vg := regexp.MustCompile("[AEIOUYaeiouy]+")
	indexes := vg.FindAllStringIndex(name, -1)
	start := 0
	for _, index := range indexes {
		if index[0] > 0 {
			ret = append(ret, name[start:index[1]])
			start = index[1] + 1
		}
	}
	if start < len(name)-1 {
		ret = append(ret, name[start:])
	}
	return
}

func main() {
	gen := 0
	flag.IntVar(&gen, "gen", 0, "generate given number of names")
	write := false
	flag.BoolVar(&write, "w", false, "write out analysis to json files")
	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		return
	}

	file, err := os.Open(flag.Arg(0)) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	twograms := make(map[string]count)
	threegrams := make(map[string]count)
	prefixes := make(count)
	joins := make(count)
	suffixes := make(count)
	vowelgroups := make(map[string]count)

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		name := strings.TrimSpace(scanner.Text())
		if len(name) == 0 {
			continue
		}
		FillGrams(NGram(name, 2), twograms)
		FillGrams(NGram(name, 3), threegrams)
		vgs := SplitOnVowelGroups(name)
		if len(vgs) > 0 {
			FillGrams(vgs, vowelgroups)
			prefix := vgs[0]
			vgs = vgs[1:]

			if _, ok := prefixes[prefix]; !ok {
				prefixes[prefix] = 1
			} else {
				prefixes[prefix] = prefixes[prefix] + 1
			}

			if len(vgs) > 0 {
				suffix := vgs[len(vgs)-1]
				vgs = vgs[:len(vgs)-1]

				if _, ok := suffixes[suffix]; !ok {
					suffixes[suffix] = 1
				} else {
					suffixes[suffix] = suffixes[suffix] + 1
				}

				if len(vgs) > 0 {
					for _, join := range vgs {
						if _, ok := joins[join]; !ok {
							joins[join] = 1
						} else {
							joins[join] = joins[join] + 1
						}
					}
				}
			}
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	if write {
		// output 2-grams
		b, err := json.Marshal(twograms)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("twograms.json", b, 0755)

		// output 3-grams
		b, err = json.Marshal(threegrams)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("threegrams.json", b, 0755)

		// output prefixes
		b, err = json.Marshal(prefixes)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("prefixes.json", b, 0755)

		// output joins
		b, err = json.Marshal(joins)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("joins.json", b, 0755)

		// output suffixes
		b, err = json.Marshal(suffixes)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("suffixes.json", b, 0755)

		// output vowel groups
		b, err = json.Marshal(vowelgroups)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("vowelgroups.json", b, 0755)
	}

	for i := 0; i < gen; i++ {
		fmt.Println(GenerateVowelGroupName(vowelgroups))
	}
}

func GenerateVowelGroupName(vowelgroups map[string]count) (ret string) {
	key := ""
	if val, ok := vowelgroups[ret]; ok {
		key = val.RandomKey()
	}
	ret = ret + key
	if val, ok := vowelgroups[key]; ok {
		key = val.RandomKey()
	} else {
		return
	}
	ret = ret + key
	if val, ok := vowelgroups[key]; ok {
		key = val.RandomKey()
	} else {
		return
	}
	ret = ret + key
	return
}
<commit_msg>Minor cleanups<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"regexp"
	"strings"
)

type count map[string]uint

// RandomKey returns a random key from the count map, weighted by the counts
func (c *count) RandomKey() string {
	var arr []string
	for key, val := range *c {
		for ; val > 0; val-- {
			arr = append(arr, key)
		}
	}
	if len(arr) == 0 {
		return ""
	}
	if len(arr) == 1 {
		return arr[0]
	}
	return arr[rand.Intn(len(arr)-1)]
}

// NGram builds an ngram list of character groupings from a string
func NGram(name string, size int) (ret []string) {
	tmp := ""
	for index, chr := range []rune(name) {
		tmp = tmp + string(chr)
		if index > 0 && (index+1)%size == 0 {
			ret = append(ret, tmp)
			tmp = ""
		}
	}
	return
}

// FillGrams makes a map of which gram leads to another, weighted by occurence
func FillGrams(gramList []string, gramMap map[string]count) {
	start := ""
	for _, gram := range gramList {
		if _, ok := gramMap[start]; !ok {
			gramMap[start] = make(count)
		}
		if _, ok := gramMap[start][gram]; !ok {
			gramMap[start][gram] = 0
		}
		gramMap[start][gram] = gramMap[start][gram] + 1
		start = gram
	}
}

// SplitOnVowelGroups breaks a string into a chunks on the start of every
// contiguous group of vowels
func SplitOnVowelGroups(name string) (ret []string) {
	vg := regexp.MustCompile("[AEIOUYaeiouy]+")
	indexes := vg.FindAllStringIndex(name, -1)
	start := 0
	for _, index := range indexes {
		if index[0] > 0 {
			ret = append(ret, name[start:index[1]])
			start = index[1] + 1
		}
	}
	if start < len(name)-1 {
		ret = append(ret, name[start:])
	}
	return
}

func main() {
	gen := 0
	flag.IntVar(&gen, "gen", 0, "generate given number of names")
	write := false
	flag.BoolVar(&write, "w", false, "write out analysis to json files")
	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		return
	}

	file, err := os.Open(flag.Arg(0)) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	twograms := make(map[string]count)
	threegrams := make(map[string]count)
	prefixes := make(count)
	joins := make(count)
	suffixes := make(count)
	vowelgroups := make(map[string]count)

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		name := strings.TrimSpace(scanner.Text())
		if len(name) == 0 {
			continue
		}
		FillGrams(NGram(name, 2), twograms)
		FillGrams(NGram(name, 3), threegrams)
		vgs := SplitOnVowelGroups(name)
		if len(vgs) > 0 {
			FillGrams(vgs, vowelgroups)
			prefix := vgs[0]
			vgs = vgs[1:]

			if _, ok := prefixes[prefix]; !ok {
				prefixes[prefix] = 1
			} else {
				prefixes[prefix] = prefixes[prefix] + 1
			}

			if len(vgs) > 0 {
				suffix := vgs[len(vgs)-1]
				vgs = vgs[:len(vgs)-1]

				if _, ok := suffixes[suffix]; !ok {
					suffixes[suffix] = 1
				} else {
					suffixes[suffix] = suffixes[suffix] + 1
				}

				if len(vgs) > 0 {
					for _, join := range vgs {
						if _, ok := joins[join]; !ok {
							joins[join] = 1
						} else {
							joins[join] = joins[join] + 1
						}
					}
				}
			}
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	if write {
		// output 2-grams
		b, err := json.Marshal(twograms)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("twograms.json", b, 0755)

		// output 3-grams
		b, err = json.Marshal(threegrams)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("threegrams.json", b, 0755)

		// output prefixes
		b, err = json.Marshal(prefixes)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("prefixes.json", b, 0755)

		// output joins
		b, err = json.Marshal(joins)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("joins.json", b, 0755)

		// output suffixes
		b, err = json.Marshal(suffixes)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("suffixes.json", b, 0755)

		// output vowel groups
		b, err = json.Marshal(vowelgroups)
		if err != nil {
			fmt.Println("error:", err)
		}
		ioutil.WriteFile("vowelgroups.json", b, 0755)
	}

	for i := 0; i < gen; i++ {
		fmt.Println(GenerateVowelGroupName(vowelgroups))
	}
}

// GenerateVowelGroupName makes a name by traversing the vowelgroup randomly.
// It limits the traversal to a maximum of 3 steps and returns immediately on a
// dead end.
func GenerateVowelGroupName(vowelgroups map[string]count) (ret string) {
	key := ""
	if val, ok := vowelgroups[ret]; ok {
		key = val.RandomKey()
	}
	ret = ret + key
	if val, ok := vowelgroups[key]; ok {
		key = val.RandomKey()
	} else {
		return
	}
	ret = ret + key
	if val, ok := vowelgroups[key]; ok {
		key = val.RandomKey()
	} else {
		return
	}
	ret = ret + key
	return
}
<|endoftext|>