<commit_before>package main

import (
	"fmt"
	"os/exec"
	"time"

	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v", key)
		if job.Job.NonBootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): %v", job.CommandKey, output)
		if len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		output := s.scheduler.getOutput(job.CommandKey)
		if s.taskComplete(job.CommandKey) {
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.RaiseIssue(ctx, fmt.Sprintf("%v Died", job.Job.Name), fmt.Sprintf("Job %v has died for reals %v", job.Job.Name, output), false)
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		if s.discover.discover(job.Job.Name, s.Registry.Identifier) != nil {
			s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running", job.Job.Name, s.Registry.Identifier), false)
		}

		// Restart this job if we need to
		if job.Job.NonBootstrap {
			version := s.getVersion(ctx, job.Job)
			if version != job.RunningVersion {
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		s.scheduler.removeJob(job.CommandKey)
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) string {
	versions, _ := s.builder.build(ctx, job)

	if len(versions) == 0 {
		return ""
	}

	return versions[0].Version

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.NonBootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if !job.NonBootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	if len(versions) == 0 {
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		return ""
	}

	t := time.Now()
	err = s.builder.copy(ctx, versions[0])
	if err != nil {
		s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
		return ""
	}
	s.stateMap[job.Name] = fmt.Sprintf("Found version %v", versions[0].Version)
	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<commit_msg>Add died<commit_after>package main

import (
	"fmt"
	"os/exec"
	"time"

	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v", key)
		if job.Job.NonBootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): %v", job.CommandKey, output)
		if len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		output := s.scheduler.getOutput(job.CommandKey)
		if s.taskComplete(job.CommandKey) {
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.RaiseIssue(ctx, fmt.Sprintf("%v Died", job.Job.Name), fmt.Sprintf("Job %v has died for reals %v", job.Job.Name, output), false)
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		if s.discover.discover(job.Job.Name, s.Registry.Identifier) != nil {
			s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running", job.Job.Name, s.Registry.Identifier), false)
		}

		// Restart this job if we need to
		if job.Job.NonBootstrap {
			version := s.getVersion(ctx, job.Job)
			if version != job.RunningVersion {
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.scheduler.removeJob(job.CommandKey)
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) string {
	versions, _ := s.builder.build(ctx, job)

	if len(versions) == 0 {
		return ""
	}

	return versions[0].Version

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.NonBootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if !job.NonBootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	if len(versions) == 0 {
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		return ""
	}

	t := time.Now()
	err = s.builder.copy(ctx, versions[0])
	if err != nil {
		s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
		return ""
	}
	s.stateMap[job.Name] = fmt.Sprintf("Found version %v", versions[0].Version)
	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<|endoftext|>


<commit_before>/*
Copyright 2013 Tamás Gulácsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package godrv

import (
	"database/sql"
	"flag"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	"github.com/tgulacsi/goracle/oracle"
)

var fDsn = flag.String("dsn", "", "Oracle DSN")

func TestNull(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()
	var (
		err error
		str string
		dt  time.Time
	)
	qry := "SELECT '' FROM DUAL"
	row := conn.QueryRow(qry)
	if err = row.Scan(&str); err != nil {
		t.Errorf("0. error with %q test: %s", qry, err)
	}
	t.Logf("0. %q result: %#v (%T)", qry, str, str)

	qry = "SELECT TO_DATE(NULL) FROM DUAL"
	row = conn.QueryRow(qry)
	if err = row.Scan(&dt); err != nil {
		t.Errorf("0. error with %q test: %s", qry, err)
	}
	t.Logf("1. %q result: %#v (%T)", qry, dt, dt)
}

func TestSimple(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()

	var (
		err error
		dst interface{}
	)
	for i, qry := range []string{
		"SELECT ROWNUM FROM DUAL",
		"SELECT 1234567890 FROM DUAL",
		"SELECT LOG(10, 2) FROM DUAL",
		"SELECT 'árvíztűrő tükörfúrógép' FROM DUAL",
		"SELECT HEXTORAW('00') FROM DUAL",
		"SELECT TO_DATE('2006-05-04 15:07:08', 'YYYY-MM-DD HH24:MI:SS') FROM DUAL",
		"SELECT NULL FROM DUAL",
		"SELECT TO_CLOB('árvíztűrő tükörfúrógép') FROM DUAL",
	} {
		row := conn.QueryRow(qry)
		if err = row.Scan(&dst); err != nil {
			t.Errorf("%d. error with %q test: %s", i, qry, err)
		}
		t.Logf("%d. %q result: %#v", i, qry, dst)
		if strings.Index(qry, " TO_CLOB(") >= 0 {
			var b []byte
			var e error
			if true {
				r := dst.(io.Reader)
				b, e = ioutil.ReadAll(r)
			} else {
				clob := dst.(*oracle.ExternalLobVar)
				b, e = clob.ReadAll()
			}
			if e != nil {
				t.Errorf("error reading clob (%v): %s", dst, e)
			} else {
				t.Logf("clob=%s", b)
			}
		}
	}

	qry := "SELECT rn, CHR(rn) FROM (SELECT ROWNUM rn FROM all_objects WHERE ROWNUM < 256)"
	rows, err := conn.Query(qry)
	if err != nil {
		t.Errorf("error with multirow test, query %q: %s", qry, err)
	}
	defer rows.Close()
	cols, err := rows.Columns()
	if err != nil {
		t.Errorf("error getting columns for %q: %s", qry, err)
	}
	t.Logf("columns for %q: %v", qry, cols)
	var (
		num int
		str string
	)
	for rows.Next() {
		if err = rows.Scan(&num, &str); err != nil {
			t.Errorf("error scanning row: %s", err)
		}
		//t.Logf("%d=%q", num, str)
	}
}

func TestNumber(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()

	oldDebug := IsDebug
	IsDebug = true
	if !oldDebug {
		defer func() { IsDebug = false }()
	}

	var (
		err, errF error
		into      int64
		intoF     float64
	)
	for i, tst := range []struct {
		in   string
		want int64
	}{
		{"1", 1},
		{"1234567890", 1234567890},
	} {
		into, intoF, errF = 0, 0, nil
		qry := "SELECT " + tst.in + " FROM DUAL"
		row := conn.QueryRow(qry)
		if err = row.Scan(&into); err != nil {
			row = conn.QueryRow(qry)
			if errF = row.Scan(&intoF); errF != nil {
				t.Errorf("%d. error with %q testF: %s", i, qry, err)
				continue
			}
			t.Logf("%d. %q result: %#v", i, qry, intoF)
			if intoF != float64(tst.want) {
				t.Errorf("%d. got %#v want %#v", i, intoF, float64(tst.want))
			}
			continue
		}
		t.Logf("%d. %q result: %#v", i, qry, into)
		if into != tst.want {
			t.Errorf("%d. got %#v want %#v", i, into, tst.want)
		}
	}
}

// TestClob
func TestClob(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()
}

func TestPrepared(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()
	stmt, err := conn.Prepare("SELECT ? FROM DUAL")
	if err != nil {
		t.Errorf("error preparing query: %v", err)
		t.FailNow()
	}
	rows, err := stmt.Query("a")
	if err != nil {
		t.Errorf("error executing query: %s", err)
		t.FailNow()
	}
	defer rows.Close()
}

func TestNULL(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()

	rows, err := conn.Query(`
		SELECT dt
		  FROM (SELECT TO_DATE('', 'YYYY-MM-DD') dt FROM DUAL
				UNION ALL SELECT SYSDATE FROM DUAL
				UNION ALL SELECT NULL FROM DUAL)`)
	if err != nil {
		t.Errorf("error executing the query: %v", err)
		t.FailNow()
	}
	var dt time.Time
	i := 0
	for rows.Next() {
		if err = rows.Scan(&dt); err != nil {
			t.Errorf("error fetching row %d: %v", i+1, err)
			break
		}
		if i == 1 && dt.IsZero() {
			t.Errorf("second row is zero: %#v", dt)
		}
		if i != 1 && !dt.IsZero() {
			t.Errorf("other row is not zero: %#v", dt)
		}
		i++
	}
}

var testDB *sql.DB

func getConnection(t *testing.T) *sql.DB {
	var err error
	if testDB != nil && testDB.Ping() == nil {
		return testDB
	}
	flag.Parse()
	if testDB, err = sql.Open("goracle", *fDsn); err != nil {
		t.Fatalf("error connecting to %q: %s", *fDsn, err)
	}
	return testDB
}
<commit_msg>add test for "ORA-01458: invalid length inside variable character string"<commit_after>/*
Copyright 2013 Tamás Gulácsi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package godrv

import (
	"database/sql"
	"flag"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"

	"github.com/tgulacsi/goracle/oracle"
)

var fDsn = flag.String("dsn", "", "Oracle DSN")

func TestNull(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()
	var (
		err error
		str string
		dt  time.Time
	)
	qry := "SELECT '' FROM DUAL"
	row := conn.QueryRow(qry)
	if err = row.Scan(&str); err != nil {
		t.Errorf("0. error with %q test: %s", qry, err)
	}
	t.Logf("0. %q result: %#v (%T)", qry, str, str)

	qry = "SELECT TO_DATE(NULL) FROM DUAL"
	row = conn.QueryRow(qry)
	if err = row.Scan(&dt); err != nil {
		t.Errorf("0. error with %q test: %s", qry, err)
	}
	t.Logf("1. %q result: %#v (%T)", qry, dt, dt)
}

func TestSimple(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()

	var (
		err error
		dst interface{}
	)
	for i, qry := range []string{
		"SELECT ROWNUM FROM DUAL",
		"SELECT 1234567890 FROM DUAL",
		"SELECT LOG(10, 2) FROM DUAL",
		"SELECT 'árvíztűrő tükörfúrógép' FROM DUAL",
		"SELECT HEXTORAW('00') FROM DUAL",
		"SELECT TO_DATE('2006-05-04 15:07:08', 'YYYY-MM-DD HH24:MI:SS') FROM DUAL",
		"SELECT NULL FROM DUAL",
		"SELECT TO_CLOB('árvíztűrő tükörfúrógép') FROM DUAL",
	} {
		row := conn.QueryRow(qry)
		if err = row.Scan(&dst); err != nil {
			t.Errorf("%d. error with %q test: %s", i, qry, err)
		}
		t.Logf("%d. %q result: %#v", i, qry, dst)
		if strings.Index(qry, " TO_CLOB(") >= 0 {
			var b []byte
			var e error
			if true {
				r := dst.(io.Reader)
				b, e = ioutil.ReadAll(r)
			} else {
				clob := dst.(*oracle.ExternalLobVar)
				b, e = clob.ReadAll()
			}
			if e != nil {
				t.Errorf("error reading clob (%v): %s", dst, e)
			} else {
				t.Logf("clob=%s", b)
			}
		}
	}

	qry := "SELECT rn, CHR(rn) FROM (SELECT ROWNUM rn FROM all_objects WHERE ROWNUM < 256)"
	rows, err := conn.Query(qry)
	if err != nil {
		t.Errorf("error with multirow test, query %q: %s", qry, err)
	}
	defer rows.Close()
	cols, err := rows.Columns()
	if err != nil {
		t.Errorf("error getting columns for %q: %s", qry, err)
	}
	t.Logf("columns for %q: %v", qry, cols)
	var (
		num int
		str string
	)
	for rows.Next() {
		if err = rows.Scan(&num, &str); err != nil {
			t.Errorf("error scanning row: %s", err)
		}
		//t.Logf("%d=%q", num, str)
	}
}

func TestNumber(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()

	oldDebug := IsDebug
	IsDebug = true
	if !oldDebug {
		defer func() { IsDebug = false }()
	}

	var (
		err, errF error
		into      int64
		intoF     float64
	)
	for i, tst := range []struct {
		in   string
		want int64
	}{
		{"1", 1},
		{"1234567890", 1234567890},
	} {
		into, intoF, errF = 0, 0, nil
		qry := "SELECT " + tst.in + " FROM DUAL"
		row := conn.QueryRow(qry)
		if err = row.Scan(&into); err != nil {
			row = conn.QueryRow(qry)
			if errF = row.Scan(&intoF); errF != nil {
				t.Errorf("%d. error with %q testF: %s", i, qry, err)
				continue
			}
			t.Logf("%d. %q result: %#v", i, qry, intoF)
			if intoF != float64(tst.want) {
				t.Errorf("%d. got %#v want %#v", i, intoF, float64(tst.want))
			}
			continue
		}
		t.Logf("%d. %q result: %#v", i, qry, into)
		if into != tst.want {
			t.Errorf("%d. got %#v want %#v", i, into, tst.want)
		}
	}
}

func TestSelectBind(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()

	tbl := `(SELECT 1 id FROM DUAL
                           UNION ALL SELECT 2 FROM DUAL
                           UNION ALL SELECT 1234567890123 FROM DUAL)`

	qry := "SELECT * FROM " + tbl
rows, err := conn.Query(qry)
	if err != nil {
		t.Errorf("get all rows: %v", err)
		return
	}

		var id int64
	i := 1
	for rows.Next() {
		if err = rows.Scan(&id); err != nil {
			t.Errorf("%d. error: %v", err)
		}
		t.Logf("%d. %d", i, id)
		i++
	}
	if err = rows.Err(); err != nil {
		t.Errorf("rows error: %v", err)
	}

	qry = "SELECT id FROM " + tbl + " WHERE id = :1"
	if err = conn.QueryRow(qry, 1234567890123).Scan(&id); err != nil {
		t.Errorf("bind: %v", err)
		return
	}
	t.Logf("bind: %d", id)
}

// TestClob
func TestClob(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()
}

func TestPrepared(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()
	stmt, err := conn.Prepare("SELECT ? FROM DUAL")
	if err != nil {
		t.Errorf("error preparing query: %v", err)
		t.FailNow()
	}
	rows, err := stmt.Query("a")
	if err != nil {
		t.Errorf("error executing query: %s", err)
		t.FailNow()
	}
	defer rows.Close()
}

func TestNULL(t *testing.T) {
	conn := getConnection(t)
	defer conn.Close()

	rows, err := conn.Query(`
		SELECT dt
		  FROM (SELECT TO_DATE('', 'YYYY-MM-DD') dt FROM DUAL
				UNION ALL SELECT SYSDATE FROM DUAL
				UNION ALL SELECT NULL FROM DUAL)`)
	if err != nil {
		t.Errorf("error executing the query: %v", err)
		t.FailNow()
	}
	var dt time.Time
	i := 0
	for rows.Next() {
		if err = rows.Scan(&dt); err != nil {
			t.Errorf("error fetching row %d: %v", i+1, err)
			break
		}
		if i == 1 && dt.IsZero() {
			t.Errorf("second row is zero: %#v", dt)
		}
		if i != 1 && !dt.IsZero() {
			t.Errorf("other row is not zero: %#v", dt)
		}
		i++
	}
}

var testDB *sql.DB

func getConnection(t *testing.T) *sql.DB {
	var err error
	if testDB != nil && testDB.Ping() == nil {
		return testDB
	}
	flag.Parse()
	if testDB, err = sql.Open("goracle", *fDsn); err != nil {
		t.Fatalf("error connecting to %q: %s", *fDsn, err)
	}
	return testDB
}
<|endoftext|>


<commit_before>package twitter

import (
	"net/http"

	"github.com/dghubble/sling"
)

// FollowerIDs is a cursored collection of follower ids.
type FollowerIDs struct {
	IDs               []int64 `json:"ids"`
	NextCursor        int64   `json:"next_cursor"`
	NextCursorStr     string  `json:"next_cursor_str"`
	PreviousCursor    int64   `json:"previous_cursor"`
	PreviousCursorStr string  `json:"previous_cursor_str"`
}

// Followers is a cursored collection of followers.
type Followers struct {
	Users             []User `json:"users"`
	NextCursor        int64  `json:"next_cursor"`
	NextCursorStr     string `json:"next_cursor_str"`
	PreviousCursor    int64  `json:"previous_cursor"`
	PreviousCursorStr string `json:"previous_cursor_str"`
}

// FollowerService provides methods for accessing Twitter followers endpoints.
type FollowerService struct {
	sling *sling.Sling
}

// newFollowerService returns a new FollowerService.
func newFollowerService(sling *sling.Sling) *FollowerService {
	return &FollowerService{
		sling: sling.Path("followers/"),
	}
}

// FollowerIDParams are the parameters for FollowerService.Ids
type FollowerIDParams struct {
	UserID     int64  `url:"user_id,omitempty"`
	ScreenName string `url:"screen_name,omitempty"`
	Cursor     int64  `url:"cursor,omitempty"`
	Count      int    `url:"count,omitempty"`
}

// IDs returns a cursored collection of user ids following the specified user.
// https://dev.twitter.com/rest/reference/get/followers/ids
func (s *FollowerService) IDs(params *FollowerIDParams) (*FollowerIDs, *http.Response, error) {
	ids := new(FollowerIDs)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("ids.json").QueryStruct(params).Receive(ids, apiError)
	return ids, resp, relevantError(err, *apiError)
}

// FollowerListParams are the parameters for FollowerService.List
type FollowerListParams struct {
	UserID              int64  `url:"user_id,omitempty"`
	ScreenName          string `url:"screen_name,omitempty"`
	Cursor              int    `url:"cursor,omitempty"`
	Count               int    `url:"count,omitempty"`
	SkipStatus          *bool  `url:"skip_status,omitempty"`
	IncludeUserEntities *bool  `url:"include_user_entities,omitempty"`
}

// List returns a cursored collection of Users following the specified user.
// https://dev.twitter.com/rest/reference/get/followers/list
func (s *FollowerService) List(params *FollowerListParams) (*Followers, *http.Response, error) {
	followers := new(Followers)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("list.json").QueryStruct(params).Receive(followers, apiError)
	return followers, resp, relevantError(err, *apiError)
}
<commit_msg>Cursor should be of the type int64<commit_after>package twitter

import (
	"net/http"

	"github.com/dghubble/sling"
)

// FollowerIDs is a cursored collection of follower ids.
type FollowerIDs struct {
	IDs               []int64 `json:"ids"`
	NextCursor        int64   `json:"next_cursor"`
	NextCursorStr     string  `json:"next_cursor_str"`
	PreviousCursor    int64   `json:"previous_cursor"`
	PreviousCursorStr string  `json:"previous_cursor_str"`
}

// Followers is a cursored collection of followers.
type Followers struct {
	Users             []User `json:"users"`
	NextCursor        int64  `json:"next_cursor"`
	NextCursorStr     string `json:"next_cursor_str"`
	PreviousCursor    int64  `json:"previous_cursor"`
	PreviousCursorStr string `json:"previous_cursor_str"`
}

// FollowerService provides methods for accessing Twitter followers endpoints.
type FollowerService struct {
	sling *sling.Sling
}

// newFollowerService returns a new FollowerService.
func newFollowerService(sling *sling.Sling) *FollowerService {
	return &FollowerService{
		sling: sling.Path("followers/"),
	}
}

// FollowerIDParams are the parameters for FollowerService.Ids
type FollowerIDParams struct {
	UserID     int64  `url:"user_id,omitempty"`
	ScreenName string `url:"screen_name,omitempty"`
	Cursor     int64  `url:"cursor,omitempty"`
	Count      int    `url:"count,omitempty"`
}

// IDs returns a cursored collection of user ids following the specified user.
// https://dev.twitter.com/rest/reference/get/followers/ids
func (s *FollowerService) IDs(params *FollowerIDParams) (*FollowerIDs, *http.Response, error) {
	ids := new(FollowerIDs)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("ids.json").QueryStruct(params).Receive(ids, apiError)
	return ids, resp, relevantError(err, *apiError)
}

// FollowerListParams are the parameters for FollowerService.List
type FollowerListParams struct {
	UserID              int64  `url:"user_id,omitempty"`
	ScreenName          string `url:"screen_name,omitempty"`
	Cursor              int64  `url:"cursor,omitempty"`
	Count               int    `url:"count,omitempty"`
	SkipStatus          *bool  `url:"skip_status,omitempty"`
	IncludeUserEntities *bool  `url:"include_user_entities,omitempty"`
}

// List returns a cursored collection of Users following the specified user.
// https://dev.twitter.com/rest/reference/get/followers/list
func (s *FollowerService) List(params *FollowerListParams) (*Followers, *http.Response, error) {
	followers := new(Followers)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("list.json").QueryStruct(params).Receive(followers, apiError)
	return followers, resp, relevantError(err, *apiError)
}
<|endoftext|>


<commit_before>package shippers

import "fmt"
import "net"
import "net/url"
import "strings"
import "time"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/Sirupsen/logrus"
import "github.com/vaughan0/go-ini"

type GraphiteShipper struct {
	debug   bool
	enabled bool
	host    string
	prefix  string
	port    string
}

func (this *GraphiteShipper) Enabled() bool {
	return this.enabled
}

func (this *GraphiteShipper) State(state bool) {
	this.enabled = state
}

func (this *GraphiteShipper) Setup(conf ini.File) {
	this.State(true)

	useDebug, ok := conf.Get("GraphiteShipper", "debug")
	if ok && useDebug == "true" {
		this.debug = true
	} else {
		this.debug = false
	}

	this.host = "127.0.0.1"
	this.port = "2003"
	useGraphiteUrl, ok := conf.Get("GraphiteShipper", "url")
	if ok {
		graphiteUrl, err := url.Parse(useGraphiteUrl)
		if err == nil {
			splitted := strings.Split(graphiteUrl.Host, ":")
			this.host, this.port = splitted[0], "2003"
			switch {
			case len(splitted) > 2:
				logrus.Warning("error parsing graphite url")
				logrus.Warning("using default 127.0.0.1:2003 for graphite url")
			case len(splitted) > 1:
				this.host, this.port = splitted[0], splitted[1]
			default:
				this.host, this.port = splitted[0], "2003"
			}
		} else {
			logrus.Warning("error parsing graphite url: %s", err)
			logrus.Warning("using default 127.0.0.1:2003 for graphite url")
		}
	}

	usePrefix, ok := conf.Get("GraphiteShipper", "prefix")
	if ok {
		this.prefix = fmt.Sprintf("%s.", usePrefix)
	}
}

func (this *GraphiteShipper) Ship(logs structs.MetricSlice) error {
	con, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", this.host, this.port), 1*time.Second)
	if err != nil {
		logrus.Warning("connecting to graphite failed with err: ", err)
		return err
	}
	defer con.Close()

	for _, item := range logs {
		serialized := item.ToGraphite(this.prefix)
		if this.debug {
			fmt.Printf("%s\n", serialized)
		}
		fmt.Fprintln(con, serialized)
	}

	return nil
}
<commit_msg>Alias import so go-plus doesn't strip it<commit_after>package shippers

import "fmt"
import "net"
import "net/url"
import "strings"
import "time"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/Sirupsen/logrus"
import ini "github.com/vaughan0/go-ini"

type GraphiteShipper struct {
	debug   bool
	enabled bool
	host    string
	prefix  string
	port    string
}

func (this *GraphiteShipper) Enabled() bool {
	return this.enabled
}

func (this *GraphiteShipper) State(state bool) {
	this.enabled = state
}

func (this *GraphiteShipper) Setup(conf ini.File) {
	this.State(true)

	useDebug, ok := conf.Get("GraphiteShipper", "debug")
	if ok && useDebug == "true" {
		this.debug = true
	} else {
		this.debug = false
	}

	this.host = "127.0.0.1"
	this.port = "2003"
	useGraphiteUrl, ok := conf.Get("GraphiteShipper", "url")
	if ok {
		graphiteUrl, err := url.Parse(useGraphiteUrl)
		if err == nil {
			splitted := strings.Split(graphiteUrl.Host, ":")
			this.host, this.port = splitted[0], "2003"
			switch {
			case len(splitted) > 2:
				logrus.Warning("error parsing graphite url")
				logrus.Warning("using default 127.0.0.1:2003 for graphite url")
			case len(splitted) > 1:
				this.host, this.port = splitted[0], splitted[1]
			default:
				this.host, this.port = splitted[0], "2003"
			}
		} else {
			logrus.Warning("error parsing graphite url: %s", err)
			logrus.Warning("using default 127.0.0.1:2003 for graphite url")
		}
	}

	usePrefix, ok := conf.Get("GraphiteShipper", "prefix")
	if ok {
		this.prefix = fmt.Sprintf("%s.", usePrefix)
	}
}

func (this *GraphiteShipper) Ship(logs structs.MetricSlice) error {
	con, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", this.host, this.port), 1*time.Second)
	if err != nil {
		logrus.Warning("connecting to graphite failed with err: ", err)
		return err
	}
	defer con.Close()

	for _, item := range logs {
		serialized := item.ToGraphite(this.prefix)
		if this.debug {
			fmt.Printf("%s\n", serialized)
		}
		fmt.Fprintln(con, serialized)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wptdashboard

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/deckarep/golang-set"
	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/urlfetch"
)

type platformAtRevision struct {
	// Platform is the string representing browser (+ version), and OS (+ version).
	Platform string

	// Revision is the SHA[0:10] of the git repo.
	Revision string
}

func parsePlatformAtRevisionSpec(spec string) (platformAtRevision platformAtRevision, err error) {
	pieces := strings.Split(spec, "@")
	if len(pieces) > 2 {
		return platformAtRevision, errors.New("invalid platform@revision spec: " + spec)
	}
	platformAtRevision.Platform = pieces[0]
	if len(pieces) < 2 {
		// No @ is assumed to be the platform only.
		platformAtRevision.Revision = "latest"
	} else {
		platformAtRevision.Revision = pieces[1]
	}
	// TODO(lukebjerring): Also handle actual platforms (with version + os)
	if IsBrowserName(platformAtRevision.Platform) {
		return platformAtRevision, nil
	}
	return platformAtRevision, errors.New("Platform " + platformAtRevision.Platform + " not found")
}

func fetchRunResultsJSONForParam(
	ctx context.Context, r *http.Request, revision string) (results map[string][]int, err error) {
	var spec platformAtRevision
	if spec, err = parsePlatformAtRevisionSpec(revision); err != nil {
		return nil, err
	}
	return fetchRunResultsJSONForSpec(ctx, r, spec)
}

func fetchRunResultsJSONForSpec(
	ctx context.Context, r *http.Request, revision platformAtRevision) (results map[string][]int, err error) {
	var run TestRun
	if run, err = fetchRunForSpec(ctx, revision); err != nil {
		return nil, err
	} else if (run == TestRun{}) {
		return nil, nil
	}
	return fetchRunResultsJSON(ctx, r, run)
}

func fetchRunForSpec(ctx context.Context, revision platformAtRevision) (TestRun, error) {
	baseQuery := datastore.
		NewQuery("TestRun").
		Order("-CreatedAt").
		Limit(1)

	var results []TestRun
	// TODO(lukebjerring): Handle actual platforms (split out version + os)
	query := baseQuery.
		Filter("BrowserName =", revision.Platform)
	if revision.Revision != "latest" {
		query = query.Filter("Revision = ", revision.Revision)
	}
	if _, err := query.GetAll(ctx, &results); err != nil {
		return TestRun{}, err
	}
	if len(results) < 1 {
		return TestRun{}, nil
	}
	return results[0], nil
}

// fetchRunResultsJSON fetches the results JSON summary for the given test run, but does not include subtests (since
// a full run can span 20k files).
func fetchRunResultsJSON(ctx context.Context, r *http.Request, run TestRun) (results map[string][]int, err error) {
	client := urlfetch.Client(ctx)
	url := run.ResultsURL
	if strings.Index(url, "/") == 0 {
		reqURL := *r.URL
		reqURL.Path = url
	}
	var resp *http.Response
	if resp, err = client.Get(url); err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("%s returned HTTP status %d", url, resp.StatusCode)
	}
	var body []byte
	if body, err = ioutil.ReadAll(resp.Body); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(body, &results); err != nil {
		return nil, err
	}
	return results, nil
}

// getResultsDiff returns a map of test name to an array of [count-different-tests, total-tests], for tests which had
// different results counts in their map (which is test name to array of [count-passed, total-tests]).
//
func getResultsDiff(before map[string][]int, after map[string][]int, filter DiffFilterParam) map[string][]int {
	diff := make(map[string][]int)
	if filter.Deleted || filter.Changed {
		for test, resultsBefore := range before {
			if !anyPathMatches(filter.Paths, test) {
				continue
			}

			if resultsAfter, ok := after[test]; !ok {
				// Missing? Then N / N tests are 'different'.
				if !filter.Deleted {
					continue
				}
				diff[test] = []int{resultsBefore[1], resultsBefore[1]}
			} else {
				if !filter.Changed {
					continue
				}
				passDiff := abs(resultsBefore[0] - resultsAfter[0])
				countDiff := abs(resultsBefore[1] - resultsAfter[1])
				if countDiff == 0 && passDiff == 0 {
					continue
				}
				// Changed tests is at most the number of different outcomes,
				// but newly introduced tests should still be counted (e.g. 0/2 => 0/5)
				diff[test] = []int{
					max(passDiff, countDiff),
					max(resultsBefore[1], resultsAfter[1]),
				}
			}
		}
	}
	if filter.Added {
		for test, resultsAfter := range after {
			if !anyPathMatches(filter.Paths, test) {
				continue
			}

			if _, ok := before[test]; !ok {
				// Missing? Then N / N tests are 'different'
				diff[test] = []int{resultsAfter[1], resultsAfter[1]}
			}
		}
	}
	return diff
}

func anyPathMatches(paths mapset.Set, testPath string) bool {
	if paths == nil {
		return true
	}
	for path := range paths.Iter() {
		if strings.Index(testPath, path.(string)) == 0 {
			return true
		}
	}
	return false
}
<commit_msg>Improve error message and handle trailing newline<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wptdashboard

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/deckarep/golang-set"
	"golang.org/x/net/context"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/urlfetch"
)

type platformAtRevision struct {
	// Platform is the string representing browser (+ version), and OS (+ version).
	Platform string

	// Revision is the SHA[0:10] of the git repo.
	Revision string
}

func parsePlatformAtRevisionSpec(spec string) (platformAtRevision platformAtRevision, err error) {
	pieces := strings.Split(spec, "@")
	if len(pieces) > 2 {
		return platformAtRevision, errors.New("invalid platform@revision spec: " + spec)
	}
	platformAtRevision.Platform = pieces[0]
	if len(pieces) < 2 {
		// No @ is assumed to be the platform only.
		platformAtRevision.Revision = "latest"
	} else {
		platformAtRevision.Revision = pieces[1]
	}
	// TODO(lukebjerring): Also handle actual platforms (with version + os)
	if IsBrowserName(platformAtRevision.Platform) {
		return platformAtRevision, nil
	}
	return platformAtRevision, errors.New("Platform " + platformAtRevision.Platform + " not found")
}

func fetchRunResultsJSONForParam(
	ctx context.Context, r *http.Request, revision string) (results map[string][]int, err error) {
	var spec platformAtRevision
	if spec, err = parsePlatformAtRevisionSpec(revision); err != nil {
		return nil, err
	}
	return fetchRunResultsJSONForSpec(ctx, r, spec)
}

func fetchRunResultsJSONForSpec(
	ctx context.Context, r *http.Request, revision platformAtRevision) (results map[string][]int, err error) {
	var run TestRun
	if run, err = fetchRunForSpec(ctx, revision); err != nil {
		return nil, err
	} else if (run == TestRun{}) {
		return nil, nil
	}
	return fetchRunResultsJSON(ctx, r, run)
}

func fetchRunForSpec(ctx context.Context, revision platformAtRevision) (TestRun, error) {
	baseQuery := datastore.
		NewQuery("TestRun").
		Order("-CreatedAt").
		Limit(1)

	var results []TestRun
	// TODO(lukebjerring): Handle actual platforms (split out version + os)
	query := baseQuery.
		Filter("BrowserName =", revision.Platform)
	if revision.Revision != "latest" {
		query = query.Filter("Revision = ", revision.Revision)
	}
	if _, err := query.GetAll(ctx, &results); err != nil {
		return TestRun{}, err
	}
	if len(results) < 1 {
		return TestRun{}, nil
	}
	return results[0], nil
}

// fetchRunResultsJSON fetches the results JSON summary for the given test run, but does not include subtests (since
// a full run can span 20k files).
func fetchRunResultsJSON(ctx context.Context, r *http.Request, run TestRun) (results map[string][]int, err error) {
	client := urlfetch.Client(ctx)
	url := strings.TrimSpace(run.ResultsURL)
	if strings.Index(url, "/") == 0 {
		reqURL := *r.URL
		reqURL.Path = url
	}
	var resp *http.Response
	if resp, err = client.Get(url); err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var body []byte
	if body, err = ioutil.ReadAll(resp.Body); err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("%s returned HTTP status %d:\n%s", url, resp.StatusCode, string(body))
	}
	if err = json.Unmarshal(body, &results); err != nil {
		return nil, err
	}
	return results, nil
}

// getResultsDiff returns a map of test name to an array of [count-different-tests, total-tests], for tests which had
// different results counts in their map (which is test name to array of [count-passed, total-tests]).
//
func getResultsDiff(before map[string][]int, after map[string][]int, filter DiffFilterParam) map[string][]int {
	diff := make(map[string][]int)
	if filter.Deleted || filter.Changed {
		for test, resultsBefore := range before {
			if !anyPathMatches(filter.Paths, test) {
				continue
			}

			if resultsAfter, ok := after[test]; !ok {
				// Missing? Then N / N tests are 'different'.
				if !filter.Deleted {
					continue
				}
				diff[test] = []int{resultsBefore[1], resultsBefore[1]}
			} else {
				if !filter.Changed {
					continue
				}
				passDiff := abs(resultsBefore[0] - resultsAfter[0])
				countDiff := abs(resultsBefore[1] - resultsAfter[1])
				if countDiff == 0 && passDiff == 0 {
					continue
				}
				// Changed tests is at most the number of different outcomes,
				// but newly introduced tests should still be counted (e.g. 0/2 => 0/5)
				diff[test] = []int{
					max(passDiff, countDiff),
					max(resultsBefore[1], resultsAfter[1]),
				}
			}
		}
	}
	if filter.Added {
		for test, resultsAfter := range after {
			if !anyPathMatches(filter.Paths, test) {
				continue
			}

			if _, ok := before[test]; !ok {
				// Missing? Then N / N tests are 'different'
				diff[test] = []int{resultsAfter[1], resultsAfter[1]}
			}
		}
	}
	return diff
}

func anyPathMatches(paths mapset.Set, testPath string) bool {
	if paths == nil {
		return true
	}
	for path := range paths.Iter() {
		if strings.Index(testPath, path.(string)) == 0 {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
)

type Charset struct {
	From, To rune
}

type Counter interface {
	ReadAll(in io.Reader)                // 读取全部
	Output(out io.Writer, mutiline bool) // 输出
	AllCount() int64                     // 全部字符数量
	Counted() int                        // 计算进的数量
}

// 不排序字符统计器
type NormalCounter struct {
	count int64

	m  map[rune]int
	ma []rune
}

func NewNormalCounter() *NormalCounter {
	return &NormalCounter{
		m:  make(map[rune]int),
		ma: make([]rune, 0, 0x9fa5-0x4e00),
	}
}
func (this *NormalCounter) ReadAll(in io.Reader) {
	r := bufio.NewReader(in)
	for {
		ru, _, err := r.ReadRune()
		if err != nil {
			break
		}
		if _, ok := this.m[ru]; ru >= 0x4e00 && ru <= 0x9fa5 {
			if !ok {
				this.m[ru] = 0
				this.ma = append(this.ma, ru)
			}
			this.count++
			this.m[ru]++
		}
	}
}
func (this *NormalCounter) Output(out io.Writer, mutiline bool) {
	w := bufio.NewWriter(out)
	for _, v := range this.ma {
		v2 := this.m[v]
		if mutiline {
			fmt.Fprintf(w, "%s : %v\n", string(v), v2)
		} else {
			w.WriteRune(v)
		}
	}
	w.Flush()
}
func (this *NormalCounter) AllCount() int64 { return this.count }
func (this *NormalCounter) Counted() int    { return len(this.ma) }

// 排序字符统计器
type SortCounter struct {
	count int64
	// charsets []Charset
	m  map[rune]int
	rm map[int][]rune
	ra []int
}

func NewSortCounter() *SortCounter { return &SortCounter{m: make(map[rune]int)} }
func (this *SortCounter) ReadAll(in io.Reader) {
	r := bufio.NewReader(in)
	for {
		ru, _, err := r.ReadRune()
		if err != nil {
			break
		}
		if _, ok := this.m[ru]; ru >= 0x4e00 && ru <= 0x9fa5 {
			if !ok {
				this.m[ru] = 0
			}
			this.count++
			this.m[ru]++
		}
	}
}
func (this *SortCounter) Sort() {
	this.rm = make(map[int][]rune)
	for i, v := range this.m {
		if _, ok := this.rm[v]; !ok {
			this.rm[v] = make([]rune, 0, 2)
		}
		this.rm[v] = append(this.rm[v], i)
	}
	this.ra = make([]int, 0, len(this.rm))
	for i, _ := range this.rm {
		this.ra = append(this.ra, i)
	}
	sort.Ints(this.ra)
}
func (this *SortCounter) Output(out io.Writer, mutiline bool) {
	this.Sort()
	w := bufio.NewWriter(out)
	for i := len(this.ra) - 1; i >= 0; i-- {
		v := this.ra[i]
		for _, v2 := range this.rm[v] {
			if mutiline {
				fmt.Fprintf(w, "%s : %v\n", string(v2), v)
			} else {
				w.WriteRune(v2)
			}
		}
	}
	w.Flush()
}
func (this *SortCounter) AllCount() int64 { return this.count }
func (this *SortCounter) Counted() int    { return len(this.m) }

func parseflags() (in string, out string, sort, mutiline, count bool) {
	i := flag.String("i", "stdin", "the file you want to use")
	o := flag.String("o", "stdout", "the file you want to output")
	s := flag.Bool("s", false, "sort by use times")
	l := flag.Bool("l", false, "output mutiline text")
	c := flag.Bool("c", false, "print char count")
	h := flag.Bool("h", false, "show help")
	flag.Parse()
	if *h {
		flag.Usage()
		os.Exit(0)
	}
	return *i, *o, *s, *l, *c
}

func main() {
	in, out, isSort, isMutiline, isCount := parseflags()
	var fin io.Reader
	if in == "stdin" {
		fin = os.Stdin
	} else {
		f, err := os.Open(in)
		if err != nil {
			fmt.Println("[ERROR]", err)
			return
		}
		defer f.Close()
		fin = f
	}
	var fout io.Writer
	if out == "stdout" {
		fout = os.Stdout
	} else if out == "stderr" {
		fout = os.Stderr
	} else {
		fw, err := os.Create(out)
		if err != nil {
			fmt.Println("[ERROR]", err)
		}
		defer fw.Close()
		fout = fw
	}
	var co Counter
	if isSort {
		co = NewSortCounter()
	} else {
		co = NewNormalCounter()
	}
	co.ReadAll(fin)
	co.Output(fout, isMutiline)
	if isCount {
		if out == "stdout" || out == "stderr" {
			fmt.Println()
		}
		fmt.Println("[All]", co.AllCount(), "[Chars]", co.Counted())
	}
}
<commit_msg>charc: fix in Interrupt<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"sort"
)

type Charset struct {
	From, To rune
}

type Counter interface {
	ReadAll(in io.Reader)                // 读取全部
	Output(out io.Writer, mutiline bool) // 输出
	AllCount() int64                     // 全部字符数量
	Counted() int                        // 计算进的数量
}

// 不排序字符统计器
type NormalCounter struct {
	count int64

	m  map[rune]int
	ma []rune
}

func NewNormalCounter() *NormalCounter {
	return &NormalCounter{
		m:  make(map[rune]int),
		ma: make([]rune, 0, 0x9fa5-0x4e00),
	}
}
func (this *NormalCounter) ReadAll(in io.Reader) {
	r := bufio.NewReader(in)
	for {
		ru, _, err := r.ReadRune()
		if err != nil {
			break
		}
		if _, ok := this.m[ru]; ru >= 0x4e00 && ru <= 0x9fa5 {
			if !ok {
				this.m[ru] = 0
				this.ma = append(this.ma, ru)
			}
			this.count++
			this.m[ru]++
		}
	}
}
func (this *NormalCounter) Output(out io.Writer, mutiline bool) {
	w := bufio.NewWriter(out)
	for _, v := range this.ma {
		v2 := this.m[v]
		if mutiline {
			fmt.Fprintf(w, "%s : %v\n", string(v), v2)
		} else {
			w.WriteRune(v)
		}
	}
	w.Flush()
}
func (this *NormalCounter) AllCount() int64 { return this.count }
func (this *NormalCounter) Counted() int    { return len(this.ma) }

// 排序字符统计器
type SortCounter struct {
	count int64
	// charsets []Charset
	m  map[rune]int
	rm map[int][]rune
	ra []int
}

func NewSortCounter() *SortCounter { return &SortCounter{m: make(map[rune]int)} }
func (this *SortCounter) ReadAll(in io.Reader) {
	r := bufio.NewReader(in)
	for {
		ru, _, err := r.ReadRune()
		if err != nil {
			break
		}
		if _, ok := this.m[ru]; ru >= 0x4e00 && ru <= 0x9fa5 {
			if !ok {
				this.m[ru] = 0
			}
			this.count++
			this.m[ru]++
		}
	}
}
func (this *SortCounter) Sort() {
	this.rm = make(map[int][]rune)
	for i, v := range this.m {
		if _, ok := this.rm[v]; !ok {
			this.rm[v] = make([]rune, 0, 2)
		}
		this.rm[v] = append(this.rm[v], i)
	}
	this.ra = make([]int, 0, len(this.rm))
	for i, _ := range this.rm {
		this.ra = append(this.ra, i)
	}
	sort.Ints(this.ra)
}
func (this *SortCounter) Output(out io.Writer, mutiline bool) {
	this.Sort()
	w := bufio.NewWriter(out)
	for i := len(this.ra) - 1; i >= 0; i-- {
		v := this.ra[i]
		for _, v2 := range this.rm[v] {
			if mutiline {
				fmt.Fprintf(w, "%s : %v\n", string(v2), v)
			} else {
				w.WriteRune(v2)
			}
		}
	}
	w.Flush()
}
func (this *SortCounter) AllCount() int64 { return this.count }
func (this *SortCounter) Counted() int    { return len(this.m) }

func parseflags() (in string, out string, sort, mutiline, count bool) {
	i := flag.String("i", "stdin", "the file you want to use")
	o := flag.String("o", "stdout", "the file you want to output")
	s := flag.Bool("s", false, "sort by use times")
	l := flag.Bool("l", false, "output mutiline text")
	c := flag.Bool("c", false, "print char count")
	h := flag.Bool("h", false, "show help")
	flag.Parse()
	if *h {
		flag.Usage()
		os.Exit(0)
	}
	return *i, *o, *s, *l, *c
}

func main() {
	in, out, isSort, isMutiline, isCount := parseflags()
	var fin io.Reader
	if in == "stdin" {
		fin = os.Stdin
	} else {
		f, err := os.Open(in)
		if err != nil {
			fmt.Println("[ERROR]", err)
			return
		}
		defer f.Close()
		fin = f
	}
	var fout io.Writer
	if out == "stdout" {
		fout = os.Stdout
	} else if out == "stderr" {
		fout = os.Stderr
	} else {
		fw, err := os.Create(out)
		if err != nil {
			fmt.Println("[ERROR]", err)
		}
		defer fw.Close()
		fout = fw
	}
	var co Counter
	if isSort {
		co = NewSortCounter()
	} else {
		co = NewNormalCounter()
	}

	// stdin 输入下防止坑爹
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			if isCount {
				fmt.Println("\n[All]", co.AllCount(), "[Chars]", co.Counted())
			}
			os.Exit(0)
		}
	}()
	co.ReadAll(fin)
	co.Output(fout, isMutiline)
	if isCount {
		fmt.Println("\n[All]", co.AllCount(), "[Chars]", co.Counted())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"fmt"

	"github.com/mdigger/rest"
)

// PostCall обрабатывает обратный вызов звонка.
func (mx *MX) PostCall(c *rest.Context) error {
	// проверяем авторизацию пользователя
	login, password, err := Authorize(c)
	if err != nil {
		return err
	}

	// Params описывает параметры, передаваемые в запроса
	type Params struct {
		RingDelay uint8  `xml:"ringdelay,attr" json:"ringDelay" form:"ringDelay"`
		VMDelay   uint8  `xml:"vmdelay,attr" json:"vmDelay" form:"vmDelay"`
		From      string `xml:"address" json:"from" form:"from"`
		To        string `xml:"-" json:"to" form:"to"`
	}
	// инициализируем параметры по умолчанию и разбираем запрос
	var params = &Params{
		RingDelay: 1,
		VMDelay:   30,
	}
	if err := c.Bind(params); err != nil {
		return err
	}
	// инициализируем пользовательское соединение с сервером MX
	client, err := mx.UserClient(login, password)
	if err != nil {
		return httpError(c, err)
	}
	defer client.Close()

	// отправляем команду на установку номера исходящего звонка
	if _, err = client.Send(&struct {
		XMLName xml.Name `xml:"iq"`
		Type    string   `xml:"type,attr"`
		ID      string   `xml:"id,attr"`
		Mode    string   `xml:"mode,attr"`
		*Params
	}{
		Type:   "set",
		ID:     "mode",
		Mode:   "remote",
		Params: params,
	}); err != nil {
		return err
	}
	// инициируем звонок на номер
	type callingDevice struct {
		Type string `xml:"typeOfNumber,attr"`
		Ext  string `xml:",chardata"`
	}
	id, err := client.Send(&struct {
		XMLName       xml.Name      `xml:"http://www.ecma-international.org/standards/ecma-323/csta/ed4 MakeCall"`
		CallingDevice callingDevice `xml:"callingDevice"`
		To            string        `xml:"calledDirectoryNumber"`
	}{
		CallingDevice: callingDevice{
			Type: "deviceID",
			Ext:  client.Ext,
		},
		To: params.To,
	})
	if err != nil {
		return err
	}

	// читаем ответ
	client.SetWait(MXReadTimeout)
read:
	resp, err := client.Receive()
	if err != nil {
		return err
	}
	if resp.ID != id {
		goto read
	}
	switch resp.Name {
	case "CSTAErrorCode":
		cstaError := new(CSTAError)
		if err := resp.Decode(cstaError); err != nil {
			return err
		}
		return cstaError
	case "MakeCallResponse":
		var result = new(struct {
			CallID       uint64 `xml:"callingDevice>callID" json:"callId"`
			DeviceID     string `xml:"callingDevice>deviceID" json:"deviceId"`
			CalledDevice string `xml:"calledDevice" json:"called"`
		})
		if err := resp.Decode(result); err != nil {
			return err
		}
		return c.Write(result)
	default:
		return fmt.Errorf("unknown makecall response %s", resp.Name)
	}
}
<commit_msg>fix CSTA receive<commit_after>package main

import (
	"encoding/xml"
	"fmt"

	"github.com/mdigger/rest"
)

// PostCall обрабатывает обратный вызов звонка.
func (mx *MX) PostCall(c *rest.Context) error {
	// проверяем авторизацию пользователя
	login, password, err := Authorize(c)
	if err != nil {
		return err
	}

	// Params описывает параметры, передаваемые в запроса
	type Params struct {
		RingDelay uint8  `xml:"ringdelay,attr" json:"ringDelay" form:"ringDelay"`
		VMDelay   uint8  `xml:"vmdelay,attr" json:"vmDelay" form:"vmDelay"`
		From      string `xml:"address" json:"from" form:"from"`
		To        string `xml:"-" json:"to" form:"to"`
	}
	// инициализируем параметры по умолчанию и разбираем запрос
	var params = &Params{
		RingDelay: 1,
		VMDelay:   30,
	}
	if err := c.Bind(params); err != nil {
		return err
	}
	// инициализируем пользовательское соединение с сервером MX
	client, err := mx.UserClient(login, password)
	if err != nil {
		return httpError(c, err)
	}
	defer client.Close()

	// отправляем команду на установку номера исходящего звонка
	if _, err = client.Send(&struct {
		XMLName xml.Name `xml:"iq"`
		Type    string   `xml:"type,attr"`
		ID      string   `xml:"id,attr"`
		Mode    string   `xml:"mode,attr"`
		*Params
	}{
		Type:   "set",
		ID:     "mode",
		Mode:   "remote",
		Params: params,
	}); err != nil {
		return err
	}
	// инициируем звонок на номер
	type callingDevice struct {
		Type string `xml:"typeOfNumber,attr"`
		Ext  string `xml:",chardata"`
	}
	var cmd = &struct {
		XMLName       xml.Name      `xml:"http://www.ecma-international.org/standards/ecma-323/csta/ed4 MakeCall"`
		CallingDevice callingDevice `xml:"callingDevice"`
		To            string        `xml:"calledDirectoryNumber"`
	}{
		CallingDevice: callingDevice{
			Type: "deviceID",
			Ext:  client.Ext,
		},
		To: params.To,
	}
	resp, err := client.SendWithResponse(cmd, MXReadTimeout)
	if err != nil {
		return err
	}
	switch resp.Name {
	case "CSTAErrorCode":
		cstaError := new(CSTAError)
		if err := resp.Decode(cstaError); err != nil {
			return err
		}
		return cstaError
	case "MakeCallResponse":
		var result = new(struct {
			CallID       uint64 `xml:"callingDevice>callID" json:"callId"`
			DeviceID     string `xml:"callingDevice>deviceID" json:"deviceId"`
			CalledDevice string `xml:"calledDevice" json:"called"`
		})
		if err := resp.Decode(result); err != nil {
			return err
		}
		return c.Write(result)
	default:
		return fmt.Errorf("unknown makecall response %s", resp.Name)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENE file.

package leveldb

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"testing"

	"github.com/syndtr/goleveldb/leveldb/storage"
	"github.com/syndtr/goleveldb/leveldb/util"
)

const typeShift = 3

var (
	tsErrInvalidFile = errors.New("leveldb.testStorage: invalid file for argument")
	tsErrFileOpen    = errors.New("leveldb.testStorage: file still open")
)

var (
	tsKeepFS = os.Getenv("GOLEVELDB_USEFS") == "2"
	tsFS     = tsKeepFS || os.Getenv("GOLEVELDB_USEFS") == "1"
	tsMU     = &sync.Mutex{}
	tsNum    = 0
)

type tsLock struct {
	ts *testStorage
	r  util.Releaser
}

func (l tsLock) Release() {
	l.r.Release()
	l.ts.t.Log("I: storage lock released")
}

type tsReader struct {
	tf tsFile
	storage.Reader
}

func (tr tsReader) Read(b []byte) (n int, err error) {
	ts := tr.tf.ts
	ts.countRead(tr.tf.Type())
	n, err = tr.Reader.Read(b)
	if err != nil && err != io.EOF {
		ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
	}
	return
}

func (tr tsReader) Close() (err error) {
	err = tr.Reader.Close()
	tr.tf.close("reader", err)
	return
}

type tsWriter struct {
	tf tsFile
	storage.Writer
}

func (tw tsWriter) Write(b []byte) (n int, err error) {
	ts := tw.tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	if ts.emuWriteErr&tw.tf.Type() != 0 {
		return 0, errors.New("leveldb.testStorage: emulated write error")
	}
	n, err = tw.Writer.Write(b)
	if err != nil {
		ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
	}
	return
}

func (tw tsWriter) Sync() (err error) {
	ts := tw.tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	for ts.emuDelaySync&tw.tf.Type() != 0 {
		ts.cond.Wait()
	}
	if ts.emuSyncErr&tw.tf.Type() != 0 {
		return errors.New("leveldb.testStorage: emulated sync error")
	}
	err = tw.Writer.Sync()
	if err != nil {
		ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
	}
	return
}

func (tw tsWriter) Close() (err error) {
	err = tw.Writer.Close()
	tw.tf.close("reader", err)
	return
}

type tsFile struct {
	ts *testStorage
	storage.File
}

func (tf tsFile) x() uint64 {
	return tf.Num()<<typeShift | uint64(tf.Type())
}

func (tf tsFile) checkOpen(m string) error {
	ts := tf.ts
	if writer, ok := ts.opens[tf.x()]; ok {
		if writer {
			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a writer still open", m, tf.Num(), tf.Type())
		} else {
			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a reader still open", m, tf.Num(), tf.Type())
		}
		return tsErrFileOpen
	}
	return nil
}

func (tf tsFile) close(m string, err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	if _, ok := ts.opens[tf.x()]; !ok {
		ts.t.Errorf("E: %s: redudant file closing, num=%d type=%v", m, tf.Num(), tf.Type())
	} else if err == nil {
		ts.t.Logf("I: %s: file closed, num=%d type=%v", m, tf.Num(), tf.Type())
	}
	delete(ts.opens, tf.x())
	if err != nil {
		ts.t.Errorf("E: %s: cannot close file, num=%d type=%v: %v", m, tf.Num(), tf.Type(), err)
	}
}

func (tf tsFile) Open() (r storage.Reader, err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	err = tf.checkOpen("open")
	if err != nil {
		return
	}
	r, err = tf.File.Open()
	if err != nil {
		ts.t.Errorf("E: cannot open file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
	} else {
		ts.t.Logf("I: file opened, num=%d type=%v", tf.Num(), tf.Type())
		ts.opens[tf.x()] = false
		r = tsReader{tf, r}
	}
	return
}

func (tf tsFile) Create() (w storage.Writer, err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	err = tf.checkOpen("create")
	if err != nil {
		return
	}
	w, err = tf.File.Create()
	if err != nil {
		ts.t.Errorf("E: cannot create file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
	} else {
		ts.t.Logf("I: file created, num=%d type=%v", tf.Num(), tf.Type())
		ts.opens[tf.x()] = true
		w = tsWriter{tf, w}
	}
	return
}

func (tf tsFile) Remove() (err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	err = tf.checkOpen("remove")
	if err != nil {
		return
	}
	err = tf.File.Remove()
	if err != nil {
		ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
	} else {
		ts.t.Logf("I: file removed, num=%d type=%v", tf.Num(), tf.Type())
	}
	return
}

type testStorage struct {
	t *testing.T
	storage.Storage
	closeFn func() error

	mu   sync.Mutex
	cond sync.Cond
	// Open files, true=writer, false=reader
	opens        map[uint64]bool
	emuDelaySync storage.FileType
	emuWriteErr  storage.FileType
	emuSyncErr   storage.FileType
	readCnt      uint64
	readCntEn    storage.FileType
}

func (ts *testStorage) DelaySync(t storage.FileType) {
	ts.mu.Lock()
	ts.emuDelaySync |= t
	ts.cond.Broadcast()
	ts.mu.Unlock()
}

func (ts *testStorage) ReleaseSync(t storage.FileType) {
	ts.mu.Lock()
	ts.emuDelaySync &= ^t
	ts.cond.Broadcast()
	ts.mu.Unlock()
}

func (ts *testStorage) SetWriteErr(t storage.FileType) {
	ts.mu.Lock()
	ts.emuWriteErr = t
	ts.mu.Unlock()
}

func (ts *testStorage) SetSyncErr(t storage.FileType) {
	ts.mu.Lock()
	ts.emuSyncErr = t
	ts.mu.Unlock()
}

func (ts *testStorage) ReadCounter() uint64 {
	ts.mu.Lock()
	defer ts.mu.Unlock()
	return ts.readCnt
}

func (ts *testStorage) ResetReadCounter() {
	ts.mu.Lock()
	ts.readCnt = 0
	ts.mu.Unlock()
}

func (ts *testStorage) SetReadCounter(t storage.FileType) {
	ts.mu.Lock()
	ts.readCntEn = t
	ts.mu.Unlock()
}

func (ts *testStorage) countRead(t storage.FileType) {
	ts.mu.Lock()
	if ts.readCntEn&t != 0 {
		ts.readCnt++
	}
	ts.mu.Unlock()
}

func (ts *testStorage) Lock() (r util.Releaser, err error) {
	r, err = ts.Storage.Lock()
	if err != nil {
		ts.t.Logf("W: storage locking failed: %v", err)
	} else {
		ts.t.Log("I: storage locked")
		r = tsLock{ts, r}
	}
	return
}

func (ts *testStorage) Log(str string) {
	ts.t.Log("L: " + str)
	ts.Storage.Log(str)
}

func (ts *testStorage) GetFile(num uint64, t storage.FileType) storage.File {
	return tsFile{ts, ts.Storage.GetFile(num, t)}
}

func (ts *testStorage) GetFiles(t storage.FileType) (ff []storage.File, err error) {
	ff0, err := ts.Storage.GetFiles(t)
	if err != nil {
		ts.t.Errorf("E: get files failed: %v", err)
		return
	}
	ff = make([]storage.File, len(ff0))
	for i, f := range ff0 {
		ff[i] = tsFile{ts, f}
	}
	ts.t.Logf("I: get files, type=0x%x count=%d", int(t), len(ff))
	return
}

func (ts *testStorage) GetManifest() (f storage.File, err error) {
	f0, err := ts.Storage.GetManifest()
	if err != nil {
		if !os.IsNotExist(err) {
			ts.t.Errorf("E: get manifest failed: %v", err)
		}
		return
	}
	f = tsFile{ts, f0}
	ts.t.Logf("I: get manifest, num=%d", f.Num())
	return
}

func (ts *testStorage) SetManifest(f storage.File) error {
	tf, ok := f.(tsFile)
	if !ok {
		ts.t.Error("E: set manifest failed: type assertion failed")
		return tsErrInvalidFile
	} else if tf.Type() != storage.TypeManifest {
		ts.t.Errorf("E: set manifest failed: invalid file type: %s", tf.Type())
		return tsErrInvalidFile
	}
	err := ts.Storage.SetManifest(tf.File)
	if err != nil {
		ts.t.Errorf("E: set manifest failed: %v", err)
	} else {
		ts.t.Logf("I: set manifest, num=%d", tf.Num())
	}
	return err
}

func (ts *testStorage) Close() error {
	err := ts.Storage.Close()
	if err != nil {
		ts.t.Errorf("E: closing storage failed: %v", err)
	} else {
		ts.t.Log("I: storage closed")
	}
	if ts.closeFn != nil {
		if err := ts.closeFn(); err != nil {
			ts.t.Errorf("E: close function: %v", err)
		}
	}
	return err
}

func (ts *testStorage) CloseCheck() {
	ts.mu.Lock()
	if len(ts.opens) == 0 {
		ts.t.Log("I: all files are closed")
	} else {
		ts.t.Errorf("E: %d files still open", len(ts.opens))
		for x, writer := range ts.opens {
			num, tt := x>>typeShift, storage.FileType(x)&storage.TypeAll
			ts.t.Errorf("E: * num=%d type=%v writer=%v", num, tt, writer)
		}
	}
	ts.mu.Unlock()
}

func newTestStorage(t *testing.T) *testStorage {
	var stor storage.Storage
	var closeFn func() error
	if tsFS {
		for {
			tsMU.Lock()
			num := tsNum
			tsNum++
			tsMU.Unlock()
			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
			if _, err := os.Stat(path); err != nil {
				stor, err = storage.OpenFile(path)
				if err != nil {
					t.Fatalf("F: cannot create storage: %v", err)
				}
				t.Logf("I: storage created: %s", path)
				closeFn = func() error {
					for _, name := range []string{"LOG.old", "LOG"} {
						f, err := os.Open(filepath.Join(path, name))
						if err != nil {
							continue
						}
						if log, err := ioutil.ReadAll(f); err != nil {
							t.Logf("---------------------- %s ----------------------", name)
							t.Logf("cannot read log: %v", err)
						} else {
							t.Logf("---------------------- %s ----------------------\n%s", name, string(log))
						}
						f.Close()
						t.Logf("---------------------- %s ----------------------", name)
					}
					if tsKeepFS {
						return nil
					}
					return os.RemoveAll(path)
				}

				break
			}
		}
	} else {
		stor = storage.NewMemStorage()
	}
	ts := &testStorage{
		t:       t,
		Storage: stor,
		closeFn: closeFn,
		opens:   make(map[uint64]bool),
	}
	ts.cond.L = &ts.mu
	return ts
}
<commit_msg>leveldb: TestStorage: Call CloseCheck before closing the storage<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENE file.

package leveldb

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"testing"

	"github.com/syndtr/goleveldb/leveldb/storage"
	"github.com/syndtr/goleveldb/leveldb/util"
)

const typeShift = 3

var (
	tsErrInvalidFile = errors.New("leveldb.testStorage: invalid file for argument")
	tsErrFileOpen    = errors.New("leveldb.testStorage: file still open")
)

var (
	tsKeepFS = os.Getenv("GOLEVELDB_USEFS") == "2"
	tsFS     = tsKeepFS || os.Getenv("GOLEVELDB_USEFS") == "1"
	tsMU     = &sync.Mutex{}
	tsNum    = 0
)

type tsLock struct {
	ts *testStorage
	r  util.Releaser
}

func (l tsLock) Release() {
	l.r.Release()
	l.ts.t.Log("I: storage lock released")
}

type tsReader struct {
	tf tsFile
	storage.Reader
}

func (tr tsReader) Read(b []byte) (n int, err error) {
	ts := tr.tf.ts
	ts.countRead(tr.tf.Type())
	n, err = tr.Reader.Read(b)
	if err != nil && err != io.EOF {
		ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
	}
	return
}

func (tr tsReader) Close() (err error) {
	err = tr.Reader.Close()
	tr.tf.close("reader", err)
	return
}

type tsWriter struct {
	tf tsFile
	storage.Writer
}

func (tw tsWriter) Write(b []byte) (n int, err error) {
	ts := tw.tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	if ts.emuWriteErr&tw.tf.Type() != 0 {
		return 0, errors.New("leveldb.testStorage: emulated write error")
	}
	n, err = tw.Writer.Write(b)
	if err != nil {
		ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
	}
	return
}

func (tw tsWriter) Sync() (err error) {
	ts := tw.tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	for ts.emuDelaySync&tw.tf.Type() != 0 {
		ts.cond.Wait()
	}
	if ts.emuSyncErr&tw.tf.Type() != 0 {
		return errors.New("leveldb.testStorage: emulated sync error")
	}
	err = tw.Writer.Sync()
	if err != nil {
		ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
	}
	return
}

func (tw tsWriter) Close() (err error) {
	err = tw.Writer.Close()
	tw.tf.close("reader", err)
	return
}

type tsFile struct {
	ts *testStorage
	storage.File
}

func (tf tsFile) x() uint64 {
	return tf.Num()<<typeShift | uint64(tf.Type())
}

func (tf tsFile) checkOpen(m string) error {
	ts := tf.ts
	if writer, ok := ts.opens[tf.x()]; ok {
		if writer {
			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a writer still open", m, tf.Num(), tf.Type())
		} else {
			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a reader still open", m, tf.Num(), tf.Type())
		}
		return tsErrFileOpen
	}
	return nil
}

func (tf tsFile) close(m string, err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	if _, ok := ts.opens[tf.x()]; !ok {
		ts.t.Errorf("E: %s: redudant file closing, num=%d type=%v", m, tf.Num(), tf.Type())
	} else if err == nil {
		ts.t.Logf("I: %s: file closed, num=%d type=%v", m, tf.Num(), tf.Type())
	}
	delete(ts.opens, tf.x())
	if err != nil {
		ts.t.Errorf("E: %s: cannot close file, num=%d type=%v: %v", m, tf.Num(), tf.Type(), err)
	}
}

func (tf tsFile) Open() (r storage.Reader, err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	err = tf.checkOpen("open")
	if err != nil {
		return
	}
	r, err = tf.File.Open()
	if err != nil {
		ts.t.Errorf("E: cannot open file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
	} else {
		ts.t.Logf("I: file opened, num=%d type=%v", tf.Num(), tf.Type())
		ts.opens[tf.x()] = false
		r = tsReader{tf, r}
	}
	return
}

func (tf tsFile) Create() (w storage.Writer, err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	err = tf.checkOpen("create")
	if err != nil {
		return
	}
	w, err = tf.File.Create()
	if err != nil {
		ts.t.Errorf("E: cannot create file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
	} else {
		ts.t.Logf("I: file created, num=%d type=%v", tf.Num(), tf.Type())
		ts.opens[tf.x()] = true
		w = tsWriter{tf, w}
	}
	return
}

func (tf tsFile) Remove() (err error) {
	ts := tf.ts
	ts.mu.Lock()
	defer ts.mu.Unlock()
	err = tf.checkOpen("remove")
	if err != nil {
		return
	}
	err = tf.File.Remove()
	if err != nil {
		ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
	} else {
		ts.t.Logf("I: file removed, num=%d type=%v", tf.Num(), tf.Type())
	}
	return
}

type testStorage struct {
	t *testing.T
	storage.Storage
	closeFn func() error

	mu   sync.Mutex
	cond sync.Cond
	// Open files, true=writer, false=reader
	opens        map[uint64]bool
	emuDelaySync storage.FileType
	emuWriteErr  storage.FileType
	emuSyncErr   storage.FileType
	readCnt      uint64
	readCntEn    storage.FileType
}

func (ts *testStorage) DelaySync(t storage.FileType) {
	ts.mu.Lock()
	ts.emuDelaySync |= t
	ts.cond.Broadcast()
	ts.mu.Unlock()
}

func (ts *testStorage) ReleaseSync(t storage.FileType) {
	ts.mu.Lock()
	ts.emuDelaySync &= ^t
	ts.cond.Broadcast()
	ts.mu.Unlock()
}

func (ts *testStorage) SetWriteErr(t storage.FileType) {
	ts.mu.Lock()
	ts.emuWriteErr = t
	ts.mu.Unlock()
}

func (ts *testStorage) SetSyncErr(t storage.FileType) {
	ts.mu.Lock()
	ts.emuSyncErr = t
	ts.mu.Unlock()
}

func (ts *testStorage) ReadCounter() uint64 {
	ts.mu.Lock()
	defer ts.mu.Unlock()
	return ts.readCnt
}

func (ts *testStorage) ResetReadCounter() {
	ts.mu.Lock()
	ts.readCnt = 0
	ts.mu.Unlock()
}

func (ts *testStorage) SetReadCounter(t storage.FileType) {
	ts.mu.Lock()
	ts.readCntEn = t
	ts.mu.Unlock()
}

func (ts *testStorage) countRead(t storage.FileType) {
	ts.mu.Lock()
	if ts.readCntEn&t != 0 {
		ts.readCnt++
	}
	ts.mu.Unlock()
}

func (ts *testStorage) Lock() (r util.Releaser, err error) {
	r, err = ts.Storage.Lock()
	if err != nil {
		ts.t.Logf("W: storage locking failed: %v", err)
	} else {
		ts.t.Log("I: storage locked")
		r = tsLock{ts, r}
	}
	return
}

func (ts *testStorage) Log(str string) {
	ts.t.Log("L: " + str)
	ts.Storage.Log(str)
}

func (ts *testStorage) GetFile(num uint64, t storage.FileType) storage.File {
	return tsFile{ts, ts.Storage.GetFile(num, t)}
}

func (ts *testStorage) GetFiles(t storage.FileType) (ff []storage.File, err error) {
	ff0, err := ts.Storage.GetFiles(t)
	if err != nil {
		ts.t.Errorf("E: get files failed: %v", err)
		return
	}
	ff = make([]storage.File, len(ff0))
	for i, f := range ff0 {
		ff[i] = tsFile{ts, f}
	}
	ts.t.Logf("I: get files, type=0x%x count=%d", int(t), len(ff))
	return
}

func (ts *testStorage) GetManifest() (f storage.File, err error) {
	f0, err := ts.Storage.GetManifest()
	if err != nil {
		if !os.IsNotExist(err) {
			ts.t.Errorf("E: get manifest failed: %v", err)
		}
		return
	}
	f = tsFile{ts, f0}
	ts.t.Logf("I: get manifest, num=%d", f.Num())
	return
}

func (ts *testStorage) SetManifest(f storage.File) error {
	tf, ok := f.(tsFile)
	if !ok {
		ts.t.Error("E: set manifest failed: type assertion failed")
		return tsErrInvalidFile
	} else if tf.Type() != storage.TypeManifest {
		ts.t.Errorf("E: set manifest failed: invalid file type: %s", tf.Type())
		return tsErrInvalidFile
	}
	err := ts.Storage.SetManifest(tf.File)
	if err != nil {
		ts.t.Errorf("E: set manifest failed: %v", err)
	} else {
		ts.t.Logf("I: set manifest, num=%d", tf.Num())
	}
	return err
}

func (ts *testStorage) Close() error {
	ts.CloseCheck()
	err := ts.Storage.Close()
	if err != nil {
		ts.t.Errorf("E: closing storage failed: %v", err)
	} else {
		ts.t.Log("I: storage closed")
	}
	if ts.closeFn != nil {
		if err := ts.closeFn(); err != nil {
			ts.t.Errorf("E: close function: %v", err)
		}
	}
	return err
}

func (ts *testStorage) CloseCheck() {
	ts.mu.Lock()
	if len(ts.opens) == 0 {
		ts.t.Log("I: all files are closed")
	} else {
		ts.t.Errorf("E: %d files still open", len(ts.opens))
		for x, writer := range ts.opens {
			num, tt := x>>typeShift, storage.FileType(x)&storage.TypeAll
			ts.t.Errorf("E: * num=%d type=%v writer=%v", num, tt, writer)
		}
	}
	ts.mu.Unlock()
}

func newTestStorage(t *testing.T) *testStorage {
	var stor storage.Storage
	var closeFn func() error
	if tsFS {
		for {
			tsMU.Lock()
			num := tsNum
			tsNum++
			tsMU.Unlock()
			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
			if _, err := os.Stat(path); err != nil {
				stor, err = storage.OpenFile(path)
				if err != nil {
					t.Fatalf("F: cannot create storage: %v", err)
				}
				t.Logf("I: storage created: %s", path)
				closeFn = func() error {
					for _, name := range []string{"LOG.old", "LOG"} {
						f, err := os.Open(filepath.Join(path, name))
						if err != nil {
							continue
						}
						if log, err := ioutil.ReadAll(f); err != nil {
							t.Logf("---------------------- %s ----------------------", name)
							t.Logf("cannot read log: %v", err)
						} else {
							t.Logf("---------------------- %s ----------------------\n%s", name, string(log))
						}
						f.Close()
						t.Logf("---------------------- %s ----------------------", name)
					}
					if tsKeepFS {
						return nil
					}
					return os.RemoveAll(path)
				}

				break
			}
		}
	} else {
		stor = storage.NewMemStorage()
	}
	ts := &testStorage{
		t:       t,
		Storage: stor,
		closeFn: closeFn,
		opens:   make(map[uint64]bool),
	}
	ts.cond.L = &ts.mu
	return ts
}
<|endoftext|>


<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This LevelDB Go implementation is based on LevelDB C++ implementation.
// Which contains the following header:
//   Copyright (c) 2011 The LevelDB Authors. All rights reserved.
//   Use of this source code is governed by a BSD-style license that can be
//   found in the LEVELDBCPP_LICENSE file. See the LEVELDBCPP_AUTHORS file
//   for names of contributors.

// Package table allows read and write sorted key/value.
package table

import (
	"runtime"

	"leveldb/block"
	"leveldb/cache"
	"leveldb/comparer"
	"leveldb/desc"
	"leveldb/errors"
	"leveldb/iter"
	"leveldb/opt"
)

// Reader represent a table reader.
type Reader struct {
	r desc.Reader
	o opt.OptionsGetter

	meta   *block.Reader
	index  *block.Reader
	filter *block.FilterReader

	dataEnd uint64
	cache   cache.Namespace
}

// NewReader create new initialized table reader.
func NewReader(r desc.Reader, size uint64, o opt.OptionsGetter, cache cache.Namespace) (p *Reader, err error) {
	mb, ib, err := readFooter(r, size)
	if err != nil {
		return
	}

	t := &Reader{r: r, o: o, dataEnd: mb.offset, cache: cache}

	// index block
	buf, err := ib.readAll(r, true)
	if err != nil {
		return
	}
	t.index, err = block.NewReader(buf, o.GetComparer())
	if err != nil {
		return
	}

	// filter block
	filter := o.GetFilter()
	if filter != nil {
		// we will ignore any errors at meta/filter block
		// since it is not essential for operation

		// meta block
		buf, err = mb.readAll(r, true)
		if err != nil {
			goto out
		}
		var meta *block.Reader
		meta, err = block.NewReader(buf, comparer.BytesComparer{})
		if err != nil {

			goto out
		}

		// check for filter name
		iter := meta.NewIterator()
		key := "filter." + filter.Name()
		if iter.Seek([]byte(key)) && string(iter.Key()) == key {
			fb := new(bInfo)
			_, err = fb.decodeFrom(iter.Value())
			if err != nil {
				return
			}

			// now the data end before filter block start offset
			t.dataEnd = fb.offset

			// filter block
			buf, err = fb.readAll(r, true)
			if err != nil {
				goto out
			}
			t.filter, err = block.NewFilterReader(buf, filter)
			if err != nil {
				goto out
			}
		}
	}

out:
	return t, nil
}

// NewIterator create new iterator over the table.
func (t *Reader) NewIterator(ro opt.ReadOptionsGetter) iter.Iterator {
	index_iter := &indexIter{t: t, ro: ro}
	t.index.InitIterator(&index_iter.Iterator)
	return iter.NewIndexedIterator(index_iter)
}

// Get lookup for given key on the table. Get returns errors.ErrNotFound if
// given key did not exist.
func (t *Reader) Get(key []byte, ro opt.ReadOptionsGetter) (rkey, rvalue []byte, err error) {
	// create an iterator of index block
	index_iter := t.index.NewIterator()
	if !index_iter.Seek(key) {
		err = index_iter.Error()
		if err == nil {
			err = errors.ErrNotFound
		}
		return
	}

	// decode data block info
	bi := new(bInfo)
	_, err = bi.decodeFrom(index_iter.Value())
	if err != nil {
		return
	}

	// get the data block
	if t.filter == nil || t.filter.KeyMayMatch(uint(bi.offset), key) {
		var it iter.Iterator
		var cache cache.Object
		it, cache, err = t.getDataIter(bi, ro)
		if err != nil {
			return
		}
		if cache != nil {
			defer cache.Release()
		}

		// seek to key
		if !it.Seek(key) {
			err = it.Error()
			if err == nil {
				err = errors.ErrNotFound
			}
			return
		}
		rkey, rvalue = it.Key(), it.Value()
	} else {
		err = errors.ErrNotFound
	}
	return
}

// ApproximateOffsetOf approximate the offset of given key in bytes.
func (t *Reader) ApproximateOffsetOf(key []byte) uint64 {
	index_iter := t.index.NewIterator()
	if index_iter.Seek(key) {
		bi := new(bInfo)
		_, err := bi.decodeFrom(index_iter.Value())
		if err == nil {
			return bi.offset
		}
	}
	// block info is corrupted or key is past the last key in the file.
	// Approximate the offset by returning offset of the end of data
	// block (which is right near the end of the file).
	return t.dataEnd
}

func (t *Reader) getDataIter(bi *bInfo, ro opt.ReadOptionsGetter) (it *block.Iterator, cache cache.Object, err error) {
	var b *block.Reader

	if t.cache != nil {
		cache, _ = t.cache.Get(bi.offset, func() (ok bool, value interface{}, charge int, fin func()) {
			var buf []byte
			buf, err = bi.readAll(t.r, ro.HasFlag(opt.RFVerifyChecksums))
			if err != nil {
				return
			}
			b, err = block.NewReader(buf, t.o.GetComparer())
			if err != nil {
				return
			}
			ok = true
			value = b
			charge = int(bi.size)
			return
		})

		if err != nil {
			return
		}

		if b == nil {
			b = cache.Value().(*block.Reader)
		}
	} else {
		var buf []byte
		buf, err = bi.readAll(t.r, ro.HasFlag(opt.RFVerifyChecksums))
		if err != nil {
			return
		}
		b, err = block.NewReader(buf, t.o.GetComparer())
		if err != nil {
			return
		}
	}

	it = b.NewIterator()
	return
}

type indexIter struct {
	block.Iterator

	t  *Reader
	ro opt.ReadOptionsGetter
}

func (i *indexIter) Get() (it iter.Iterator, err error) {
	bi := new(bInfo)
	_, err = bi.decodeFrom(i.Value())
	if err != nil {
		return
	}

	x, cache, err := i.t.getDataIter(bi, i.ro)
	if err != nil {
		return
	}
	if cache != nil {
		runtime.SetFinalizer(x, func(x *block.Iterator) {
			cache.Release()
		})
	}
	return x, nil
}
<commit_msg>table: reader: respect RFDontFillCache read flag<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This LevelDB Go implementation is based on LevelDB C++ implementation.
// Which contains the following header:
//   Copyright (c) 2011 The LevelDB Authors. All rights reserved.
//   Use of this source code is governed by a BSD-style license that can be
//   found in the LEVELDBCPP_LICENSE file. See the LEVELDBCPP_AUTHORS file
//   for names of contributors.

// Package table allows read and write sorted key/value.
package table

import (
	"runtime"

	"leveldb/block"
	"leveldb/cache"
	"leveldb/comparer"
	"leveldb/desc"
	"leveldb/errors"
	"leveldb/iter"
	"leveldb/opt"
)

// Reader represent a table reader.
type Reader struct {
	r desc.Reader
	o opt.OptionsGetter

	meta   *block.Reader
	index  *block.Reader
	filter *block.FilterReader

	dataEnd uint64
	cache   cache.Namespace
}

// NewReader create new initialized table reader.
func NewReader(r desc.Reader, size uint64, o opt.OptionsGetter, cache cache.Namespace) (p *Reader, err error) {
	mb, ib, err := readFooter(r, size)
	if err != nil {
		return
	}

	t := &Reader{r: r, o: o, dataEnd: mb.offset, cache: cache}

	// index block
	buf, err := ib.readAll(r, true)
	if err != nil {
		return
	}
	t.index, err = block.NewReader(buf, o.GetComparer())
	if err != nil {
		return
	}

	// filter block
	filter := o.GetFilter()
	if filter != nil {
		// we will ignore any errors at meta/filter block
		// since it is not essential for operation

		// meta block
		buf, err = mb.readAll(r, true)
		if err != nil {
			goto out
		}
		var meta *block.Reader
		meta, err = block.NewReader(buf, comparer.BytesComparer{})
		if err != nil {

			goto out
		}

		// check for filter name
		iter := meta.NewIterator()
		key := "filter." + filter.Name()
		if iter.Seek([]byte(key)) && string(iter.Key()) == key {
			fb := new(bInfo)
			_, err = fb.decodeFrom(iter.Value())
			if err != nil {
				return
			}

			// now the data end before filter block start offset
			t.dataEnd = fb.offset

			// filter block
			buf, err = fb.readAll(r, true)
			if err != nil {
				goto out
			}
			t.filter, err = block.NewFilterReader(buf, filter)
			if err != nil {
				goto out
			}
		}
	}

out:
	return t, nil
}

// NewIterator create new iterator over the table.
func (t *Reader) NewIterator(ro opt.ReadOptionsGetter) iter.Iterator {
	index_iter := &indexIter{t: t, ro: ro}
	t.index.InitIterator(&index_iter.Iterator)
	return iter.NewIndexedIterator(index_iter)
}

// Get lookup for given key on the table. Get returns errors.ErrNotFound if
// given key did not exist.
func (t *Reader) Get(key []byte, ro opt.ReadOptionsGetter) (rkey, rvalue []byte, err error) {
	// create an iterator of index block
	index_iter := t.index.NewIterator()
	if !index_iter.Seek(key) {
		err = index_iter.Error()
		if err == nil {
			err = errors.ErrNotFound
		}
		return
	}

	// decode data block info
	bi := new(bInfo)
	_, err = bi.decodeFrom(index_iter.Value())
	if err != nil {
		return
	}

	// get the data block
	if t.filter == nil || t.filter.KeyMayMatch(uint(bi.offset), key) {
		var it iter.Iterator
		var cache cache.Object
		it, cache, err = t.getDataIter(bi, ro)
		if err != nil {
			return
		}
		if cache != nil {
			defer cache.Release()
		}

		// seek to key
		if !it.Seek(key) {
			err = it.Error()
			if err == nil {
				err = errors.ErrNotFound
			}
			return
		}
		rkey, rvalue = it.Key(), it.Value()
	} else {
		err = errors.ErrNotFound
	}
	return
}

// ApproximateOffsetOf approximate the offset of given key in bytes.
func (t *Reader) ApproximateOffsetOf(key []byte) uint64 {
	index_iter := t.index.NewIterator()
	if index_iter.Seek(key) {
		bi := new(bInfo)
		_, err := bi.decodeFrom(index_iter.Value())
		if err == nil {
			return bi.offset
		}
	}
	// block info is corrupted or key is past the last key in the file.
	// Approximate the offset by returning offset of the end of data
	// block (which is right near the end of the file).
	return t.dataEnd
}

func (t *Reader) getBlock(bi *bInfo, ro opt.ReadOptionsGetter) (b *block.Reader, err error) {
	buf, err := bi.readAll(t.r, ro.HasFlag(opt.RFVerifyChecksums))
	if err != nil {
		return
	}
	b, err = block.NewReader(buf, t.o.GetComparer())
	return
}

func (t *Reader) getDataIter(bi *bInfo, ro opt.ReadOptionsGetter) (it *block.Iterator, cache cache.Object, err error) {
	var b *block.Reader

	if t.cache != nil {
		var ok bool
		cache, ok = t.cache.Get(bi.offset, func() (ok bool, value interface{}, charge int, fin func()) {
			if ro.HasFlag(opt.RFDontFillCache) {
				return
			}
			b, err = t.getBlock(bi, ro)
			if err == nil {
				ok = true
				value = b
				charge = int(bi.size)
			}
			return
		})

		if err != nil {
			return
		}

		if !ok {
			b, err = t.getBlock(bi, ro)
			if err != nil {
				return
			}
		} else if b == nil {
			b = cache.Value().(*block.Reader)
		}
	} else {
		b, err = t.getBlock(bi, ro)
		if err != nil {
			return
		}
	}

	it = b.NewIterator()
	return
}

type indexIter struct {
	block.Iterator

	t  *Reader
	ro opt.ReadOptionsGetter
}

func (i *indexIter) Get() (it iter.Iterator, err error) {
	bi := new(bInfo)
	_, err = bi.decodeFrom(i.Value())
	if err != nil {
		return
	}

	x, cache, err := i.t.getDataIter(bi, i.ro)
	if err != nil {
		return
	}
	if cache != nil {
		runtime.SetFinalizer(x, func(x *block.Iterator) {
			cache.Release()
		})
	}
	return x, nil
}
<|endoftext|>


<commit_before>/*

Config is a simple library that manages config set-up for servers based on a config file.

*/
package config

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
)

type Config struct {
	Base *ConfigMode
	Dev  *ConfigMode
	Prod *ConfigMode
}

type ConfigMode struct {
	AllowedOrigins    string
	DefaultPort       string
	FirebaseProjectId string
	AdminUserIds      []string
	//This is a dangerous config. Only enable in Dev!
	DisableAdminChecking bool
	StorageConfig        map[string]string
}

//derive takes a raw input and creates a struct with fully derived values in
//Dev/Prod ready for use.
func (c *Config) derive() {
	if c.Base == nil {
		return
	}

	c.Prod = c.Base.extend(c.Prod)
	c.Dev = c.Base.extend(c.Dev)

}

func (c *Config) validate() error {

	if c.Dev == nil && c.Prod == nil {
		return errors.New("Neither dev nor prod configuration was valid")
	}

	if c.Dev != nil {
		if err := c.Dev.validate(true); err != nil {
			return err
		}
	}
	if c.Prod != nil {
		return c.Prod.validate(false)
	}
	return nil
}

func (c *ConfigMode) validate(isDev bool) error {
	if c.DefaultPort == "" {
		return errors.New("No default port provided")
	}
	//AllowedOrigins will just be default allow
	if c.AllowedOrigins == "" {
		log.Println("No AllowedOrigins found. Defaulting to '*'")
		c.AllowedOrigins = "*"
	}
	if c.StorageConfig == nil {
		c.StorageConfig = make(map[string]string)
	}
	if c.DisableAdminChecking && !isDev {
		return errors.New("DisableAdminChecking enabled in prod, which is illegal")
	}
	return nil
}

//copy returns a deep copy of the config mode.
func (c *ConfigMode) copy() *ConfigMode {

	result := &ConfigMode{}

	(*result) = *c
	result.AdminUserIds = make([]string, len(c.AdminUserIds))
	copy(result.AdminUserIds, c.AdminUserIds)
	result.StorageConfig = make(map[string]string, len(c.StorageConfig))
	for key, val := range c.StorageConfig {
		result.StorageConfig[key] = val
	}

	return result

}

//extend takes a given base config mode, extends it with properties set in
//other (with any non-zero value overwriting the base values) and returns a
//*new* config representing the merged one.
func (c *ConfigMode) extend(other *ConfigMode) *ConfigMode {

	result := c.copy()

	if other == nil {
		return result
	}

	if other.AllowedOrigins != "" {
		result.AllowedOrigins = other.AllowedOrigins
	}

	if other.DefaultPort != "" {
		result.DefaultPort = other.DefaultPort
	}

	if other.FirebaseProjectId != "" {
		result.FirebaseProjectId = other.FirebaseProjectId
	}

	if other.DisableAdminChecking {
		result.DisableAdminChecking = true
	}

	//Extend adminID, but no duplicates
	adminIdsSet := make(map[string]bool, len(result.AdminUserIds))
	for _, key := range result.AdminUserIds {
		adminIdsSet[key] = true
	}

	for _, key := range other.AdminUserIds {
		if adminIdsSet[key] {
			//Already in the set, don't add a duplicate
			continue
		}
		result.AdminUserIds = append(result.AdminUserIds, key)
	}

	for key, val := range other.StorageConfig {
		result.StorageConfig[key] = val
	}

	return result

}

func (c *ConfigMode) OriginAllowed(origin string) bool {

	originUrl, err := url.Parse(origin)

	if err != nil {
		return false
	}

	if c.AllowedOrigins == "" {
		return false
	}
	if c.AllowedOrigins == "*" {
		return true
	}
	allowedOrigins := strings.Split(c.AllowedOrigins, ",")
	for _, allowedOrigin := range allowedOrigins {
		u, err := url.Parse(allowedOrigin)

		if err != nil {
			continue
		}

		if u.Scheme == originUrl.Scheme && u.Host == originUrl.Host {
			return true
		}
	}
	return false
}

const (
	configFileName       = "config.SECRET.json"
	sampleConfigFileName = "config.SAMPLE.json"
)

func Get() (*Config, error) {

	fileNameToUse := configFileName

	if _, err := os.Stat(configFileName); os.IsNotExist(err) {

		if _, err := os.Stat(sampleConfigFileName); os.IsNotExist(err) {
			return nil, errors.New("Couldn't find a " + configFileName + " in current directory (or a SAMPLE). This file is required. Copy a starter one from boardgame/server/api/config.SAMPLE.json")
		}

		fileNameToUse = sampleConfigFileName

	}

	contents, err := ioutil.ReadFile(fileNameToUse)

	if err != nil {
		return nil, errors.New("Couldn't read config file: " + err.Error())
	}

	var config Config

	if err := json.Unmarshal(contents, &config); err != nil {
		return nil, errors.New("couldn't unmarshal config file: " + err.Error())
	}

	config.derive()

	if err := config.validate(); err != nil {
		return nil, errors.New("Couldn't validate config: " + err.Error())
	}

	return &config, nil

}
<commit_msg>Add Config.copy(), Config.extend(), which we'll shortly use for PUBLIC/SECRET config files. Part of #655.<commit_after>/*

Config is a simple library that manages config set-up for servers based on a config file.

*/
package config

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
)

type Config struct {
	Base *ConfigMode
	Dev  *ConfigMode
	Prod *ConfigMode
}

type ConfigMode struct {
	AllowedOrigins    string
	DefaultPort       string
	FirebaseProjectId string
	AdminUserIds      []string
	//This is a dangerous config. Only enable in Dev!
	DisableAdminChecking bool
	StorageConfig        map[string]string
}

//derive takes a raw input and creates a struct with fully derived values in
//Dev/Prod ready for use.
func (c *Config) derive() {
	if c.Base == nil {
		return
	}

	c.Prod = c.Base.extend(c.Prod)
	c.Dev = c.Base.extend(c.Dev)

}

func (c *Config) copy() *Config {
	return &Config{
		c.Base.copy(),
		c.Dev.copy(),
		c.Prod.copy(),
	}
}

//extend takes an other config and returns a *new* config where any non-zero
//value for other extends base.
func (c *Config) extend(other *Config) *Config {

	result := c.copy()

	if other == nil {
		return result
	}

	result.Base = c.Base.extend(other.Base)
	result.Dev = c.Dev.extend(other.Dev)
	result.Prod = c.Prod.extend(other.Prod)

	return result

}

func (c *Config) validate() error {

	if c.Dev == nil && c.Prod == nil {
		return errors.New("Neither dev nor prod configuration was valid")
	}

	if c.Dev != nil {
		if err := c.Dev.validate(true); err != nil {
			return err
		}
	}
	if c.Prod != nil {
		return c.Prod.validate(false)
	}
	return nil
}

func (c *ConfigMode) validate(isDev bool) error {
	if c.DefaultPort == "" {
		return errors.New("No default port provided")
	}
	//AllowedOrigins will just be default allow
	if c.AllowedOrigins == "" {
		log.Println("No AllowedOrigins found. Defaulting to '*'")
		c.AllowedOrigins = "*"
	}
	if c.StorageConfig == nil {
		c.StorageConfig = make(map[string]string)
	}
	if c.DisableAdminChecking && !isDev {
		return errors.New("DisableAdminChecking enabled in prod, which is illegal")
	}
	return nil
}

//copy returns a deep copy of the config mode.
func (c *ConfigMode) copy() *ConfigMode {

	if c == nil {
		return nil
	}

	result := &ConfigMode{}

	(*result) = *c
	result.AdminUserIds = make([]string, len(c.AdminUserIds))
	copy(result.AdminUserIds, c.AdminUserIds)
	result.StorageConfig = make(map[string]string, len(c.StorageConfig))
	for key, val := range c.StorageConfig {
		result.StorageConfig[key] = val
	}

	return result

}

//extend takes a given base config mode, extends it with properties set in
//other (with any non-zero value overwriting the base values) and returns a
//*new* config representing the merged one.
func (c *ConfigMode) extend(other *ConfigMode) *ConfigMode {

	result := c.copy()

	if other == nil {
		return result
	}

	if other.AllowedOrigins != "" {
		result.AllowedOrigins = other.AllowedOrigins
	}

	if other.DefaultPort != "" {
		result.DefaultPort = other.DefaultPort
	}

	if other.FirebaseProjectId != "" {
		result.FirebaseProjectId = other.FirebaseProjectId
	}

	if other.DisableAdminChecking {
		result.DisableAdminChecking = true
	}

	//Extend adminID, but no duplicates
	adminIdsSet := make(map[string]bool, len(result.AdminUserIds))
	for _, key := range result.AdminUserIds {
		adminIdsSet[key] = true
	}

	for _, key := range other.AdminUserIds {
		if adminIdsSet[key] {
			//Already in the set, don't add a duplicate
			continue
		}
		result.AdminUserIds = append(result.AdminUserIds, key)
	}

	for key, val := range other.StorageConfig {
		result.StorageConfig[key] = val
	}

	return result

}

func (c *ConfigMode) OriginAllowed(origin string) bool {

	originUrl, err := url.Parse(origin)

	if err != nil {
		return false
	}

	if c.AllowedOrigins == "" {
		return false
	}
	if c.AllowedOrigins == "*" {
		return true
	}
	allowedOrigins := strings.Split(c.AllowedOrigins, ",")
	for _, allowedOrigin := range allowedOrigins {
		u, err := url.Parse(allowedOrigin)

		if err != nil {
			continue
		}

		if u.Scheme == originUrl.Scheme && u.Host == originUrl.Host {
			return true
		}
	}
	return false
}

const (
	configFileName       = "config.SECRET.json"
	sampleConfigFileName = "config.SAMPLE.json"
)

func Get() (*Config, error) {

	fileNameToUse := configFileName

	if _, err := os.Stat(configFileName); os.IsNotExist(err) {

		if _, err := os.Stat(sampleConfigFileName); os.IsNotExist(err) {
			return nil, errors.New("Couldn't find a " + configFileName + " in current directory (or a SAMPLE). This file is required. Copy a starter one from boardgame/server/api/config.SAMPLE.json")
		}

		fileNameToUse = sampleConfigFileName

	}

	contents, err := ioutil.ReadFile(fileNameToUse)

	if err != nil {
		return nil, errors.New("Couldn't read config file: " + err.Error())
	}

	var config Config

	if err := json.Unmarshal(contents, &config); err != nil {
		return nil, errors.New("couldn't unmarshal config file: " + err.Error())
	}

	config.derive()

	if err := config.validate(); err != nil {
		return nil, errors.New("Couldn't validate config: " + err.Error())
	}

	return &config, nil

}
<|endoftext|>


<commit_before>package libcats

import "fmt"

func GetCats(name string) string {
	return fmt.Sprintf("Meow, I'm %s!\n", name)
}
<commit_msg>download an image of a cat<commit_after>package libcats

import (
	"fmt"
	"io/ioutil"

	"github.com/gregjones/httpcache"
	"github.com/gregjones/httpcache/diskcache"
	"github.com/peterbourgon/diskv"
)

var transport *httpcache.Transport

func Init(cachePath string) {
	transport = newTransportWithDiskCache(cachePath)
}

func GetCats(name string) string {
	return fmt.Sprintf("Meow, I'm %s!\n", name)
}

func DownloadCat() ([]byte, error) {

	resp, err := transport.Client().Get("http://i.imgur.com/3UD7Aqz.jpg")
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	fmt.Printf("from cache? %s\n", resp.Header.Get("X-From-Cache"))

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return body, nil
	// return base64.StdEncoding.EncodeToString(body), nil
}

func newTransportWithDiskCache(basePath string) *httpcache.Transport {
	d := diskv.New(diskv.Options{
		BasePath:     basePath,
		CacheSizeMax: 100 * 1024 * 10, // 10MB
	})

	c := diskcache.NewWithDiskv(d)

	return httpcache.NewTransport(c)
}
<|endoftext|>