<commit_before>// Package test contains simple test helpers that should not
// have any dependencies on horizon's packages.  think constants,
// custom matchers, generic helpers etc.
package test

import (
	"bytes"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	hlog "github.com/stellar/horizon/log"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

//go:generate go get github.com/jteeuwen/go-bindata/go-bindata
//go:generate go-bindata -pkg test scenarios

var (
	coreDB    *sqlx.DB
	horizonDB *sqlx.DB
)

const (
	// DefaultTestDatabaseURL is the default postgres connection string for
	// horizon's test database.
	DefaultTestDatabaseURL = "postgres://localhost:5432/horizon_test?sslmode=disable"

	// DefaultTestStellarCoreDatabaseURL is the default postgres connection string
	// for horizon's test stellar core database.
	DefaultTestStellarCoreDatabaseURL = "postgres://localhost:5432/stellar-core_test?sslmode=disable"
)

// StaticMockServer is a test helper that records it's last request
type StaticMockServer struct {
	*httptest.Server
	LastRequest *http.Request
}

// T provides a common set of functionality for each test in horizon
type T struct {
	T          *testing.T
	Assert     *assert.Assertions
	Require    *require.Assertions
	Ctx        context.Context
	HorizonDB  *sqlx.DB
	CoreDB     *sqlx.DB
	Logger     *hlog.Entry
	LogMetrics *hlog.Metrics
	LogBuffer  *bytes.Buffer
}

// Context provides a context suitable for testing in tests that do not create
// a full App instance (in which case your tests should be using the app's
// context).  This context has a logger bound to it suitable for testing.
func Context() context.Context {
	return hlog.Set(context.Background(), testLogger)
}

// ContextWithLogBuffer returns a context and a buffer into which the new, bound
// logger will write into.  This method allows you to inspect what data was
// logged more easily in your tests.
func ContextWithLogBuffer() (context.Context, *bytes.Buffer) {
	output := new(bytes.Buffer)
	l, _ := hlog.New()
	l.Logger.Out = output
	l.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	l.Logger.Level = logrus.DebugLevel

	ctx := hlog.Set(context.Background(), l)
	return ctx, output

}

// Database returns a connection to the horizon test database
func Database() *sqlx.DB {
	if horizonDB != nil {
		return horizonDB
	}
	horizonDB = OpenDatabase(DatabaseURL())
	return horizonDB
}

// DatabaseURL returns the database connection the url any test
// use when connecting to the history/horizon database
func DatabaseURL() string {
	databaseURL := os.Getenv("DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestDatabaseURL
	}

	return databaseURL
}

// LoadScenario populates the test databases with pre-created scenarios.  Each
// scenario is in the scenarios subfolder of this package and are a pair of
// sql files, one per database.
func LoadScenario(scenarioName string) {
	loadScenario(scenarioName, true)
}

// LoadScenarioWithoutHorizon populates the test Stellar core database a with
// pre-created scenario.  Unlike `LoadScenario`, this
func LoadScenarioWithoutHorizon(scenarioName string) {
	loadScenario(scenarioName, false)
}

// OpenDatabase opens a database, panicing if it cannot
func OpenDatabase(dsn string) *sqlx.DB {
	db, err := sqlx.Open("postgres", dsn)

	if err != nil {
		log.Panic(err)
	}

	return db
}

// Start initializes a new test helper object and conceptually "starts" a new
// test
func Start(t *testing.T) *T {
	result := &T{}

	result.T = t
	result.LogBuffer = new(bytes.Buffer)
	result.Logger, result.LogMetrics = hlog.New()
	result.Logger.Logger.Out = result.LogBuffer
	result.Logger.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	result.Logger.Logger.Level = logrus.DebugLevel

	result.Ctx = hlog.Set(context.Background(), result.Logger)
	result.HorizonDB = Database()
	result.CoreDB = StellarCoreDatabase()
	result.Assert = assert.New(t)
	result.Require = require.New(t)

	return result
}

// StellarCoreDatabase returns a connection to the stellar core test database
func StellarCoreDatabase() *sqlx.DB {
	if coreDB != nil {
		return coreDB
	}
	coreDB = OpenDatabase(StellarCoreDatabaseURL())
	return coreDB
}

// StellarCoreDatabaseURL returns the database connection the url any test
// use when connecting to the stellar-core database
func StellarCoreDatabaseURL() string {
	databaseURL := os.Getenv("STELLAR_CORE_DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestStellarCoreDatabaseURL
	}

	return databaseURL
}
<commit_msg>Ensure pq is included when the test helper is included<commit_after>// Package test contains simple test helpers that should not
// have any dependencies on horizon's packages.  think constants,
// custom matchers, generic helpers etc.
package test

import (
	"bytes"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	hlog "github.com/stellar/horizon/log"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

//go:generate go get github.com/jteeuwen/go-bindata/go-bindata
//go:generate go-bindata -pkg test scenarios

var (
	coreDB    *sqlx.DB
	horizonDB *sqlx.DB
)

const (
	// DefaultTestDatabaseURL is the default postgres connection string for
	// horizon's test database.
	DefaultTestDatabaseURL = "postgres://localhost:5432/horizon_test?sslmode=disable"

	// DefaultTestStellarCoreDatabaseURL is the default postgres connection string
	// for horizon's test stellar core database.
	DefaultTestStellarCoreDatabaseURL = "postgres://localhost:5432/stellar-core_test?sslmode=disable"
)

// StaticMockServer is a test helper that records it's last request
type StaticMockServer struct {
	*httptest.Server
	LastRequest *http.Request
}

// T provides a common set of functionality for each test in horizon
type T struct {
	T          *testing.T
	Assert     *assert.Assertions
	Require    *require.Assertions
	Ctx        context.Context
	HorizonDB  *sqlx.DB
	CoreDB     *sqlx.DB
	Logger     *hlog.Entry
	LogMetrics *hlog.Metrics
	LogBuffer  *bytes.Buffer
}

// Context provides a context suitable for testing in tests that do not create
// a full App instance (in which case your tests should be using the app's
// context).  This context has a logger bound to it suitable for testing.
func Context() context.Context {
	return hlog.Set(context.Background(), testLogger)
}

// ContextWithLogBuffer returns a context and a buffer into which the new, bound
// logger will write into.  This method allows you to inspect what data was
// logged more easily in your tests.
func ContextWithLogBuffer() (context.Context, *bytes.Buffer) {
	output := new(bytes.Buffer)
	l, _ := hlog.New()
	l.Logger.Out = output
	l.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	l.Logger.Level = logrus.DebugLevel

	ctx := hlog.Set(context.Background(), l)
	return ctx, output

}

// Database returns a connection to the horizon test database
func Database() *sqlx.DB {
	if horizonDB != nil {
		return horizonDB
	}
	horizonDB = OpenDatabase(DatabaseURL())
	return horizonDB
}

// DatabaseURL returns the database connection the url any test
// use when connecting to the history/horizon database
func DatabaseURL() string {
	databaseURL := os.Getenv("DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestDatabaseURL
	}

	return databaseURL
}

// LoadScenario populates the test databases with pre-created scenarios.  Each
// scenario is in the scenarios subfolder of this package and are a pair of
// sql files, one per database.
func LoadScenario(scenarioName string) {
	loadScenario(scenarioName, true)
}

// LoadScenarioWithoutHorizon populates the test Stellar core database a with
// pre-created scenario.  Unlike `LoadScenario`, this
func LoadScenarioWithoutHorizon(scenarioName string) {
	loadScenario(scenarioName, false)
}

// OpenDatabase opens a database, panicing if it cannot
func OpenDatabase(dsn string) *sqlx.DB {
	db, err := sqlx.Open("postgres", dsn)

	if err != nil {
		log.Panic(err)
	}

	return db
}

// Start initializes a new test helper object and conceptually "starts" a new
// test
func Start(t *testing.T) *T {
	result := &T{}

	result.T = t
	result.LogBuffer = new(bytes.Buffer)
	result.Logger, result.LogMetrics = hlog.New()
	result.Logger.Logger.Out = result.LogBuffer
	result.Logger.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	result.Logger.Logger.Level = logrus.DebugLevel

	result.Ctx = hlog.Set(context.Background(), result.Logger)
	result.HorizonDB = Database()
	result.CoreDB = StellarCoreDatabase()
	result.Assert = assert.New(t)
	result.Require = require.New(t)

	return result
}

// StellarCoreDatabase returns a connection to the stellar core test database
func StellarCoreDatabase() *sqlx.DB {
	if coreDB != nil {
		return coreDB
	}
	coreDB = OpenDatabase(StellarCoreDatabaseURL())
	return coreDB
}

// StellarCoreDatabaseURL returns the database connection the url any test
// use when connecting to the stellar-core database
func StellarCoreDatabaseURL() string {
	databaseURL := os.Getenv("STELLAR_CORE_DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestStellarCoreDatabaseURL
	}

	return databaseURL
}
<|endoftext|>


<commit_before>// Package test contains simple test helpers that should not
// have any dependencies on horizon's packages.  think constants,
// custom matchers, generic helpers etc.
package test

import (
	"bytes"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	hlog "github.com/stellar/horizon/log"
	tdb "github.com/stellar/horizon/test/db"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

// StaticMockServer is a test helper that records it's last request
type StaticMockServer struct {
	*httptest.Server
	LastRequest *http.Request
}

// T provides a common set of functionality for each test in horizon
type T struct {
	T          *testing.T
	Assert     *assert.Assertions
	Require    *require.Assertions
	Ctx        context.Context
	HorizonDB  *sqlx.DB
	CoreDB     *sqlx.DB
	Logger     *hlog.Entry
	LogMetrics *hlog.Metrics
	LogBuffer  *bytes.Buffer
}

// Context provides a context suitable for testing in tests that do not create
// a full App instance (in which case your tests should be using the app's
// context).  This context has a logger bound to it suitable for testing.
func Context() context.Context {
	return hlog.Set(context.Background(), testLogger)
}

// ContextWithLogBuffer returns a context and a buffer into which the new, bound
// logger will write into.  This method allows you to inspect what data was
// logged more easily in your tests.
func ContextWithLogBuffer() (context.Context, *bytes.Buffer) {
	output := new(bytes.Buffer)
	l, _ := hlog.New()
	l.Logger.Out = output
	l.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	l.Logger.Level = logrus.DebugLevel

	ctx := hlog.Set(context.Background(), l)
	return ctx, output

}

// Database returns a connection to the horizon test database
//
// DEPRECATED:  use `Horizon()` from test/db package
func Database() *sqlx.DB {
	return tdb.Horizon()
}

// DatabaseURL returns the database connection the url any test
// use when connecting to the history/horizon database
//
// DEPRECATED:  use `HorizonURL()` from test/db package
func DatabaseURL() string {
	return tdb.HorizonURL()
}

// LoadScenario populates the test databases with pre-created scenarios.  Each
// scenario is in the scenarios subfolder of this package and are a pair of
// sql files, one per database.
func LoadScenario(scenarioName string) {
	loadScenario(scenarioName, true)
}

// LoadScenarioWithoutHorizon populates the test Stellar core database a with
// pre-created scenario.  Unlike `LoadScenario`, this
func LoadScenarioWithoutHorizon(scenarioName string) {
	loadScenario(scenarioName, false)
}

func OverrideLogger(l *hlog.Entry) {
	if oldDefault != nil {
		panic("logger already overridden")
	}

	oldDefault = hlog.DefaultLogger
	hlog.DefaultLogger = l
}

func RestoreLogger() {
	if oldDefault == nil {
		panic("logger not overridden, cannot restore")
	}

	hlog.DefaultLogger = oldDefault
	oldDefault = nil
}

// Start initializes a new test helper object and conceptually "starts" a new
// test
func Start(t *testing.T) *T {
	result := &T{}

	result.T = t
	result.LogBuffer = new(bytes.Buffer)
	result.Logger, result.LogMetrics = hlog.New()
	result.Logger.Logger.Out = result.LogBuffer
	result.Logger.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	result.Logger.Logger.Level = logrus.DebugLevel

	OverrideLogger(result.Logger)

	result.Ctx = hlog.Set(context.Background(), result.Logger)
	result.HorizonDB = Database()
	result.CoreDB = StellarCoreDatabase()
	result.Assert = assert.New(t)
	result.Require = require.New(t)

	return result
}

// StellarCoreDatabase returns a connection to the stellar core test database
//
// DEPRECATED:  use `StellarCore()` from test/db package
func StellarCoreDatabase() *sqlx.DB {
	return tdb.StellarCore()
}

// StellarCoreDatabaseURL returns the database connection the url any test
// use when connecting to the stellar-core database
//
// DEPRECATED:  use `StellarCoreURL()` from test/db package
func StellarCoreDatabaseURL() string {
	return tdb.StellarCoreURL()
}

var oldDefault *hlog.Entry = nil
<commit_msg>Delint<commit_after>// Package test contains simple test helpers that should not
// have any dependencies on horizon's packages.  think constants,
// custom matchers, generic helpers etc.
package test

import (
	"bytes"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	hlog "github.com/stellar/horizon/log"
	tdb "github.com/stellar/horizon/test/db"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

// StaticMockServer is a test helper that records it's last request
type StaticMockServer struct {
	*httptest.Server
	LastRequest *http.Request
}

// T provides a common set of functionality for each test in horizon
type T struct {
	T          *testing.T
	Assert     *assert.Assertions
	Require    *require.Assertions
	Ctx        context.Context
	HorizonDB  *sqlx.DB
	CoreDB     *sqlx.DB
	Logger     *hlog.Entry
	LogMetrics *hlog.Metrics
	LogBuffer  *bytes.Buffer
}

// Context provides a context suitable for testing in tests that do not create
// a full App instance (in which case your tests should be using the app's
// context).  This context has a logger bound to it suitable for testing.
func Context() context.Context {
	return hlog.Set(context.Background(), testLogger)
}

// ContextWithLogBuffer returns a context and a buffer into which the new, bound
// logger will write into.  This method allows you to inspect what data was
// logged more easily in your tests.
func ContextWithLogBuffer() (context.Context, *bytes.Buffer) {
	output := new(bytes.Buffer)
	l, _ := hlog.New()
	l.Logger.Out = output
	l.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	l.Logger.Level = logrus.DebugLevel

	ctx := hlog.Set(context.Background(), l)
	return ctx, output

}

// Database returns a connection to the horizon test database
//
// DEPRECATED:  use `Horizon()` from test/db package
func Database() *sqlx.DB {
	return tdb.Horizon()
}

// DatabaseURL returns the database connection the url any test
// use when connecting to the history/horizon database
//
// DEPRECATED:  use `HorizonURL()` from test/db package
func DatabaseURL() string {
	return tdb.HorizonURL()
}

// LoadScenario populates the test databases with pre-created scenarios.  Each
// scenario is in the scenarios subfolder of this package and are a pair of
// sql files, one per database.
func LoadScenario(scenarioName string) {
	loadScenario(scenarioName, true)
}

// LoadScenarioWithoutHorizon populates the test Stellar core database a with
// pre-created scenario.  Unlike `LoadScenario`, this
func LoadScenarioWithoutHorizon(scenarioName string) {
	loadScenario(scenarioName, false)
}

// OverrideLogger sets the default logger used by horizon to `l`.  This is used
// by the testing system so that we can collect output from logs during test
// runs.  Panics if the logger is already overridden.
func OverrideLogger(l *hlog.Entry) {
	if oldDefault != nil {
		panic("logger already overridden")
	}

	oldDefault = hlog.DefaultLogger
	hlog.DefaultLogger = l
}

// RestoreLogger restores the default horizon logger after it is overridden
// using a call to `OverrideLogger`.  Panics if the default logger is not
// presently overridden.
func RestoreLogger() {
	if oldDefault == nil {
		panic("logger not overridden, cannot restore")
	}

	hlog.DefaultLogger = oldDefault
	oldDefault = nil
}

// Start initializes a new test helper object and conceptually "starts" a new
// test
func Start(t *testing.T) *T {
	result := &T{}

	result.T = t
	result.LogBuffer = new(bytes.Buffer)
	result.Logger, result.LogMetrics = hlog.New()
	result.Logger.Logger.Out = result.LogBuffer
	result.Logger.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	result.Logger.Logger.Level = logrus.DebugLevel

	OverrideLogger(result.Logger)

	result.Ctx = hlog.Set(context.Background(), result.Logger)
	result.HorizonDB = Database()
	result.CoreDB = StellarCoreDatabase()
	result.Assert = assert.New(t)
	result.Require = require.New(t)

	return result
}

// StellarCoreDatabase returns a connection to the stellar core test database
//
// DEPRECATED:  use `StellarCore()` from test/db package
func StellarCoreDatabase() *sqlx.DB {
	return tdb.StellarCore()
}

// StellarCoreDatabaseURL returns the database connection the url any test
// use when connecting to the stellar-core database
//
// DEPRECATED:  use `StellarCoreURL()` from test/db package
func StellarCoreDatabaseURL() string {
	return tdb.StellarCoreURL()
}

var oldDefault *hlog.Entry = nil
<|endoftext|>


<commit_before>package parser

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cznic/c/internal/cc"
	"github.com/cznic/c/internal/xc"
)

type Config struct {
	Arch            TargetArch
	DefinesPath     string
	IncludePaths    []string
	SysIncludePaths []string
	TargetPaths     []string
}

func NewConfig(paths ...string) *Config {
	return &Config{
		TargetPaths: paths,
	}
}

func checkConfig(cfg *Config) *Config {
	if cfg == nil {
		cfg = &Config{}
	}
	if cfg.Arch != Arch32 && cfg.Arch != Arch64 {
		// default to 64-bit arch
		cfg.Arch = Arch64
	}
	return cfg
}

type Parser struct {
	cfg        *Config
	predefined string
	model      cc.Model
}

func New(cfg *Config) (*Parser, error) {
	p := &Parser{
		cfg: checkConfig(cfg),
	}
	if len(cfg.TargetPaths) > 0 {
		// workaround for cznic's cc (it panics if supplied path is a dir)
		var saneFiles []string
		for _, path := range cfg.TargetPaths {
			if info, err := os.Stat(path); err != nil || info.IsDir() {
				continue
			}
			if absPath, err := filepath.Abs(path); err != nil {
				path = absPath
			}
			saneFiles = append(saneFiles, path)
		}
		cfg.TargetPaths = saneFiles
	}
	if len(cfg.TargetPaths) == 0 {
		return nil, errors.New("parser: no target paths specified")
	}
	if model, ok := models[cfg.Arch]; !ok {
		return nil, errors.New(fmt.Sprintf("parser: the target model not defined: %v", cfg.Arch))
	} else {
		p.model = model
	}
	if def, ok := predefines[cfg.Arch]; !ok {
		p.predefined = predefinedBase
	} else {
		p.predefined = def
	}
	if len(cfg.IncludePaths) == 0 {
		cfg.IncludePaths = []string{"."}
	}
	cfg.SysIncludePaths = []string{"/usr/include", "/usr/local/Cellar/libvpx/1.4.0/include/vpx"}
	if len(cfg.DefinesPath) > 0 {
		if buf, err := ioutil.ReadFile(cfg.DefinesPath); err != nil {
			return nil, errors.New("parser: custom defines file provided but can't be read")
		} else if len(buf) > 0 {
			p.predefined = fmt.Sprintf("%s\n// custom defines below\n%s", p.predefined, buf)
		}
	}
	return p, nil
}

func (p *Parser) Parse() (unit *cc.TranslationUnit, macros []string, err error) {
	unit, err = cc.Parse(p.predefined, p.cfg.TargetPaths, p.model,
		cc.IncludePaths(p.cfg.IncludePaths),
		cc.SysIncludePaths(p.cfg.SysIncludePaths))
	if err != nil {
		unit = nil
		return
	}
	for id := range cc.Macros {
		if macro := xc.Dict.S(id); len(macro) > 0 {
			macros = append(macros, string(macro))
		}
	}
	return
}
<commit_msg>[]int macros in parser<commit_after>package parser

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cznic/c/internal/cc"
)

type Config struct {
	Arch            TargetArch
	DefinesPath     string
	IncludePaths    []string
	SysIncludePaths []string
	TargetPaths     []string
}

func NewConfig(paths ...string) *Config {
	return &Config{
		TargetPaths: paths,
	}
}

func checkConfig(cfg *Config) *Config {
	if cfg == nil {
		cfg = &Config{}
	}
	if cfg.Arch != Arch32 && cfg.Arch != Arch64 {
		// default to 64-bit arch
		cfg.Arch = Arch64
	}
	return cfg
}

type Parser struct {
	cfg        *Config
	predefined string
	model      cc.Model
}

func New(cfg *Config) (*Parser, error) {
	p := &Parser{
		cfg: checkConfig(cfg),
	}
	if len(cfg.TargetPaths) > 0 {
		// workaround for cznic's cc (it panics if supplied path is a dir)
		var saneFiles []string
		for _, path := range cfg.TargetPaths {
			if info, err := os.Stat(path); err != nil || info.IsDir() {
				continue
			}
			if absPath, err := filepath.Abs(path); err != nil {
				path = absPath
			}
			saneFiles = append(saneFiles, path)
		}
		cfg.TargetPaths = saneFiles
	}
	if len(cfg.TargetPaths) == 0 {
		return nil, errors.New("parser: no target paths specified")
	}
	if model, ok := models[cfg.Arch]; !ok {
		return nil, errors.New(fmt.Sprintf("parser: the target model not defined: %v", cfg.Arch))
	} else {
		p.model = model
	}
	if def, ok := predefines[cfg.Arch]; !ok {
		p.predefined = predefinedBase
	} else {
		p.predefined = def
	}
	if len(cfg.IncludePaths) == 0 {
		cfg.IncludePaths = []string{"."}
	}
	if len(cfg.DefinesPath) > 0 {
		if buf, err := ioutil.ReadFile(cfg.DefinesPath); err != nil {
			return nil, errors.New("parser: custom defines file provided but can't be read")
		} else if len(buf) > 0 {
			p.predefined = fmt.Sprintf("%s\n// custom defines below\n%s", p.predefined, buf)
		}
	}
	return p, nil
}

func (p *Parser) Parse() (unit *cc.TranslationUnit, macros []int, err error) {
	unit, err = cc.Parse(p.predefined, p.cfg.TargetPaths, p.model,
		cc.IncludePaths(p.cfg.IncludePaths),
		cc.SysIncludePaths(p.cfg.SysIncludePaths))
	if err != nil {
		unit = nil
		return
	}
	for id := range cc.Macros {
		macros = append(macros, id)
	}
	return
}
<|endoftext|>


<commit_before>package api

import (
	"errors"
	"fmt"
	"net/http"

	"github.com/github/git-lfs/httputil"
)

type ObjectError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

func (e *ObjectError) Error() string {
	return fmt.Sprintf("[%d] %s", e.Code, e.Message)
}

type ObjectResource struct {
	Oid     string                   `json:"oid,omitempty"`
	Size    int64                    `json:"size"`
	Actions map[string]*LinkRelation `json:"actions,omitempty"`
	Links   map[string]*LinkRelation `json:"_links,omitempty"`
	Error   *ObjectError             `json:"error,omitempty"`
}

// TODO LEGACY API: remove when legacy API removed
func (o *ObjectResource) NewRequest(relation, method string) (*http.Request, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		if relation == "download" {
			return nil, errors.New("Object not found on the server.")
		}
		return nil, fmt.Errorf("No %q action for this object.", relation)

	}

	req, err := httputil.NewHttpRequest(method, rel.Href, rel.Header)
	if err != nil {
		return nil, err
	}

	return req, nil
}

func (o *ObjectResource) Rel(name string) (*LinkRelation, bool) {
	var rel *LinkRelation
	var ok bool

	if o.Actions != nil {
		rel, ok = o.Actions[name]
	} else {
		rel, ok = o.Links[name]
	}

	return rel, ok
}

type LinkRelation struct {
	Href   string            `json:"href"`
	Header map[string]string `json:"header,omitempty"`
}
<commit_msg>api: support ExpiresAt property on Actions<commit_after>package api

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/github/git-lfs/httputil"
)

type ObjectError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

func (e *ObjectError) Error() string {
	return fmt.Sprintf("[%d] %s", e.Code, e.Message)
}

type ObjectResource struct {
	Oid     string                   `json:"oid,omitempty"`
	Size    int64                    `json:"size"`
	Actions map[string]*LinkRelation `json:"actions,omitempty"`
	Links   map[string]*LinkRelation `json:"_links,omitempty"`
	Error   *ObjectError             `json:"error,omitempty"`
}

// TODO LEGACY API: remove when legacy API removed
func (o *ObjectResource) NewRequest(relation, method string) (*http.Request, error) {
	rel, ok := o.Rel(relation)
	if !ok {
		if relation == "download" {
			return nil, errors.New("Object not found on the server.")
		}
		return nil, fmt.Errorf("No %q action for this object.", relation)

	}

	req, err := httputil.NewHttpRequest(method, rel.Href, rel.Header)
	if err != nil {
		return nil, err
	}

	return req, nil
}

func (o *ObjectResource) Rel(name string) (*LinkRelation, bool) {
	var rel *LinkRelation
	var ok bool

	if o.Actions != nil {
		rel, ok = o.Actions[name]
	} else {
		rel, ok = o.Links[name]
	}

	return rel, ok
}

type LinkRelation struct {
	Href      string            `json:"href"`
	Header    map[string]string `json:"header,omitempty"`
	ExpiresAt time.Time         `json:"expires_at,omitempty"`
}
<|endoftext|>


<commit_before>package fronted

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/idletiming"
	"github.com/getlantern/tlsdialer"
)

var (
	poolCh      = make(chan *x509.CertPool, 1)
	candidateCh = make(chan *Masquerade, 1)
	masqCh      = make(chan *Masquerade, 1)
)

func Configure(pool *x509.CertPool, masquerades map[string][]*Masquerade) {
	if masquerades == nil || len(masquerades) == 0 {
		log.Errorf("No masquerades!!")
	}

	// Make a copy of the masquerades to avoid data races.
	masq := make(map[string][]*Masquerade)
	for k, v := range masquerades {
		c := make([]*Masquerade, len(v))
		copy(c, v)
		masq[k] = c
	}
	size := 0
	for _, arr := range masq {
		shuffle(arr)
		size += len(arr)
	}

	// Make an unblocked channel the same size as our group
	// of masquerades and push all of them into it.
	candidateCh = make(chan *Masquerade, size)

	log.Debugf("Adding %v candidates...", size)
	for _, arr := range masq {
		for _, m := range arr {
			candidateCh <- m
		}
	}
	poolCh <- pool
}

func shuffle(slc []*Masquerade) {
	n := len(slc)
	for i := 0; i < n; i++ {
		// choose index uniformly in [i, n-1]
		r := i + rand.Intn(n-i)
		slc[r], slc[i] = slc[i], slc[r]
	}
}

func getCertPool() *x509.CertPool {
	pool := <-poolCh
	if len(poolCh) == 0 {
		poolCh <- pool
	}
	return pool
}

type direct struct {
	tlsConfigs      map[string]*tls.Config
	tlsConfigsMutex sync.Mutex
}

func NewDirect() *direct {
	d := &direct{
		tlsConfigs: make(map[string]*tls.Config),
	}
	return d
}

// directTransport is a wrapper struct enabling us to modify the protocol of outgoing
// requests to make them all HTTP instead of potentially HTTPS, which breaks our particular
// implemenation of direct domain fronting.
type directTransport struct {
	http.Transport
}

func (ddf *directTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	// The connection is already encrypted by domain fronting.  We need to rewrite URLs starting
	// with "https://" to "http://", lest we get an error for doubling up on TLS.

	// The RoundTrip interface requires that we not modify the memory in the request, so we just
	// create a copy.
	norm := new(http.Request)
	*norm = *req // includes shallow copies of maps, but okay
	norm.URL = new(url.URL)
	*norm.URL = *req.URL
	norm.URL.Scheme = "http"
	return ddf.Transport.RoundTrip(norm)
}

// NewDirectHttpClient creates a new http.Client that does direct domain fronting.
func (d *direct) NewDirectHttpClient() *http.Client {
	trans := &directTransport{}
	trans.Dial = d.Dial
	trans.TLSHandshakeTimeout = 40 * time.Second
	trans.DisableKeepAlives = true
	return &http.Client{
		Transport: trans,
	}
}

// Do continually retries a given request until it succeeds because some fronting providers
// will return a 403 for some domains.
func (d *direct) Do(req *http.Request) (*http.Response, error) {
	for i := 0; i < 6; i++ {
		client := d.NewDirectHttpClient()
		if resp, err := client.Do(req); err != nil {
			log.Errorf("Could not complete request %v", err)
		} else if resp.StatusCode > 199 && resp.StatusCode < 400 {
			return resp, err
		} else {
			_ = resp.Body.Close()
		}
	}
	return nil, errors.New("Could not complete request even with retries")
}

// Dial persistently dials masquerades until one succeeds.
func (d *direct) Dial(network, addr string) (net.Conn, error) {
	for {
		select {
		case m := <-candidateCh:
			log.Debugf("Dialing to %v", m)

			// We do the full TLS connection here because in practice the domains at a given IP
			// address can change frequently on CDNs, so the certificate may not match what
			// we expect.
			conn, err := d.dialServerWith(m)
			if err != nil {
				log.Debugf("Could not dial to %v, %v", m.IpAddress, err)
				// Don't re-add this candidate if it's any certificate error, as that
				// will just keep failing and will waste connections. We can't access the underlying
				// error at this point so just look for "certificate".
				if strings.Contains(err.Error(), "certificate") {
					log.Debugf("Continuing on certificate error")
				} else {
					candidateCh <- m
				}
			} else {
				log.Debugf("Got successful connection to: %v", m)
				// Requeue the working connection
				candidateCh <- m
				idleTimeout := 70 * time.Second

				log.Debug("Wrapping connecting in idletiming connection")
				conn = idletiming.Conn(conn, idleTimeout, func() {
					log.Debugf("Connection to %s via %s idle for %v, closing", addr, conn.RemoteAddr(), idleTimeout)
					if err := conn.Close(); err != nil {
						log.Debugf("Unable to close connection: %v", err)
					}
				})
				log.Debug("Returning connection")
				return conn, nil
			}
		default:
			return nil, errors.New("Could not dial any masquerade?")
		}
	}
}

func (d *direct) dialServerWith(masquerade *Masquerade) (net.Conn, error) {
	tlsConfig := d.tlsConfig(masquerade)
	dialTimeout := 30 * time.Second
	sendServerNameExtension := false

	cwt, err := tlsdialer.DialForTimings(
		&net.Dialer{
			Timeout: dialTimeout,
		},
		"tcp",
		masquerade.IpAddress+":443",
		sendServerNameExtension, // SNI or no
		tlsConfig)

	if err != nil && masquerade != nil {
		err = fmt.Errorf("Unable to dial masquerade %s: %s", masquerade.Domain, err)
	}
	return cwt.Conn, err
}

// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *direct) tlsConfig(m *Masquerade) *tls.Config {
	d.tlsConfigsMutex.Lock()
	defer d.tlsConfigsMutex.Unlock()

	tlsConfig := d.tlsConfigs[m.Domain]
	if tlsConfig == nil {
		tlsConfig = &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(1000),
			InsecureSkipVerify: false,
			ServerName:         m.Domain,
			RootCAs:            getCertPool(),
		}
		d.tlsConfigs[m.Domain] = tlsConfig
	}

	return tlsConfig
}
<commit_msg>Only giving up on dialing direct after we've already gotten at least one candidate<commit_after>package fronted

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/idletiming"
	"github.com/getlantern/tlsdialer"
)

var (
	poolCh      = make(chan *x509.CertPool, 1)
	candidateCh = make(chan *Masquerade, 1)
	masqCh      = make(chan *Masquerade, 1)
)

func Configure(pool *x509.CertPool, masquerades map[string][]*Masquerade) {
	if masquerades == nil || len(masquerades) == 0 {
		log.Errorf("No masquerades!!")
	}

	// Make a copy of the masquerades to avoid data races.
	masq := make(map[string][]*Masquerade)
	for k, v := range masquerades {
		c := make([]*Masquerade, len(v))
		copy(c, v)
		masq[k] = c
	}
	size := 0
	for _, arr := range masq {
		shuffle(arr)
		size += len(arr)
	}

	// Make an unblocked channel the same size as our group
	// of masquerades and push all of them into it.
	candidateCh = make(chan *Masquerade, size)

	go func() {
		log.Debugf("Adding %v candidates...", size)
		for _, arr := range masq {
			for _, m := range arr {
				candidateCh <- m
			}
		}
		poolCh <- pool
	}()
}

func shuffle(slc []*Masquerade) {
	n := len(slc)
	for i := 0; i < n; i++ {
		// choose index uniformly in [i, n-1]
		r := i + rand.Intn(n-i)
		slc[r], slc[i] = slc[i], slc[r]
	}
}

func getCertPool() *x509.CertPool {
	pool := <-poolCh
	if len(poolCh) == 0 {
		poolCh <- pool
	}
	return pool
}

type direct struct {
	tlsConfigs      map[string]*tls.Config
	tlsConfigsMutex sync.Mutex
}

func NewDirect() *direct {
	d := &direct{
		tlsConfigs: make(map[string]*tls.Config),
	}
	return d
}

// directTransport is a wrapper struct enabling us to modify the protocol of outgoing
// requests to make them all HTTP instead of potentially HTTPS, which breaks our particular
// implemenation of direct domain fronting.
type directTransport struct {
	http.Transport
}

func (ddf *directTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	// The connection is already encrypted by domain fronting.  We need to rewrite URLs starting
	// with "https://" to "http://", lest we get an error for doubling up on TLS.

	// The RoundTrip interface requires that we not modify the memory in the request, so we just
	// create a copy.
	norm := new(http.Request)
	*norm = *req // includes shallow copies of maps, but okay
	norm.URL = new(url.URL)
	*norm.URL = *req.URL
	norm.URL.Scheme = "http"
	return ddf.Transport.RoundTrip(norm)
}

// NewDirectHttpClient creates a new http.Client that does direct domain fronting.
func (d *direct) NewDirectHttpClient() *http.Client {
	trans := &directTransport{}
	trans.Dial = d.Dial
	trans.TLSHandshakeTimeout = 40 * time.Second
	trans.DisableKeepAlives = true
	return &http.Client{
		Transport: trans,
	}
}

// Do continually retries a given request until it succeeds because some fronting providers
// will return a 403 for some domains.
func (d *direct) Do(req *http.Request) (*http.Response, error) {
	for i := 0; i < 6; i++ {
		client := d.NewDirectHttpClient()
		if resp, err := client.Do(req); err != nil {
			log.Errorf("Could not complete request %v", err)
		} else if resp.StatusCode > 199 && resp.StatusCode < 400 {
			return resp, err
		} else {
			_ = resp.Body.Close()
		}
	}
	return nil, errors.New("Could not complete request even with retries")
}

// Dial persistently dials masquerades until one succeeds.
func (d *direct) Dial(network, addr string) (net.Conn, error) {
	gotFirst := false
	for {
		select {
		case m := <-candidateCh:
			gotFirst = true
			log.Debugf("Dialing to %v", m)

			// We do the full TLS connection here because in practice the domains at a given IP
			// address can change frequently on CDNs, so the certificate may not match what
			// we expect.
			conn, err := d.dialServerWith(m)
			if err != nil {
				log.Debugf("Could not dial to %v, %v", m.IpAddress, err)
				// Don't re-add this candidate if it's any certificate error, as that
				// will just keep failing and will waste connections. We can't access the underlying
				// error at this point so just look for "certificate".
				if strings.Contains(err.Error(), "certificate") {
					log.Debugf("Continuing on certificate error")
				} else {
					candidateCh <- m
				}
			} else {
				log.Debugf("Got successful connection to: %v", m)
				// Requeue the working connection
				candidateCh <- m
				idleTimeout := 70 * time.Second

				log.Debug("Wrapping connecting in idletiming connection")
				conn = idletiming.Conn(conn, idleTimeout, func() {
					log.Debugf("Connection to %s via %s idle for %v, closing", addr, conn.RemoteAddr(), idleTimeout)
					if err := conn.Close(); err != nil {
						log.Debugf("Unable to close connection: %v", err)
					}
				})
				log.Debug("Returning connection")
				return conn, nil
			}
		default:
			if gotFirst {
				return nil, errors.New("Could not dial any masquerade?")
			}
		}
	}
}

func (d *direct) dialServerWith(masquerade *Masquerade) (net.Conn, error) {
	tlsConfig := d.tlsConfig(masquerade)
	dialTimeout := 30 * time.Second
	sendServerNameExtension := false

	cwt, err := tlsdialer.DialForTimings(
		&net.Dialer{
			Timeout: dialTimeout,
		},
		"tcp",
		masquerade.IpAddress+":443",
		sendServerNameExtension, // SNI or no
		tlsConfig)

	if err != nil && masquerade != nil {
		err = fmt.Errorf("Unable to dial masquerade %s: %s", masquerade.Domain, err)
	}
	return cwt.Conn, err
}

// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *direct) tlsConfig(m *Masquerade) *tls.Config {
	d.tlsConfigsMutex.Lock()
	defer d.tlsConfigsMutex.Unlock()

	tlsConfig := d.tlsConfigs[m.Domain]
	if tlsConfig == nil {
		tlsConfig = &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(1000),
			InsecureSkipVerify: false,
			ServerName:         m.Domain,
			RootCAs:            getCertPool(),
		}
		d.tlsConfigs[m.Domain] = tlsConfig
	}

	return tlsConfig
}
<|endoftext|>


<commit_before>package cienv

import (
	"encoding/json"
	"errors"
	"os"
)

// https://help.github.com/en/articles/contexts-and-expression-syntax-for-github-actions#github-context
type GitHubContext struct {
	Sha       string `json:"sha"`
	HeadRef   string `json:"head_ref"`
	EventName string `json:"event_name"`
	Event     struct {
		Number      int `json:"number"`
		PullRequest struct {
			Head struct {
				Sha string `json:"sha"`
			} `json:"head"`
		} `json:"pull_request"`
		Repository struct {
			Owner struct {
				Login string `json:"login"`
			} `json:"owner"`
			Name string `json:"name"`
		} `json:"repository"`
	} `json:"event"`
}

func getBuildInfoFromGitHubAction() (prInfo *BuildInfo, isPR bool, err error) {
	eventPath := os.Getenv("GITHUB_EVENT_PATH")
	if eventPath == "" {
		return nil, false, errors.New("GITHUB_EVENT_PATH not found")
	}
	return getBuildInfoFromGitHubActionEventPath(eventPath)
}
func getBuildInfoFromGitHubActionEventPath(eventPath string) (prInfo *BuildInfo, isPR bool, err error) {
	f, err := os.Open(eventPath)
	if err != nil {
		return nil, false, err
	}
	defer f.Close()
	var ghCtx GitHubContext
	if err := json.NewDecoder(f).Decode(&ghCtx); err != nil {
		return nil, false, err
	}
	info := &BuildInfo{
		Owner:       ghCtx.Event.Repository.Owner.Login,
		Repo:        ghCtx.Event.Repository.Name,
		PullRequest: ghCtx.Event.Number,
		Branch:      ghCtx.HeadRef,
	}
	if ghCtx.Event.PullRequest.Head.Sha != "" {
		info.SHA = ghCtx.Event.PullRequest.Head.Sha
	} else {
		info.SHA = ghCtx.Sha
	}
	return info, info.PullRequest != 0, nil
}

func IsInGitHubAction() bool {
	// https://help.github.com/en/articles/virtual-environments-for-github-actions#default-environment-variables
	return os.Getenv("GITHUB_ACTION") != ""
}
<commit_msg>Stop using named return<commit_after>package cienv

import (
	"encoding/json"
	"errors"
	"os"
)

// https://help.github.com/en/articles/contexts-and-expression-syntax-for-github-actions#github-context
type GitHubContext struct {
	Sha       string `json:"sha"`
	HeadRef   string `json:"head_ref"`
	EventName string `json:"event_name"`
	Event     struct {
		Number      int `json:"number"`
		PullRequest struct {
			Head struct {
				Sha string `json:"sha"`
			} `json:"head"`
		} `json:"pull_request"`
		Repository struct {
			Owner struct {
				Login string `json:"login"`
			} `json:"owner"`
			Name string `json:"name"`
		} `json:"repository"`
	} `json:"event"`
}

func getBuildInfoFromGitHubAction() (*BuildInfo, bool, error) {
	eventPath := os.Getenv("GITHUB_EVENT_PATH")
	if eventPath == "" {
		return nil, false, errors.New("GITHUB_EVENT_PATH not found")
	}
	return getBuildInfoFromGitHubActionEventPath(eventPath)
}
func getBuildInfoFromGitHubActionEventPath(eventPath string) (*BuildInfo, bool, error) {
	f, err := os.Open(eventPath)
	if err != nil {
		return nil, false, err
	}
	defer f.Close()
	var ghCtx GitHubContext
	if err := json.NewDecoder(f).Decode(&ghCtx); err != nil {
		return nil, false, err
	}
	info := &BuildInfo{
		Owner:       ghCtx.Event.Repository.Owner.Login,
		Repo:        ghCtx.Event.Repository.Name,
		PullRequest: ghCtx.Event.Number,
		Branch:      ghCtx.HeadRef,
	}
	if ghCtx.Event.PullRequest.Head.Sha != "" {
		info.SHA = ghCtx.Event.PullRequest.Head.Sha
	} else {
		info.SHA = ghCtx.Sha
	}
	return info, info.PullRequest != 0, nil
}

func IsInGitHubAction() bool {
	// https://help.github.com/en/articles/virtual-environments-for-github-actions#default-environment-variables
	return os.Getenv("GITHUB_ACTION") != ""
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	mailgun "github.com/riobard/go-mailgun"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

var user_id = flag.Int("user_id", 0, "ID of user to output.")
var all_users = flag.Bool("all_users", false, "Whether to email all users.")
var send_email = flag.Bool("send_email", false, "Whether to send the email to the user.")
var mark_read = flag.Bool("mark_read", true, "Whether to mark activity sent as read. False means subsequent calls will send the same info.")

type User struct {
	Id       int
	username string
	Email    string
}

type GithubUser struct {
	Id    int
	Login string
	Url   string
}

// db thing.
type GithubRepo struct {
	Id       int
	User     string // should probably be Login
	RepoName string // should probably be Name
}

func (g *GithubRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.User, g.RepoName)
}

type GithubApiRepo struct {
	Id          int
	Owner       GithubUser
	Name        NString
	Description NString
}

func (g *GithubApiRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.Owner.Login, g.Name)
}

type Activity struct {
	Id            int
	github_id     int
	activity_type string // should be enum
	created_at    time.Time
	Username      string
	repo          GithubRepo // this isn't going to be memory efficient
	Meta          string     // full payload of json object
}

type Treeish struct {
	Label    string
	Sha      string
	Repo     GithubApiRepo
	Html_url string
	User     GithubUser
}

type Commit struct {
	Sha     string
	Message string
	Author  struct {
		Name  string
		Email string
	}
}

func (c *Commit) ShortSha() string {
	return c.Sha[0:6]
}

func (c *Commit) ShortCommit() string {
	msg := strings.Split(c.Message, "\n")[0]
	if len(msg) > 80 {
		return msg[0:77] + "..."
	}
	return msg
}

// Nullable String, per https://groups.google.com/d/msg/golang-nuts/JOFWAqrTbUs/KZIGEPclSpwJ
type NString string

func (n *NString) UnmarshalJSON(b []byte) (err error) {
	if string(b) == "null" {
		return nil
	}
	return json.Unmarshal(b, (*string)(n))
}

func getUserRepos(db *sql.DB, user_id int) ([]GithubRepo, error) {
	// guess as to initial size. Likely very few users following < 10 repos.
	var repos = make([]GithubRepo, 10)

	// TODO(justinabrahms): Should really alter the schema such
	// that the join from user <-> repo doesn't go through
	// userprofiles.
	rows, err := db.Query(
		`SELECT r.id, username, project_name
	           FROM streamer_repo r
		   JOIN streamer_userprofile_repos upr ON upr.repo_id = r.id
		   JOIN streamer_userprofile up ON up.id = upr.userprofile_id
                   WHERE up.user_id = ?;`, user_id)
	// what's a good way to handle this not working?
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var username, project_name string
	var pk int
	for rows.Next() {
		err = rows.Scan(&pk, &username, &project_name)
		if err != nil {
			fmt.Println("Error!: ", err)
		}
		var repo = GithubRepo{pk, username, project_name}
		repos = append(repos, repo)
	}

	return repos, err
}

func getUser(db *sql.DB, uid int) (u User, err error) {
	row := db.QueryRow(
		`SELECT id, username, email
		   FROM auth_user
		   WHERE id = ?`, uid)
	err = row.Scan(&u.Id, &u.username, &u.Email)
	return
}

func getUsers(db *sql.DB) (users []User, err error) {
	rows, err := db.Query(
		`SELECT id, username, email
		   FROM auth_user`)
	if err != nil {
		return
	}
	defer rows.Close()

	for rows.Next() {
		var u User
		err = rows.Scan(&u.Id, &u.username, &u.Email)
		if err != nil {
			return
		}
		users = append(users, u)
	}
	return
}

func getRepoActivity(db *sql.DB, repo *GithubRepo) (activity_list []Activity, err error) {
	activity_list = make([]Activity, 0)
	rows, err := db.Query(
		`SELECT a.id, a.event_id, a.type, a.created_at, ghu.name, r.username, r.project_name, meta
		  FROM streamer_activity a
		  JOIN streamer_repo r on r.id=a.repo_id
		  JOIN streamer_githubuser ghu on ghu.id=a.user_id
		  JOIN streamer_userprofile_repos upr on r.id=upr.repo_id
		  WHERE r.id = ?
		  AND a.created_at > DATE_SUB(NOW(), INTERVAL 5 day) -- don't send things more than a few days old. Think, new users who subscribe to rails/rails
		  AND (upr.last_sent is null -- hasn't been sent at all
		    OR a.created_at > upr.last_sent) --  or hasn't been sent since we've gotten new stuff
                  GROUP BY a.id`, // and unique-ify based on activity id to prevent dupes.
		repo.Id)
	if err != nil {
		return
	}
	defer rows.Close()

	var (
		pk, github_id, repo_id                                              int
		activity_type, username, repo_project, repo_user, meta, created_str string
		created_at                                                          time.Time
	)

	for rows.Next() {
		err = rows.Scan(&pk, &github_id, &activity_type, &created_str, &username, &repo_user, &repo_project, &meta)
		if err != nil {
			fmt.Println("ERR: ", err)
		}

		created_at, err = time.Parse("2006-01-02 15:04:05", created_str)
		if err != nil {
			fmt.Println("Can't parse the format of ", created_str)
			return nil, err
		}

		activity_list = append(activity_list,
			Activity{pk, github_id, activity_type, created_at, username,
				GithubRepo{repo_id, repo_user, repo_project}, meta})
	}

	return
}

func repoToTemplate(repo GithubRepo, activities []Activity, render_map map[string]func([]Activity, bool) string) (response string) {
	if len(activities) == 0 {
		return ""
	}
	var activity_map = make(map[string][]Activity)
	for _, activity := range activities {
		// This seems like a lot of juggling. Is there a better way?
		arr := activity_map[activity.activity_type]
		if arr == nil {
			arr = make([]Activity, 0)
			activity_map[activity.activity_type] = arr
		}
		arr = append(arr, activity)
		activity_map[activity.activity_type] = arr
	}

	// activity_map: activity_type => []activity
	for activity_type, activities := range activity_map {
		fn, ok := render_map[activity_type]
		if ok {
			response += fn(activities, true)
		} else {
			log.Print("Not sure how to render activites of type ", activity_type)
		}
	}
	if len(response) > 0 {
		response = fmt.Sprintf("%s:\n%s", repo.FullName(), response)
	}
	return

}

func repoToString(db *sql.DB, repo GithubRepo, response chan string) {
	activities, err := getRepoActivity(db, &repo)
	if err != nil {
		fmt.Println("ERR: ", err)
		os.Exit(1)
	}

	activity_type_to_renderer := map[string]func([]Activity, bool) string{
		"P":  pushRender,
		"PR": pullRequestRender,
		"D":  deleteRender,
		"C":  createRender,
		"IC": issueCommentRender,
		"Gl": wikiRender, // Gl is for Gollum, Github's wiki thing.
		"I":  issueRender,
		"Pb": publicRender,

		// Watches and forks for repos are WAYY too spammy.
		// "W":  watchRender,
		// "F":  forkRender,
	}

	response <- repoToTemplate(repo, activities, activity_type_to_renderer)
}

func markUserRepoSent(db *sql.DB, user User, repos []GithubRepo) (err error) {
	// finds the streamer_userprofile_repo row for the repo / user
	// combo, mark its last_sent as now
	ids := make([]string, 0)
	for _, repo := range repos {
		// TODO: why are they 0?
		if repo.Id != 0 {
			ids = append(ids, strconv.FormatInt(int64(repo.Id), 10))
		}
	}

	// There is likely a better way to get parameterization, but
	// it wasn't working for me with ?'s.
	str := fmt.Sprintf(
		`UPDATE streamer_userprofile_repos
		   SET last_sent=NOW()
		   WHERE repo_id IN (%s)`, strings.Join(ids, ","))
	_, err = db.Exec(str)
	return
}

// TODO: Github Users

// TODO: Need to finish the following activity types:
// - TA (team add)
// - FA (fork apply)
// - DO (Download)
// - RC (Pull Request Review Comment)
// - Fl (Follow)
// - G (Gist)
// - M (Member)
// - CC (Commit Comment)

func main() {
	// Expects the following environment variables:
	// DB_USER, DB_PASS, DB_DB  (database user, password and database)
	// MAILGUN_API_KEY  (api key from mailgun.org)

	flag.Parse()

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s?charset=utf8", os.Getenv("DB_USER"), os.Getenv("DB_PASS"), os.Getenv("DB_DB")))
	if err != nil {
		log.Fatalf("Unable to connect to mysql. ", err)
	}
	defer db.Close()

	var users []User
	if *user_id != 0 {
		users = make([]User, 1)
		user, err := getUser(db, *user_id)
		if err != nil {
			log.Fatalf("Couldn't return user %d. %s", *user_id, err)
		}
		users[0] = user
	} else if *all_users {
		users, err = getUsers(db)
		if err != nil {
			log.Fatal("Couldn't fetch all users.")
		}
	} else {
		log.Fatal("You must specify either to mail all users or a specific user id.")
	}

	for _, user := range users {
		repos, err := getUserRepos(db, user.Id)
		if err != nil {
			log.Print("Error fetching user's repos. %s", err)
			continue
		}

		response_chan := make(chan string, len(repos))
		for _, repo := range repos {
			repoToString(db, repo, response_chan)
		}

		var response string
		for num_responses := 0; num_responses < len(repos); {
			select {
			case r := <-response_chan:
				num_responses++
				response += r
			}
		}

		if *mark_read {
			fmt.Println("Marking read.")
			err = markUserRepoSent(db, user, repos)
			if err != nil {
				log.Print("Error updating repositories as sent.")
				continue
			}
		}

		if len(response) == 0 {
			log.Print("No content to email.")
			continue
		}

		if *send_email {
			mg := mailgun.Open(os.Getenv("MAILGUN_API_KEY"))
			e := &Email{
				from:    "justin@gitstreams.mailgun.org",
				to:      []string{user.Email},
				subject: "Gitstreams Digest Email",
				text:    response,
			}

			id, err := mg.Send(e)

			if err != nil {
				log.Print("Unable to send email. ", err)
			}
			log.Printf("MessageId = %s for uid:%s", id, user.Id)
		} else {
			fmt.Println("Would have sent the following email.")
			fmt.Println(response)
		}
	}

	return
}
<commit_msg>preceeding newlines between repos to give them a bit more distance.<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	mailgun "github.com/riobard/go-mailgun"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

var user_id = flag.Int("user_id", 0, "ID of user to output.")
var all_users = flag.Bool("all_users", false, "Whether to email all users.")
var send_email = flag.Bool("send_email", false, "Whether to send the email to the user.")
var mark_read = flag.Bool("mark_read", true, "Whether to mark activity sent as read. False means subsequent calls will send the same info.")

type User struct {
	Id       int
	username string
	Email    string
}

type GithubUser struct {
	Id    int
	Login string
	Url   string
}

// db thing.
type GithubRepo struct {
	Id       int
	User     string // should probably be Login
	RepoName string // should probably be Name
}

func (g *GithubRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.User, g.RepoName)
}

type GithubApiRepo struct {
	Id          int
	Owner       GithubUser
	Name        NString
	Description NString
}

func (g *GithubApiRepo) FullName() string {
	return fmt.Sprintf("%s/%s", g.Owner.Login, g.Name)
}

type Activity struct {
	Id            int
	github_id     int
	activity_type string // should be enum
	created_at    time.Time
	Username      string
	repo          GithubRepo // this isn't going to be memory efficient
	Meta          string     // full payload of json object
}

type Treeish struct {
	Label    string
	Sha      string
	Repo     GithubApiRepo
	Html_url string
	User     GithubUser
}

type Commit struct {
	Sha     string
	Message string
	Author  struct {
		Name  string
		Email string
	}
}

func (c *Commit) ShortSha() string {
	return c.Sha[0:6]
}

func (c *Commit) ShortCommit() string {
	msg := strings.Split(c.Message, "\n")[0]
	if len(msg) > 80 {
		return msg[0:77] + "..."
	}
	return msg
}

// Nullable String, per https://groups.google.com/d/msg/golang-nuts/JOFWAqrTbUs/KZIGEPclSpwJ
type NString string

func (n *NString) UnmarshalJSON(b []byte) (err error) {
	if string(b) == "null" {
		return nil
	}
	return json.Unmarshal(b, (*string)(n))
}

func getUserRepos(db *sql.DB, user_id int) ([]GithubRepo, error) {
	// guess as to initial size. Likely very few users following < 10 repos.
	var repos = make([]GithubRepo, 10)

	// TODO(justinabrahms): Should really alter the schema such
	// that the join from user <-> repo doesn't go through
	// userprofiles.
	rows, err := db.Query(
		`SELECT r.id, username, project_name
	           FROM streamer_repo r
		   JOIN streamer_userprofile_repos upr ON upr.repo_id = r.id
		   JOIN streamer_userprofile up ON up.id = upr.userprofile_id
                   WHERE up.user_id = ?;`, user_id)
	// what's a good way to handle this not working?
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var username, project_name string
	var pk int
	for rows.Next() {
		err = rows.Scan(&pk, &username, &project_name)
		if err != nil {
			fmt.Println("Error!: ", err)
		}
		var repo = GithubRepo{pk, username, project_name}
		repos = append(repos, repo)
	}

	return repos, err
}

func getUser(db *sql.DB, uid int) (u User, err error) {
	row := db.QueryRow(
		`SELECT id, username, email
		   FROM auth_user
		   WHERE id = ?`, uid)
	err = row.Scan(&u.Id, &u.username, &u.Email)
	return
}

func getUsers(db *sql.DB) (users []User, err error) {
	rows, err := db.Query(
		`SELECT id, username, email
		   FROM auth_user`)
	if err != nil {
		return
	}
	defer rows.Close()

	for rows.Next() {
		var u User
		err = rows.Scan(&u.Id, &u.username, &u.Email)
		if err != nil {
			return
		}
		users = append(users, u)
	}
	return
}

func getRepoActivity(db *sql.DB, repo *GithubRepo) (activity_list []Activity, err error) {
	activity_list = make([]Activity, 0)
	rows, err := db.Query(
		`SELECT a.id, a.event_id, a.type, a.created_at, ghu.name, r.username, r.project_name, meta
		  FROM streamer_activity a
		  JOIN streamer_repo r on r.id=a.repo_id
		  JOIN streamer_githubuser ghu on ghu.id=a.user_id
		  JOIN streamer_userprofile_repos upr on r.id=upr.repo_id
		  WHERE r.id = ?
		  AND a.created_at > DATE_SUB(NOW(), INTERVAL 5 day) -- don't send things more than a few days old. Think, new users who subscribe to rails/rails
		  AND (upr.last_sent is null -- hasn't been sent at all
		    OR a.created_at > upr.last_sent) --  or hasn't been sent since we've gotten new stuff
                  GROUP BY a.id`, // and unique-ify based on activity id to prevent dupes.
		repo.Id)
	if err != nil {
		return
	}
	defer rows.Close()

	var (
		pk, github_id, repo_id                                              int
		activity_type, username, repo_project, repo_user, meta, created_str string
		created_at                                                          time.Time
	)

	for rows.Next() {
		err = rows.Scan(&pk, &github_id, &activity_type, &created_str, &username, &repo_user, &repo_project, &meta)
		if err != nil {
			fmt.Println("ERR: ", err)
		}

		created_at, err = time.Parse("2006-01-02 15:04:05", created_str)
		if err != nil {
			fmt.Println("Can't parse the format of ", created_str)
			return nil, err
		}

		activity_list = append(activity_list,
			Activity{pk, github_id, activity_type, created_at, username,
				GithubRepo{repo_id, repo_user, repo_project}, meta})
	}

	return
}

func repoToTemplate(repo GithubRepo, activities []Activity, render_map map[string]func([]Activity, bool) string) (response string) {
	if len(activities) == 0 {
		return ""
	}
	var activity_map = make(map[string][]Activity)
	for _, activity := range activities {
		// This seems like a lot of juggling. Is there a better way?
		arr := activity_map[activity.activity_type]
		if arr == nil {
			arr = make([]Activity, 0)
			activity_map[activity.activity_type] = arr
		}
		arr = append(arr, activity)
		activity_map[activity.activity_type] = arr
	}

	// activity_map: activity_type => []activity
	for activity_type, activities := range activity_map {
		fn, ok := render_map[activity_type]
		if ok {
			response += fn(activities, true)
		} else {
			log.Print("Not sure how to render activites of type ", activity_type)
		}
	}
	if len(response) > 0 {
		response = fmt.Sprintf("\n\n%s:\n%s", repo.FullName(), response)
	}
	return

}

func repoToString(db *sql.DB, repo GithubRepo, response chan string) {
	activities, err := getRepoActivity(db, &repo)
	if err != nil {
		fmt.Println("ERR: ", err)
		os.Exit(1)
	}

	activity_type_to_renderer := map[string]func([]Activity, bool) string{
		"P":  pushRender,
		"PR": pullRequestRender,
		"D":  deleteRender,
		"C":  createRender,
		"IC": issueCommentRender,
		"Gl": wikiRender, // Gl is for Gollum, Github's wiki thing.
		"I":  issueRender,
		"Pb": publicRender,

		// Watches and forks for repos are WAYY too spammy.
		// "W":  watchRender,
		// "F":  forkRender,
	}

	response <- repoToTemplate(repo, activities, activity_type_to_renderer)
}

func markUserRepoSent(db *sql.DB, user User, repos []GithubRepo) (err error) {
	// finds the streamer_userprofile_repo row for the repo / user
	// combo, mark its last_sent as now
	ids := make([]string, 0)
	for _, repo := range repos {
		// TODO: why are they 0?
		if repo.Id != 0 {
			ids = append(ids, strconv.FormatInt(int64(repo.Id), 10))
		}
	}

	// There is likely a better way to get parameterization, but
	// it wasn't working for me with ?'s.
	str := fmt.Sprintf(
		`UPDATE streamer_userprofile_repos
		   SET last_sent=NOW()
		   WHERE repo_id IN (%s)`, strings.Join(ids, ","))
	_, err = db.Exec(str)
	return
}

// TODO: Github Users

// TODO: Need to finish the following activity types:
// - TA (team add)
// - FA (fork apply)
// - DO (Download)
// - RC (Pull Request Review Comment)
// - Fl (Follow)
// - G (Gist)
// - M (Member)
// - CC (Commit Comment)

func main() {
	// Expects the following environment variables:
	// DB_USER, DB_PASS, DB_DB  (database user, password and database)
	// MAILGUN_API_KEY  (api key from mailgun.org)

	flag.Parse()

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@/%s?charset=utf8", os.Getenv("DB_USER"), os.Getenv("DB_PASS"), os.Getenv("DB_DB")))
	if err != nil {
		log.Fatalf("Unable to connect to mysql. ", err)
	}
	defer db.Close()

	var users []User
	if *user_id != 0 {
		users = make([]User, 1)
		user, err := getUser(db, *user_id)
		if err != nil {
			log.Fatalf("Couldn't return user %d. %s", *user_id, err)
		}
		users[0] = user
	} else if *all_users {
		users, err = getUsers(db)
		if err != nil {
			log.Fatal("Couldn't fetch all users.")
		}
	} else {
		log.Fatal("You must specify either to mail all users or a specific user id.")
	}

	for _, user := range users {
		repos, err := getUserRepos(db, user.Id)
		if err != nil {
			log.Print("Error fetching user's repos. %s", err)
			continue
		}

		response_chan := make(chan string, len(repos))
		for _, repo := range repos {
			repoToString(db, repo, response_chan)
		}

		var response string
		for num_responses := 0; num_responses < len(repos); {
			select {
			case r := <-response_chan:
				num_responses++
				response += r
			}
		}

		if *mark_read {
			fmt.Println("Marking read.")
			err = markUserRepoSent(db, user, repos)
			if err != nil {
				log.Print("Error updating repositories as sent.")
				continue
			}
		}

		if len(response) == 0 {
			log.Print("No content to email.")
			continue
		}

		if *send_email {
			mg := mailgun.Open(os.Getenv("MAILGUN_API_KEY"))
			e := &Email{
				from:    "justin@gitstreams.mailgun.org",
				to:      []string{user.Email},
				subject: "Gitstreams Digest Email",
				text:    response,
			}

			id, err := mg.Send(e)

			if err != nil {
				log.Print("Unable to send email. ", err)
			}
			log.Printf("MessageId = %s for uid:%s", id, user.Id)
		} else {
			fmt.Println("Would have sent the following email.")
			fmt.Println(response)
		}
	}

	return
}
<|endoftext|>


<commit_before>package probe

import (
	"context"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	log "github.com/sirupsen/logrus"
	"golang.org/x/time/rate"

	"github.com/weaveworks/scope/report"
)

const (
	spiedReportBufferSize    = 16
	shortcutReportBufferSize = 1024
)

// ReportPublisher publishes reports, probably to a remote collector.
type ReportPublisher interface {
	Publish(r report.Report) error
}

// Probe sits there, generating and publishing reports.
type Probe struct {
	spyInterval, publishInterval time.Duration
	publisher                    ReportPublisher
	rateLimiter                  *rate.Limiter
	noControls                   bool

	tickers   []Ticker
	reporters []Reporter
	taggers   []Tagger

	quit chan struct{}
	done sync.WaitGroup

	spiedReports    chan report.Report
	shortcutReports chan report.Report
}

// Tagger tags nodes with value-add node metadata.
type Tagger interface {
	Name() string
	Tag(r report.Report) (report.Report, error)
}

// Reporter generates Reports.
type Reporter interface {
	Name() string
	Report() (report.Report, error)
}

// ReporterFunc uses a function to implement a Reporter
func ReporterFunc(name string, f func() (report.Report, error)) Reporter {
	return reporterFunc{name, f}
}

type reporterFunc struct {
	name string
	f    func() (report.Report, error)
}

func (r reporterFunc) Name() string                   { return r.name }
func (r reporterFunc) Report() (report.Report, error) { return r.f() }

// Ticker is something which will be invoked every spyDuration.
// It's useful for things that should be updated on that interval.
// For example, cached shared state between Taggers and Reporters.
type Ticker interface {
	Name() string
	Tick() error
}

// New makes a new Probe.
func New(
	spyInterval, publishInterval time.Duration,
	publisher ReportPublisher,
	noControls bool,
) *Probe {
	result := &Probe{
		spyInterval:     spyInterval,
		publishInterval: publishInterval,
		publisher:       publisher,
		rateLimiter:     rate.NewLimiter(rate.Every(publishInterval/100), 1),
		noControls:      noControls,
		quit:            make(chan struct{}),
		spiedReports:    make(chan report.Report, spiedReportBufferSize),
		shortcutReports: make(chan report.Report, shortcutReportBufferSize),
	}
	return result
}

// AddTagger adds a new Tagger to the Probe
func (p *Probe) AddTagger(ts ...Tagger) {
	p.taggers = append(p.taggers, ts...)
}

// AddReporter adds a new Reported to the Probe
func (p *Probe) AddReporter(rs ...Reporter) {
	p.reporters = append(p.reporters, rs...)
}

// AddTicker adds a new Ticker to the Probe
func (p *Probe) AddTicker(ts ...Ticker) {
	p.tickers = append(p.tickers, ts...)
}

// Start starts the probe
func (p *Probe) Start() {
	p.done.Add(2)
	go p.spyLoop()
	go p.publishLoop()
}

// Stop stops the probe
func (p *Probe) Stop() error {
	close(p.quit)
	p.done.Wait()
	return nil
}

// Publish will queue a report for immediate publication,
// bypassing the spy tick
func (p *Probe) Publish(rpt report.Report) {
	rpt = p.tag(rpt)
	p.shortcutReports <- rpt
}

func (p *Probe) spyLoop() {
	defer p.done.Done()
	spyTick := time.Tick(p.spyInterval)

	for {
		select {
		case <-spyTick:
			p.tick()
			rpt := p.report()
			rpt = p.tag(rpt)
			p.spiedReports <- rpt
		case <-p.quit:
			return
		}
	}
}

func (p *Probe) tick() {
	for _, ticker := range p.tickers {
		t := time.Now()
		err := ticker.Tick()
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "ticker"},
			{Name: "module", Value: ticker.Name()},
		})
		if err != nil {
			log.Errorf("Error doing ticker: %v", err)
		}
	}
}

func (p *Probe) report() report.Report {
	reports := make(chan report.Report, len(p.reporters))
	for _, rep := range p.reporters {
		go func(rep Reporter) {
			t := time.Now()
			timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v reporter took longer than %v", rep.Name(), p.spyInterval) })
			newReport, err := rep.Report()
			if !timer.Stop() {
				log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
			}
			metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
				{Name: "operation", Value: "reporter"},
				{Name: "module", Value: rep.Name()},
			})
			if err != nil {
				log.Errorf("Error generating %s report: %v", rep.Name(), err)
				newReport = report.MakeReport() // empty is OK to merge
			}
			reports <- newReport
		}(rep)
	}

	result := report.MakeReport()
	for i := 0; i < cap(reports); i++ {
		result = result.Merge(<-reports)
	}
	return result
}

func (p *Probe) tag(r report.Report) report.Report {
	var err error
	for _, tagger := range p.taggers {
		t := time.Now()
		timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v tagger took longer than %v", tagger.Name(), p.spyInterval) })
		r, err = tagger.Tag(r)
		if !timer.Stop() {
			log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
		}
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "tagger"},
			{Name: "module", Value: tagger.Name()},
		})
		if err != nil {
			log.Errorf("Error applying tagger: %v", err)
		}
	}
	return r
}

func (p *Probe) drainAndPublish(rpt report.Report, rs chan report.Report) {
	p.rateLimiter.Wait(context.Background())
ForLoop:
	for {
		select {
		case r := <-rs:
			rpt = rpt.Merge(r)
		default:
			break ForLoop
		}
	}

	if p.noControls {
		rpt.WalkTopologies(func(t *report.Topology) {
			t.Controls = report.Controls{}
		})
	}
	if err := p.publisher.Publish(rpt); err != nil {
		log.Infof("Publish: %v", err)
	}
}

func (p *Probe) publishLoop() {
	defer p.done.Done()
	pubTick := time.Tick(p.publishInterval)

	for {
		select {
		case <-pubTick:
			p.drainAndPublish(report.MakeReport(), p.spiedReports)

		case rpt := <-p.shortcutReports:
			p.drainAndPublish(rpt, p.shortcutReports)

		case <-p.quit:
			return
		}
	}
}
<commit_msg>Refactor: pull Publish() call up to publishLoop()<commit_after>package probe

import (
	"context"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	log "github.com/sirupsen/logrus"
	"golang.org/x/time/rate"

	"github.com/weaveworks/scope/report"
)

const (
	spiedReportBufferSize    = 16
	shortcutReportBufferSize = 1024
)

// ReportPublisher publishes reports, probably to a remote collector.
type ReportPublisher interface {
	Publish(r report.Report) error
}

// Probe sits there, generating and publishing reports.
type Probe struct {
	spyInterval, publishInterval time.Duration
	publisher                    ReportPublisher
	rateLimiter                  *rate.Limiter
	noControls                   bool

	tickers   []Ticker
	reporters []Reporter
	taggers   []Tagger

	quit chan struct{}
	done sync.WaitGroup

	spiedReports    chan report.Report
	shortcutReports chan report.Report
}

// Tagger tags nodes with value-add node metadata.
type Tagger interface {
	Name() string
	Tag(r report.Report) (report.Report, error)
}

// Reporter generates Reports.
type Reporter interface {
	Name() string
	Report() (report.Report, error)
}

// ReporterFunc uses a function to implement a Reporter
func ReporterFunc(name string, f func() (report.Report, error)) Reporter {
	return reporterFunc{name, f}
}

type reporterFunc struct {
	name string
	f    func() (report.Report, error)
}

func (r reporterFunc) Name() string                   { return r.name }
func (r reporterFunc) Report() (report.Report, error) { return r.f() }

// Ticker is something which will be invoked every spyDuration.
// It's useful for things that should be updated on that interval.
// For example, cached shared state between Taggers and Reporters.
type Ticker interface {
	Name() string
	Tick() error
}

// New makes a new Probe.
func New(
	spyInterval, publishInterval time.Duration,
	publisher ReportPublisher,
	noControls bool,
) *Probe {
	result := &Probe{
		spyInterval:     spyInterval,
		publishInterval: publishInterval,
		publisher:       publisher,
		rateLimiter:     rate.NewLimiter(rate.Every(publishInterval/100), 1),
		noControls:      noControls,
		quit:            make(chan struct{}),
		spiedReports:    make(chan report.Report, spiedReportBufferSize),
		shortcutReports: make(chan report.Report, shortcutReportBufferSize),
	}
	return result
}

// AddTagger adds a new Tagger to the Probe
func (p *Probe) AddTagger(ts ...Tagger) {
	p.taggers = append(p.taggers, ts...)
}

// AddReporter adds a new Reported to the Probe
func (p *Probe) AddReporter(rs ...Reporter) {
	p.reporters = append(p.reporters, rs...)
}

// AddTicker adds a new Ticker to the Probe
func (p *Probe) AddTicker(ts ...Ticker) {
	p.tickers = append(p.tickers, ts...)
}

// Start starts the probe
func (p *Probe) Start() {
	p.done.Add(2)
	go p.spyLoop()
	go p.publishLoop()
}

// Stop stops the probe
func (p *Probe) Stop() error {
	close(p.quit)
	p.done.Wait()
	return nil
}

// Publish will queue a report for immediate publication,
// bypassing the spy tick
func (p *Probe) Publish(rpt report.Report) {
	rpt = p.tag(rpt)
	p.shortcutReports <- rpt
}

func (p *Probe) spyLoop() {
	defer p.done.Done()
	spyTick := time.Tick(p.spyInterval)

	for {
		select {
		case <-spyTick:
			p.tick()
			rpt := p.report()
			rpt = p.tag(rpt)
			p.spiedReports <- rpt
		case <-p.quit:
			return
		}
	}
}

func (p *Probe) tick() {
	for _, ticker := range p.tickers {
		t := time.Now()
		err := ticker.Tick()
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "ticker"},
			{Name: "module", Value: ticker.Name()},
		})
		if err != nil {
			log.Errorf("Error doing ticker: %v", err)
		}
	}
}

func (p *Probe) report() report.Report {
	reports := make(chan report.Report, len(p.reporters))
	for _, rep := range p.reporters {
		go func(rep Reporter) {
			t := time.Now()
			timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v reporter took longer than %v", rep.Name(), p.spyInterval) })
			newReport, err := rep.Report()
			if !timer.Stop() {
				log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
			}
			metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
				{Name: "operation", Value: "reporter"},
				{Name: "module", Value: rep.Name()},
			})
			if err != nil {
				log.Errorf("Error generating %s report: %v", rep.Name(), err)
				newReport = report.MakeReport() // empty is OK to merge
			}
			reports <- newReport
		}(rep)
	}

	result := report.MakeReport()
	for i := 0; i < cap(reports); i++ {
		result = result.Merge(<-reports)
	}
	return result
}

func (p *Probe) tag(r report.Report) report.Report {
	var err error
	for _, tagger := range p.taggers {
		t := time.Now()
		timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v tagger took longer than %v", tagger.Name(), p.spyInterval) })
		r, err = tagger.Tag(r)
		if !timer.Stop() {
			log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
		}
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "tagger"},
			{Name: "module", Value: tagger.Name()},
		})
		if err != nil {
			log.Errorf("Error applying tagger: %v", err)
		}
	}
	return r
}

func (p *Probe) drainAndSanitise(rpt report.Report, rs chan report.Report) report.Report {
	p.rateLimiter.Wait(context.Background())
ForLoop:
	for {
		select {
		case r := <-rs:
			rpt = rpt.Merge(r)
		default:
			break ForLoop
		}
	}

	if p.noControls {
		rpt.WalkTopologies(func(t *report.Topology) {
			t.Controls = report.Controls{}
		})
	}
	return rpt
}

func (p *Probe) publishLoop() {
	defer p.done.Done()
	pubTick := time.Tick(p.publishInterval)

	for {
		var err error
		select {
		case <-pubTick:
			rpt := p.drainAndSanitise(report.MakeReport(), p.spiedReports)
			err = p.publisher.Publish(rpt)

		case rpt := <-p.shortcutReports:
			rpt = p.drainAndSanitise(rpt, p.shortcutReports)
			err = p.publisher.Publish(rpt)

		case <-p.quit:
			return
		}
		if err != nil {
			log.Infof("Publish: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dynamic

import (
	"context"
	"fmt"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/rest"
)

type dynamicClient struct {
	client *rest.RESTClient
}

var _ Interface = &dynamicClient{}

// ConfigFor returns a copy of the provided config with the
// appropriate dynamic client defaults set.
func ConfigFor(inConfig *rest.Config) *rest.Config {
	config := rest.CopyConfig(inConfig)
	config.AcceptContentTypes = "application/json"
	config.ContentType = "application/json"
	config.NegotiatedSerializer = basicNegotiatedSerializer{} // this gets used for discovery and error handling types
	if config.UserAgent == "" {
		config.UserAgent = rest.DefaultKubernetesUserAgent()
	}
	return config
}

// NewForConfigOrDie creates a new Interface for the given config and
// panics if there is an error in the config.
func NewForConfigOrDie(c *rest.Config) Interface {
	ret, err := NewForConfig(c)
	if err != nil {
		panic(err)
	}
	return ret
}

// NewForConfig creates a new dynamic client or returns an error.
func NewForConfig(inConfig *rest.Config) (Interface, error) {
	config := ConfigFor(inConfig)
	// for serializing the options
	config.GroupVersion = &schema.GroupVersion{}
	config.APIPath = "/if-you-see-this-search-for-the-break"

	restClient, err := rest.RESTClientFor(config)
	if err != nil {
		return nil, err
	}

	return &dynamicClient{client: restClient}, nil
}

type dynamicResourceClient struct {
	client    *dynamicClient
	namespace string
	resource  schema.GroupVersionResource
}

func (c *dynamicClient) Resource(resource schema.GroupVersionResource) NamespaceableResourceInterface {
	return &dynamicResourceClient{client: c, resource: resource}
}

func (c *dynamicResourceClient) Namespace(ns string) ResourceInterface {
	ret := *c
	ret.namespace = ns
	return &ret
}

func (c *dynamicResourceClient) Create(ctx context.Context, obj *unstructured.Unstructured, opts metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) {
	outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
	if err != nil {
		return nil, err
	}
	name := ""
	if len(subresources) > 0 {
		accessor, err := meta.Accessor(obj)
		if err != nil {
			return nil, err
		}
		name = accessor.GetName()
		if len(name) == 0 {
			return nil, fmt.Errorf("name is required")
		}
	}

	result := c.client.client.
		Post().
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(outBytes).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}

	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) Update(ctx context.Context, obj *unstructured.Unstructured, opts metav1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) {
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return nil, err
	}
	name := accessor.GetName()
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}
	outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
	if err != nil {
		return nil, err
	}

	result := c.client.client.
		Put().
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(outBytes).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}

	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) UpdateStatus(ctx context.Context, obj *unstructured.Unstructured, opts metav1.UpdateOptions) (*unstructured.Unstructured, error) {
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return nil, err
	}
	name := accessor.GetName()
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}

	outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
	if err != nil {
		return nil, err
	}

	result := c.client.client.
		Put().
		AbsPath(append(c.makeURLSegments(name), "status")...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(outBytes).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}

	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions, subresources ...string) error {
	if len(name) == 0 {
		return fmt.Errorf("name is required")
	}
	deleteOptionsByte, err := runtime.Encode(deleteOptionsCodec.LegacyCodec(schema.GroupVersion{Version: "v1"}), &opts)
	if err != nil {
		return err
	}

	result := c.client.client.
		Delete().
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(deleteOptionsByte).
		Do(ctx)
	return result.Error()
}

func (c *dynamicResourceClient) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOptions metav1.ListOptions) error {
	deleteOptionsByte, err := runtime.Encode(deleteOptionsCodec.LegacyCodec(schema.GroupVersion{Version: "v1"}), &opts)
	if err != nil {
		return err
	}

	result := c.client.client.
		Delete().
		AbsPath(c.makeURLSegments("")...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(deleteOptionsByte).
		SpecificallyVersionedParams(&listOptions, dynamicParameterCodec, versionV1).
		Do(ctx)
	return result.Error()
}

func (c *dynamicResourceClient) Get(ctx context.Context, name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) {
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}
	result := c.client.client.Get().AbsPath(append(c.makeURLSegments(name), subresources...)...).SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}
	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) List(ctx context.Context, opts metav1.ListOptions) (*unstructured.UnstructuredList, error) {
	result := c.client.client.Get().AbsPath(c.makeURLSegments("")...).SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}
	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	if list, ok := uncastObj.(*unstructured.UnstructuredList); ok {
		return list, nil
	}

	list, err := uncastObj.(*unstructured.Unstructured).ToList()
	if err != nil {
		return nil, err
	}
	return list, nil
}

func (c *dynamicResourceClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
	opts.Watch = true
	return c.client.client.Get().AbsPath(c.makeURLSegments("")...).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Watch(ctx)
}

func (c *dynamicResourceClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}
	result := c.client.client.
		Patch(pt).
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		Body(data).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}
	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) makeURLSegments(name string) []string {
	url := []string{}
	if len(c.resource.Group) == 0 {
		url = append(url, "api")
	} else {
		url = append(url, "apis", c.resource.Group)
	}
	url = append(url, c.resource.Version)

	if len(c.namespace) > 0 {
		url = append(url, "namespaces", c.namespace)
	}
	url = append(url, c.resource.Resource)

	if len(name) > 0 {
		url = append(url, name)
	}

	return url
}
<commit_msg>expose NewForConfigAndClient for the dynamic client<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dynamic

import (
	"context"
	"fmt"
	"net/http"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/rest"
)

type dynamicClient struct {
	client *rest.RESTClient
}

var _ Interface = &dynamicClient{}

// ConfigFor returns a copy of the provided config with the
// appropriate dynamic client defaults set.
func ConfigFor(inConfig *rest.Config) *rest.Config {
	config := rest.CopyConfig(inConfig)
	config.AcceptContentTypes = "application/json"
	config.ContentType = "application/json"
	config.NegotiatedSerializer = basicNegotiatedSerializer{} // this gets used for discovery and error handling types
	if config.UserAgent == "" {
		config.UserAgent = rest.DefaultKubernetesUserAgent()
	}
	return config
}

// NewForConfigOrDie creates a new Interface for the given config and
// panics if there is an error in the config.
func NewForConfigOrDie(c *rest.Config) Interface {
	ret, err := NewForConfig(c)
	if err != nil {
		panic(err)
	}
	return ret
}

// NewForConfig creates a new dynamic client or returns an error.
// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
// where httpClient was generated with rest.HTTPClientFor(c).
func NewForConfig(inConfig *rest.Config) (Interface, error) {
	config := ConfigFor(inConfig)

	httpClient, err := rest.HTTPClientFor(config)
	if err != nil {
		return nil, err
	}
	return NewForConfigAndClient(config, httpClient)
}

// NewForConfigAndClient creates a new dynamic client for the given config and http client.
// Note the http client provided takes precedence over the configured transport values.
func NewForConfigAndClient(inConfig *rest.Config, h *http.Client) (Interface, error) {
	config := ConfigFor(inConfig)
	// for serializing the options
	config.GroupVersion = &schema.GroupVersion{}
	config.APIPath = "/if-you-see-this-search-for-the-break"

	restClient, err := rest.RESTClientForConfigAndClient(config, h)
	if err != nil {
		return nil, err
	}
	return &dynamicClient{client: restClient}, nil
}

type dynamicResourceClient struct {
	client    *dynamicClient
	namespace string
	resource  schema.GroupVersionResource
}

func (c *dynamicClient) Resource(resource schema.GroupVersionResource) NamespaceableResourceInterface {
	return &dynamicResourceClient{client: c, resource: resource}
}

func (c *dynamicResourceClient) Namespace(ns string) ResourceInterface {
	ret := *c
	ret.namespace = ns
	return &ret
}

func (c *dynamicResourceClient) Create(ctx context.Context, obj *unstructured.Unstructured, opts metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) {
	outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
	if err != nil {
		return nil, err
	}
	name := ""
	if len(subresources) > 0 {
		accessor, err := meta.Accessor(obj)
		if err != nil {
			return nil, err
		}
		name = accessor.GetName()
		if len(name) == 0 {
			return nil, fmt.Errorf("name is required")
		}
	}

	result := c.client.client.
		Post().
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(outBytes).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}

	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) Update(ctx context.Context, obj *unstructured.Unstructured, opts metav1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) {
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return nil, err
	}
	name := accessor.GetName()
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}
	outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
	if err != nil {
		return nil, err
	}

	result := c.client.client.
		Put().
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(outBytes).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}

	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) UpdateStatus(ctx context.Context, obj *unstructured.Unstructured, opts metav1.UpdateOptions) (*unstructured.Unstructured, error) {
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return nil, err
	}
	name := accessor.GetName()
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}

	outBytes, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
	if err != nil {
		return nil, err
	}

	result := c.client.client.
		Put().
		AbsPath(append(c.makeURLSegments(name), "status")...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(outBytes).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}

	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions, subresources ...string) error {
	if len(name) == 0 {
		return fmt.Errorf("name is required")
	}
	deleteOptionsByte, err := runtime.Encode(deleteOptionsCodec.LegacyCodec(schema.GroupVersion{Version: "v1"}), &opts)
	if err != nil {
		return err
	}

	result := c.client.client.
		Delete().
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(deleteOptionsByte).
		Do(ctx)
	return result.Error()
}

func (c *dynamicResourceClient) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOptions metav1.ListOptions) error {
	deleteOptionsByte, err := runtime.Encode(deleteOptionsCodec.LegacyCodec(schema.GroupVersion{Version: "v1"}), &opts)
	if err != nil {
		return err
	}

	result := c.client.client.
		Delete().
		AbsPath(c.makeURLSegments("")...).
		SetHeader("Content-Type", runtime.ContentTypeJSON).
		Body(deleteOptionsByte).
		SpecificallyVersionedParams(&listOptions, dynamicParameterCodec, versionV1).
		Do(ctx)
	return result.Error()
}

func (c *dynamicResourceClient) Get(ctx context.Context, name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) {
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}
	result := c.client.client.Get().AbsPath(append(c.makeURLSegments(name), subresources...)...).SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}
	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) List(ctx context.Context, opts metav1.ListOptions) (*unstructured.UnstructuredList, error) {
	result := c.client.client.Get().AbsPath(c.makeURLSegments("")...).SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}
	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	if list, ok := uncastObj.(*unstructured.UnstructuredList); ok {
		return list, nil
	}

	list, err := uncastObj.(*unstructured.Unstructured).ToList()
	if err != nil {
		return nil, err
	}
	return list, nil
}

func (c *dynamicResourceClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
	opts.Watch = true
	return c.client.client.Get().AbsPath(c.makeURLSegments("")...).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Watch(ctx)
}

func (c *dynamicResourceClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) {
	if len(name) == 0 {
		return nil, fmt.Errorf("name is required")
	}
	result := c.client.client.
		Patch(pt).
		AbsPath(append(c.makeURLSegments(name), subresources...)...).
		Body(data).
		SpecificallyVersionedParams(&opts, dynamicParameterCodec, versionV1).
		Do(ctx)
	if err := result.Error(); err != nil {
		return nil, err
	}
	retBytes, err := result.Raw()
	if err != nil {
		return nil, err
	}
	uncastObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, retBytes)
	if err != nil {
		return nil, err
	}
	return uncastObj.(*unstructured.Unstructured), nil
}

func (c *dynamicResourceClient) makeURLSegments(name string) []string {
	url := []string{}
	if len(c.resource.Group) == 0 {
		url = append(url, "api")
	} else {
		url = append(url, "apis", c.resource.Group)
	}
	url = append(url, c.resource.Version)

	if len(c.namespace) > 0 {
		url = append(url, "namespaces", c.namespace)
	}
	url = append(url, c.resource.Resource)

	if len(name) > 0 {
		url = append(url, name)
	}

	return url
}
<|endoftext|>


<commit_before>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Type represents an LXD network driver type.
type Type interface {
	FillConfig(config map[string]string) error
	Info() Info
	ValidateName(name string) error
	Type() string
	DBType() db.NetworkType
}

// Network represents an instantiated LXD network.
type Network interface {
	Type

	// Load.
	init(state *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode)

	// Config.
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Description() string
	Status() string
	LocalStatus() string
	Config() map[string]string
	IsUsed() (bool, error)
	IsManaged() bool
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType request.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType request.ClientType) error
}
<commit_msg>lxd/network/network/interface: Adds Project function<commit_after>package network

import (
	"net"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Type represents an LXD network driver type.
type Type interface {
	FillConfig(config map[string]string) error
	Info() Info
	ValidateName(name string) error
	Type() string
	DBType() db.NetworkType
}

// Network represents an instantiated LXD network.
type Network interface {
	Type

	// Load.
	init(state *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode)

	// Config.
	Validate(config map[string]string) error
	ID() int64
	Name() string
	Project() string
	Description() string
	Status() string
	LocalStatus() string
	Config() map[string]string
	IsUsed() (bool, error)
	IsManaged() bool
	DHCPv4Subnet() *net.IPNet
	DHCPv6Subnet() *net.IPNet
	DHCPv4Ranges() []shared.IPRange
	DHCPv6Ranges() []shared.IPRange

	// Actions.
	Create(clientType request.ClientType) error
	Start() error
	Stop() error
	Rename(name string) error
	Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error
	HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error
	Delete(clientType request.ClientType) error
}
<|endoftext|>


<commit_before>package WatchDog

import (
	"sync/atomic"
	"time"

	"github.com/6xiao/go/Common"
)

type WatchDog struct {
	wait  time.Duration
	hung  func()
	meat  int32
	count int
}

func NewDog(duration time.Duration, meat int32, hung func()) *WatchDog {
	d := new(WatchDog)
	d.wait = duration
	d.hung = hung
	d.meat = meat

	go d.eat()
	return d
}

func (this *WatchDog) eat() {
	defer Common.CheckPanic()

	for this.hung != nil && this.count < 3 {
		time.Sleep(this.wait)

		m := atomic.LoadInt32(&this.meat)
		if m < 0 {
			return
		}

		if m == 0 {
			this.hung()
			this.count++
		} else {
			atomic.StoreInt32(&this.meat, m/2)
			this.count = 0
		}
	}
}

func (this *WatchDog) Feed(meat uint16) bool {
	defer Common.CheckPanic()
	return atomic.AddInt32(&this.meat, int32(meat)) > 0
}

func (this *WatchDog) Kill() {
	defer Common.CheckPanic()
	atomic.StoreInt32(&this.meat, -65536)
}
<commit_msg>add Living method<commit_after>package WatchDog

import (
	"sync/atomic"
	"time"

	"github.com/6xiao/go/Common"
)

type WatchDog struct {
	wait  time.Duration
	hung  func()
	meat  int32
	count int32
}

func NewDog(duration time.Duration, meat int32, hung func()) *WatchDog {
	d := new(WatchDog)
	d.wait = duration
	d.hung = hung
	d.meat = meat

	go d.eat()
	return d
}

func (this *WatchDog) eat() {
	defer Common.CheckPanic()

	for this.hung != nil && atomic.LoadInt32(&this.count) < 3 {
		time.Sleep(this.wait)

		m := atomic.LoadInt32(&this.meat)
		if m < 0 {
			return
		}

		if m == 0 {
			this.hung()
			atomic.AddInt32(&this.count, 1)
		} else {
			atomic.StoreInt32(&this.meat, m/2)
			atomic.StoreInt32(&this.count, 0)
		}
	}
}

func (this *WatchDog) Feed(meat uint16) bool {
	defer Common.CheckPanic()

	// meat enought
	if atomic.LoadInt32(&this.meat) > 1024*65536 {
		return true
	}

	return atomic.AddInt32(&this.meat, int32(meat)) > 0
}

func (this *WatchDog) Kill() {
	defer Common.CheckPanic()
	atomic.StoreInt32(&this.meat, -65536)
}

func (this *WatchDog) Living() bool {
	defer Common.CheckPanic()
	return atomic.LoadInt32(&this.count) == 0
}
<|endoftext|>