<commit_before>package config

import (
	"github.com/ghodss/yaml"
	os_servers "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	log "tes/logger"
	pbr "tes/server/proto"
	"time"
)

// Weights describes the scheduler score weights.
// All fields should be float32 type.
type Weights map[string]float32

// StorageConfig describes configuration for all storage types
type StorageConfig struct {
	Local LocalStorage
	S3    S3Storage
	GS    GSStorage
}

// LocalStorage describes the directories TES can read from and write to
type LocalStorage struct {
	AllowedDirs []string
}

// GSStorage describes configuration for the Google Cloud storage backend.
type GSStorage struct {
	AccountFile string
	FromEnv     bool
}

// Valid validates the GSStorage configuration.
func (g GSStorage) Valid() bool {
	return g.FromEnv || g.AccountFile != ""
}

// Valid validates the LocalStorage configuration
func (l LocalStorage) Valid() bool {
	return len(l.AllowedDirs) > 0
}

// S3Storage describes the directories TES can read from and write to
type S3Storage struct {
	Endpoint string
	Key      string
	Secret   string
}

// Valid validates the LocalStorage configuration
func (l S3Storage) Valid() bool {
	return l.Endpoint != "" && l.Key != "" && l.Secret != ""
}

// LocalScheduler describes configuration for the local scheduler.
type LocalScheduler struct {
	Weights Weights
}

// OpenStackScheduler describes configuration for the openstack scheduler.
type OpenStackScheduler struct {
	KeyPair    string
	ConfigPath string
	Server     os_servers.CreateOpts
	Weights    Weights
}

// GCEScheduler describes configuration for the Google Cloud scheduler.
type GCEScheduler struct {
	AccountFile string
	Project     string
	Zone        string
	Templates   []string
	Weights     Weights
}

// Schedulers describes configuration for all schedulers.
type Schedulers struct {
	Local     LocalScheduler
	Condor    LocalScheduler
	OpenStack OpenStackScheduler
	GCE       GCEScheduler
}

// Config describes configuration for TES.
type Config struct {
	Storage       []*StorageConfig
	ServerAddress string
	Scheduler     string
	Schedulers    Schedulers
	Worker        Worker
	DBPath        string
	HTTPPort      string
	RPCPort       string
	ContentDir    string
	WorkDir       string
	LogLevel      string
	MaxJobLogSize int
	ScheduleRate  time.Duration
	ScheduleChunk int
	// How long to wait for a worker ping before marking it as dead
	WorkerPingTimeout time.Duration
	// How long to wait for worker initialization before marking it dead
	WorkerInitTimeout time.Duration
}

// DefaultConfig returns configuration with simple defaults.
func DefaultConfig() Config {
	workDir := "tes-work-dir"
	return Config{
		ServerAddress: "localhost:9090",
		DBPath:        path.Join(workDir, "tes_task.db"),
		HTTPPort:      "8000",
		RPCPort:       "9090",
		ContentDir:    defaultContentDir(),
		WorkDir:       workDir,
		LogLevel:      "debug",
		Scheduler:     "local",
		Schedulers: Schedulers{
			Local: LocalScheduler{},
			GCE: GCEScheduler{
				Weights: Weights{
					"startup time": 1.0,
				},
			},
		},
		Worker:            WorkerDefaultConfig(),
		MaxJobLogSize:     10000,
		ScheduleRate:      time.Second,
		ScheduleChunk:     10,
		WorkerPingTimeout: time.Minute,
		WorkerInitTimeout: time.Minute * 5,
	}
}

// Worker contains worker configuration.
type Worker struct {
	ID string
	// Address of the scheduler, e.g. "1.2.3.4:9090"
	ServerAddress string
	// Directory to write job files to
	WorkDir string
	// How long (seconds) to wait before tearing down an inactive worker
	// Default, -1, indicates to tear down the worker immediately after completing
	// its job
	Timeout time.Duration
	// How often the worker sends update requests to the server
	UpdateRate time.Duration
	// How often the worker sends job log updates
	LogUpdateRate time.Duration
	TrackerRate   time.Duration
	LogTailSize   int64
	Storage       []*StorageConfig
	LogPath       string
	LogLevel      string
	Resources     *pbr.Resources
	// Timeout duration for UpdateWorker() and UpdateJobLogs() RPC calls
	UpdateTimeout time.Duration
	Metadata      map[string]string
}

// WorkerDefaultConfig returns simple, default worker configuration.
func WorkerDefaultConfig() Worker {
	return Worker{
		ServerAddress: "localhost:9090",
		WorkDir:       "tes-work-dir",
		Timeout:       -1,
		// TODO these get reset to zero when not found in yaml?
		UpdateRate:    time.Second * 5,
		LogUpdateRate: time.Second * 5,
		TrackerRate:   time.Second * 5,
		LogTailSize:   10000,
		LogLevel:      "debug",
		UpdateTimeout: time.Second,
	}
}

// ToYaml formats the configuration into YAML and returns the bytes.
func (c Worker) ToYaml() []byte {
	// TODO handle error
	yamlstr, _ := yaml.Marshal(c)
	return yamlstr
}

// ToYamlFile writes the configuration to a YAML file.
func (c Worker) ToYamlFile(p string) {
	// TODO handle error
	ioutil.WriteFile(p, c.ToYaml(), 0600)
}

// ToYamlTempFile writes the configuration to a YAML temp. file.
func (c Worker) ToYamlTempFile(name string) (string, func()) {
	// I'm creating a temp. directory instead of a temp. file so that
	// the file can have an expected name. This is helpful for the HTCondor scheduler.
	tmpdir, _ := ioutil.TempDir("", "")

	cleanup := func() {
		os.RemoveAll(tmpdir)
	}

	p := filepath.Join(tmpdir, name)
	c.ToYamlFile(p)
	return p, cleanup
}

func defaultContentDir() string {
	// TODO this depends on having the entire repo available
	//      which prevents us from releasing a single binary.
	//      Not the worst, but maybe there's a good way to make it optional.
	// TODO handle error
	dir, _ := filepath.Abs(os.Args[0])
	return filepath.Join(dir, "..", "..", "share")
}

// ParseConfigFile parses a TES config file, which is formatted in YAML,
// and returns a Config struct.
func ParseConfigFile(path string, doc interface{}) error {
	source, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	err = yaml.Unmarshal(source, &doc)
	if err != nil {
		return err
	}
	return nil
}

// LoadConfigOrExit tries to load the config from the given file.
// If the file cannot be loaded, os.Exit() is called.
func LoadConfigOrExit(relpath string, config interface{}) {
	var err error
	if relpath != "" {
		var abspath string
		abspath, err = filepath.Abs(relpath)
		if err != nil {
			log.Error("Failure reading config", "path", abspath, "error", err)
			os.Exit(1)
		}
		log.Info("Using config file", "path", abspath)
		err = ParseConfigFile(abspath, &config)
		if err != nil {
			log.Error("Failure reading config", "path", abspath, "error", err)
			os.Exit(1)
		}
	}
}
<commit_msg>Default Disk resource for local worker<commit_after>package config

import (
	"github.com/ghodss/yaml"
	os_servers "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	log "tes/logger"
	pbr "tes/server/proto"
	"time"
)

// Weights describes the scheduler score weights.
// All fields should be float32 type.
type Weights map[string]float32

// StorageConfig describes configuration for all storage types
type StorageConfig struct {
	Local LocalStorage
	S3    S3Storage
	GS    GSStorage
}

// LocalStorage describes the directories TES can read from and write to
type LocalStorage struct {
	AllowedDirs []string
}

// GSStorage describes configuration for the Google Cloud storage backend.
type GSStorage struct {
	AccountFile string
	FromEnv     bool
}

// Valid validates the GSStorage configuration.
func (g GSStorage) Valid() bool {
	return g.FromEnv || g.AccountFile != ""
}

// Valid validates the LocalStorage configuration
func (l LocalStorage) Valid() bool {
	return len(l.AllowedDirs) > 0
}

// S3Storage describes the directories TES can read from and write to
type S3Storage struct {
	Endpoint string
	Key      string
	Secret   string
}

// Valid validates the LocalStorage configuration
func (l S3Storage) Valid() bool {
	return l.Endpoint != "" && l.Key != "" && l.Secret != ""
}

// LocalScheduler describes configuration for the local scheduler.
type LocalScheduler struct {
	Weights Weights
}

// OpenStackScheduler describes configuration for the openstack scheduler.
type OpenStackScheduler struct {
	KeyPair    string
	ConfigPath string
	Server     os_servers.CreateOpts
	Weights    Weights
}

// GCEScheduler describes configuration for the Google Cloud scheduler.
type GCEScheduler struct {
	AccountFile string
	Project     string
	Zone        string
	Templates   []string
	Weights     Weights
}

// Schedulers describes configuration for all schedulers.
type Schedulers struct {
	Local     LocalScheduler
	Condor    LocalScheduler
	OpenStack OpenStackScheduler
	GCE       GCEScheduler
}

// Config describes configuration for TES.
type Config struct {
	Storage       []*StorageConfig
	ServerAddress string
	Scheduler     string
	Schedulers    Schedulers
	Worker        Worker
	DBPath        string
	HTTPPort      string
	RPCPort       string
	ContentDir    string
	WorkDir       string
	LogLevel      string
	MaxJobLogSize int
	ScheduleRate  time.Duration
	ScheduleChunk int
	// How long to wait for a worker ping before marking it as dead
	WorkerPingTimeout time.Duration
	// How long to wait for worker initialization before marking it dead
	WorkerInitTimeout time.Duration
}

// DefaultConfig returns configuration with simple defaults.
func DefaultConfig() Config {
	workDir := "tes-work-dir"
	return Config{
		ServerAddress: "localhost:9090",
		DBPath:        path.Join(workDir, "tes_task.db"),
		HTTPPort:      "8000",
		RPCPort:       "9090",
		ContentDir:    defaultContentDir(),
		WorkDir:       workDir,
		LogLevel:      "debug",
		Scheduler:     "local",
		Schedulers: Schedulers{
			Local: LocalScheduler{},
			GCE: GCEScheduler{
				Weights: Weights{
					"startup time": 1.0,
				},
			},
		},
		Worker:            WorkerDefaultConfig(),
		MaxJobLogSize:     10000,
		ScheduleRate:      time.Second,
		ScheduleChunk:     10,
		WorkerPingTimeout: time.Minute,
		WorkerInitTimeout: time.Minute * 5,
	}
}

// Worker contains worker configuration.
type Worker struct {
	ID string
	// Address of the scheduler, e.g. "1.2.3.4:9090"
	ServerAddress string
	// Directory to write job files to
	WorkDir string
	// How long (seconds) to wait before tearing down an inactive worker
	// Default, -1, indicates to tear down the worker immediately after completing
	// its job
	Timeout time.Duration
	// How often the worker sends update requests to the server
	UpdateRate time.Duration
	// How often the worker sends job log updates
	LogUpdateRate time.Duration
	TrackerRate   time.Duration
	LogTailSize   int64
	Storage       []*StorageConfig
	LogPath       string
	LogLevel      string
	Resources     *pbr.Resources
	// Timeout duration for UpdateWorker() and UpdateJobLogs() RPC calls
	UpdateTimeout time.Duration
	Metadata      map[string]string
}

// WorkerDefaultConfig returns simple, default worker configuration.
func WorkerDefaultConfig() Worker {
	return Worker{
		ServerAddress: "localhost:9090",
		WorkDir:       "tes-work-dir",
		Timeout:       -1,
		// TODO these get reset to zero when not found in yaml?
		UpdateRate:    time.Second * 5,
		LogUpdateRate: time.Second * 5,
		TrackerRate:   time.Second * 5,
		LogTailSize:   10000,
		LogLevel:      "debug",
		UpdateTimeout: time.Second,
		Resources: &pbr.Resources{
			Disk: 100.0,
		},
	}
}

// ToYaml formats the configuration into YAML and returns the bytes.
func (c Worker) ToYaml() []byte {
	// TODO handle error
	yamlstr, _ := yaml.Marshal(c)
	return yamlstr
}

// ToYamlFile writes the configuration to a YAML file.
func (c Worker) ToYamlFile(p string) {
	// TODO handle error
	ioutil.WriteFile(p, c.ToYaml(), 0600)
}

// ToYamlTempFile writes the configuration to a YAML temp. file.
func (c Worker) ToYamlTempFile(name string) (string, func()) {
	// I'm creating a temp. directory instead of a temp. file so that
	// the file can have an expected name. This is helpful for the HTCondor scheduler.
	tmpdir, _ := ioutil.TempDir("", "")

	cleanup := func() {
		os.RemoveAll(tmpdir)
	}

	p := filepath.Join(tmpdir, name)
	c.ToYamlFile(p)
	return p, cleanup
}

func defaultContentDir() string {
	// TODO this depends on having the entire repo available
	//      which prevents us from releasing a single binary.
	//      Not the worst, but maybe there's a good way to make it optional.
	// TODO handle error
	dir, _ := filepath.Abs(os.Args[0])
	return filepath.Join(dir, "..", "..", "share")
}

// ParseConfigFile parses a TES config file, which is formatted in YAML,
// and returns a Config struct.
func ParseConfigFile(path string, doc interface{}) error {
	source, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	err = yaml.Unmarshal(source, &doc)
	if err != nil {
		return err
	}
	return nil
}

// LoadConfigOrExit tries to load the config from the given file.
// If the file cannot be loaded, os.Exit() is called.
func LoadConfigOrExit(relpath string, config interface{}) {
	var err error
	if relpath != "" {
		var abspath string
		abspath, err = filepath.Abs(relpath)
		if err != nil {
			log.Error("Failure reading config", "path", abspath, "error", err)
			os.Exit(1)
		}
		log.Info("Using config file", "path", abspath)
		err = ParseConfigFile(abspath, &config)
		if err != nil {
			log.Error("Failure reading config", "path", abspath, "error", err)
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
  "bytes"
  "config"
  "fmt"
  "io"
  _ "io/ioutil"
  "log"
  "net"
  "net/http"
  "net/url"
  "strings"
  "time"
)

/**
http Header Copay
*/
func header_copy(s http.Header, d *http.Header) {
  for hk, _ := range s {
    d.Set(hk, s.Get(hk))
  }
}

func show_error(w http.ResponseWriter, status int, msg []byte) {
  w.WriteHeader(status)
  w.Header().Set("Content-Type", "text/plain; charset=utf-8")
  w.Write(msg)
}

func access_log(w http.ResponseWriter, r *http.Request, query_url string, startTime time.Time) {

  remoteAddr := strings.Split(r.RemoteAddr, ":")[0] //客户端地址
  if remoteAddr == "[" || len(remoteAddr) == 0 {
    remoteAddr = "127.0.0.1"
  }
  r.ParseForm()
  var postValues []string
  for k, _ := range r.PostForm {
    postValues = append(postValues, fmt.Sprintf("%s=%s", k, r.FormValue(k)))
  }
  if len(postValues) == 0 {
    postValues = append(postValues, "-")
  }
  logLine := fmt.Sprintf(`%s [%s] S:"%s %s F:{%s} %s %s" D:"%s" %.05fs`,
    remoteAddr,
    time.Now().Format("2006-01-02 15:04:05.999999999 -0700 MST"),
    r.Method,
    r.RequestURI,
    strings.Join(postValues, "&"),
    r.Proto,
    w.Header().Get("Content-Length"),
    fmt.Sprintf("%s F:{%s}", query_url, strings.Join(postValues, "&")),
    time.Now().Sub(startTime).Seconds(),
  )
  g_env.AccessLog.Println(logLine)
}

func parse_querys(r *http.Request) (raw_query []string) {
  r.ParseForm()
  for k, _ := range r.Form {
    raw_query = append(raw_query, fmt.Sprintf("%s=%s", k, r.Form.Get(k)))
  }
  if len(r.Referer()) > 0 {
    if uri, err := url.Parse(r.Referer()); err == nil {
      for k, _ := range uri.Query() {
        raw_query = append(raw_query, fmt.Sprintf("%s=%s", k, uri.Query().Get(k)))
      }
    }
  }
  return
}

/**
获取目标地址
*/
func target_path(r *http.Request, cfg *config.Config) (t string) {
  if len(cfg.TargetPath) > 0 {
    t = cfg.TargetPath
  } else {
    t = r.URL.Path
  }
  return
}

/**
获取目标服务服务器
*/
func target_server(cfg *config.Config) (s string) {
  if len(cfg.TargetServer) > 0 {
    s = cfg.TargetServer
  } else {
    s = g_config.Default.TargetServer
  }
  return
}

/**
获取查询参数并做替换
*/
func swap_raw_query(r *http.Request, cfg *config.Config) (q string) {
  if len(cfg.TargetParamNameSwap) == 0 {
    q = r.URL.RawQuery
    return
  }
  var tmpSlice []string
  for k, _ := range r.URL.Query() {
    if v, ok := cfg.TargetParamNameSwap[k]; ok {
      tmpSlice = append(tmpSlice, fmt.Sprintf("%s=%s", v, r.URL.Query().Get(k)))
    } else {
      tmpSlice = append(tmpSlice, fmt.Sprintf("%s=%s", k, r.URL.Query().Get(k)))
    }
  }
  q = strings.Join(tmpSlice, "&")
  return
}

func timeout_dialer(conn_timeout int, rw_timeout int) func(net, addr string) (c net.Conn, err error) {
  return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, time.Duration(conn_timeout)*time.Second)
    if err != nil {
      log.Printf("Failed to connect to [%s]. Timed out after %d seconds\n", addr, rw_timeout)
      return nil, err
    }
    conn.SetDeadline(time.Now().Add(time.Duration(rw_timeout) * time.Second))
    return conn, nil
  }
}

func handler(w http.ResponseWriter, r *http.Request) {
  var (
    cfg                                  *config.Config
    cfg_err                              *config.ConfigErr
    conntction_timeout, response_timeout int
    req                                  *http.Request
    err                                  error
    //raw_body                             []byte
    raw_query []string //get ,post params
  )
  defer func() {
    if re := recover(); re != nil {
      g_env.ErrorLog.Println("Recovered in backendServer:", re)
    }
  }()

  defer r.Body.Close()

  start_at := time.Now()
  raw_query = parse_querys(r)

  if err != nil {
    g_env.ErrorLog.Println(req, err)
    show_error(w, http.StatusInternalServerError, []byte("Read Body Error."))
    return
  }

  //获取配置文件
  if cfg, cfg_err = g_config.FindBySourcePathAndParams(raw_query, r.URL.Path); cfg_err != nil {
    cfg = g_config.FindByParamsOrSourcePath(raw_query, r.URL.Path)
  }

  if conntction_timeout = cfg.ConnectionTimeout; conntction_timeout <= 0 {
    conntction_timeout = 15
  }

  if response_timeout = cfg.ResponseTimeout; response_timeout <= 0 {
    response_timeout = 120
  }

  transport := http.Transport{
    Dial: timeout_dialer(conntction_timeout, response_timeout),
    ResponseHeaderTimeout: time.Duration(response_timeout) * time.Second,
    DisableCompression:    false,
    DisableKeepAlives:     true,
    MaxIdleConnsPerHost:   2,
  }
  defer transport.CloseIdleConnections()

  client := &http.Client{
    Transport: &transport,
  }

  query_url, _ := url.Parse(target_server(cfg) + target_path(r, cfg) + "?" + swap_raw_query(r, cfg))

  switch r.Method {
  case "GET", "HEAD":
    req, err = http.NewRequest(r.Method, query_url.String(), nil)
  case "POST":
    req, err = http.NewRequest(r.Method, query_url.String(), bytes.NewBufferString(strings.Join(raw_query, "&")))
    req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
  default:
    show_error(w, http.StatusMethodNotAllowed, []byte("MethodNotAllowed"))
    return
  }
  req.Close = true

  header_copy(r.Header, &req.Header)

  if err != nil {
    g_env.ErrorLog.Println(err)
    show_error(w, http.StatusInternalServerError, []byte(err.Error()))
    return
  }

  resp, err := client.Do(req)
  defer resp.Body.Close()

  if err != nil {
    g_env.ErrorLog.Println(req, err)
    show_error(w, http.StatusInternalServerError, []byte(err.Error()))
    return
  }

  for hk, _ := range resp.Header {
    w.Header().Set(hk, resp.Header.Get(hk))
  }
  w.Header().Set("X-Transit-Ver", "0.0.1")

  if len(w.Header().Get("Server")) == 0 {
    w.Header().Set("Server", "X-Transit")
  }

  w.WriteHeader(resp.StatusCode)
  io.Copy(w, resp.Body)
  access_log(w, r, query_url.String(), start_at)
}

func Run() {
  g_env.ErrorLog.Printf("start@ %s:%d %v \n", g_config.Listen.Host, g_config.Listen.Port, time.Now())
  fmt.Printf("start@ %s:%d %v \n", g_config.Listen.Host, g_config.Listen.Port, time.Now())
  http.HandleFunc("/", handler)
  if err := http.ListenAndServe(fmt.Sprintf("%s:%d", g_config.Listen.Host, g_config.Listen.Port), nil); err != nil {
    log.Fatal(err)
  }
}
<commit_msg>定制http.Server<commit_after>package main

import (
  "bytes"
  "config"
  "fmt"
  "io"
  _ "io/ioutil"
  "log"
  "net"
  "net/http"
  "net/url"
  "strings"
  "time"
)

/**
http Header Copay
*/
func header_copy(s http.Header, d *http.Header) {
  for hk, _ := range s {
    d.Set(hk, s.Get(hk))
  }
}

func show_error(w http.ResponseWriter, status int, msg []byte) {
  w.WriteHeader(status)
  w.Header().Set("Content-Type", "text/plain; charset=utf-8")
  w.Write(msg)
}

func access_log(w http.ResponseWriter, r *http.Request, query_url string, startTime time.Time) {

  remoteAddr := strings.Split(r.RemoteAddr, ":")[0] //客户端地址
  if remoteAddr == "[" || len(remoteAddr) == 0 {
    remoteAddr = "127.0.0.1"
  }
  r.ParseForm()
  var postValues []string
  for k, _ := range r.PostForm {
    postValues = append(postValues, fmt.Sprintf("%s=%s", k, r.FormValue(k)))
  }
  if len(postValues) == 0 {
    postValues = append(postValues, "-")
  }
  logLine := fmt.Sprintf(`%s [%s] S:"%s %s F:{%s} %s %s" D:"%s" %.05fs`,
    remoteAddr,
    time.Now().Format("2006-01-02 15:04:05.999999999 -0700 MST"),
    r.Method,
    r.RequestURI,
    strings.Join(postValues, "&"),
    r.Proto,
    w.Header().Get("Content-Length"),
    fmt.Sprintf("%s F:{%s}", query_url, strings.Join(postValues, "&")),
    time.Now().Sub(startTime).Seconds(),
  )
  g_env.AccessLog.Println(logLine)
}

func parse_querys(r *http.Request) (raw_query []string) {
  r.ParseForm()
  for k, _ := range r.Form {
    raw_query = append(raw_query, fmt.Sprintf("%s=%s", k, r.Form.Get(k)))
  }
  if len(r.Referer()) > 0 {
    if uri, err := url.Parse(r.Referer()); err == nil {
      for k, _ := range uri.Query() {
        raw_query = append(raw_query, fmt.Sprintf("%s=%s", k, uri.Query().Get(k)))
      }
    }
  }
  return
}

/**
获取目标地址
*/
func target_path(r *http.Request, cfg *config.Config) (t string) {
  if len(cfg.TargetPath) > 0 {
    t = cfg.TargetPath
  } else {
    t = r.URL.Path
  }
  return
}

/**
获取目标服务服务器
*/
func target_server(cfg *config.Config) (s string) {
  if len(cfg.TargetServer) > 0 {
    s = cfg.TargetServer
  } else {
    s = g_config.Default.TargetServer
  }
  return
}

/**
获取查询参数并做替换
*/
func swap_raw_query(r *http.Request, cfg *config.Config) (q string) {
  if len(cfg.TargetParamNameSwap) == 0 {
    q = r.URL.RawQuery
    return
  }
  var tmpSlice []string
  for k, _ := range r.URL.Query() {
    if v, ok := cfg.TargetParamNameSwap[k]; ok {
      tmpSlice = append(tmpSlice, fmt.Sprintf("%s=%s", v, r.URL.Query().Get(k)))
    } else {
      tmpSlice = append(tmpSlice, fmt.Sprintf("%s=%s", k, r.URL.Query().Get(k)))
    }
  }
  q = strings.Join(tmpSlice, "&")
  return
}

func timeout_dialer(conn_timeout int, rw_timeout int) func(net, addr string) (c net.Conn, err error) {
  return func(netw, addr string) (net.Conn, error) {
    conn, err := net.DialTimeout(netw, addr, time.Duration(conn_timeout)*time.Second)
    if err != nil {
      log.Printf("Failed to connect to [%s]. Timed out after %d seconds\n", addr, rw_timeout)
      return nil, err
    }
    conn.SetDeadline(time.Now().Add(time.Duration(rw_timeout) * time.Second))
    return conn, nil
  }
}

func handler_func(w http.ResponseWriter, r *http.Request) {
  var (
    cfg                                  *config.Config
    cfg_err                              *config.ConfigErr
    conntction_timeout, response_timeout int
    req                                  *http.Request
    err                                  error
    //raw_body                             []byte
    raw_query []string //get ,post params
  )
  defer func() {
    if re := recover(); re != nil {
      g_env.ErrorLog.Println("Recovered in backendServer:", re)
    }
  }()

  defer r.Body.Close()

  start_at := time.Now()
  raw_query = parse_querys(r)

  if err != nil {
    g_env.ErrorLog.Println(req, err)
    http.Error(w, "Read Body Error.", http.StatusInternalServerError)
    return
  }

  //获取配置文件
  if cfg, cfg_err = g_config.FindBySourcePathAndParams(raw_query, r.URL.Path); cfg_err != nil {
    cfg = g_config.FindByParamsOrSourcePath(raw_query, r.URL.Path)
  }

  if conntction_timeout = cfg.ConnectionTimeout; conntction_timeout <= 0 {
    conntction_timeout = 15
  }

  if response_timeout = cfg.ResponseTimeout; response_timeout <= 0 {
    response_timeout = 120
  }

  transport := http.Transport{
    Dial: timeout_dialer(conntction_timeout, response_timeout),
    ResponseHeaderTimeout: time.Duration(response_timeout) * time.Second,
    DisableCompression:    false,
    DisableKeepAlives:     true,
    MaxIdleConnsPerHost:   2,
  }
  defer transport.CloseIdleConnections()

  client := &http.Client{
    Transport: &transport,
  }

  query_url, _ := url.Parse(target_server(cfg) + target_path(r, cfg) + "?" + swap_raw_query(r, cfg))

  switch r.Method {
  case "GET", "HEAD":
    req, err = http.NewRequest(r.Method, query_url.String(), nil)
  case "POST":
    req, err = http.NewRequest(r.Method, query_url.String(), bytes.NewBufferString(strings.Join(raw_query, "&")))
    req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
  default:
    http.Error(w, "MethodNotAllowed", http.StatusMethodNotAllowed)
    return
  }
  req.Close = true

  header_copy(r.Header, &req.Header)

  if err != nil {
    g_env.ErrorLog.Println(err)
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
  }

  resp, err := client.Do(req)
  defer resp.Body.Close()

  if err != nil {
    g_env.ErrorLog.Println(req, err)
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
  }

  for hk, _ := range resp.Header {
    w.Header().Set(hk, resp.Header.Get(hk))
  }
  w.Header().Set("X-Transit-Ver", "0.0.1")

  if len(w.Header().Get("Server")) == 0 {
    w.Header().Set("Server", "X-Transit")
  }

  w.WriteHeader(resp.StatusCode)
  io.Copy(w, resp.Body)
  access_log(w, r, query_url.String(), start_at)
}

func Run() {
  g_env.ErrorLog.Printf("start@ %s:%d %v \n", g_config.Listen.Host, g_config.Listen.Port, time.Now())
  fmt.Printf("start@ %s:%d %v \n", g_config.Listen.Host, g_config.Listen.Port, time.Now())
  s := &http.Server{
    Addr:           fmt.Sprintf("%s:%d", g_config.Listen.Host, g_config.Listen.Port),
    Handler:        http.HandlerFunc(handler_func),
    ReadTimeout:    10 * time.Second,
    WriteTimeout:   10 * time.Second,
    MaxHeaderBytes: 1 << 20,
  }
  //http.HandleFunc("/", handler)
  //if err := http.ListenAndServe(fmt.Sprintf("%s:%d", g_config.Listen.Host, g_config.Listen.Port), nil); err != nil {
  //log.Fatal(err)
  //}
  log.Fatal(s.ListenAndServe())
}
<|endoftext|>


<commit_before><commit_msg>Use Go's backtick syntax for strings with quotes<commit_after><|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"time"

	"k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

var (
	requestLatency = metrics.NewHistogramVec(
		&metrics.HistogramOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "request_duration_seconds",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"status"},
	)
	requestCount = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "request_total",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"status"},
	)
	fetchCount = metrics.NewGaugeVec(
		&metrics.GaugeOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "fetch_total",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"status"},
	)
	blockCount = metrics.NewGauge(
		&metrics.GaugeOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "block_count",
			StabilityLevel: metrics.ALPHA,
		},
	)
)

func init() {
	legacyregistry.MustRegister(
		requestLatency,
		requestCount,
		fetchCount,
		blockCount,
	)
}

const (
	hitTag  = "hit"
	missTag = "miss"

	fetchActiveTag = "active"
	fetchFailedTag = "error"
	fetchOkTag     = "ok"
)

type statsCollector struct{}

var stats = statsCollector{}

func (statsCollector) authenticating() func(hit bool) {
	start := time.Now()
	return func(hit bool) {
		var tag string
		if hit {
			tag = hitTag
		} else {
			tag = missTag
		}

		latency := time.Since(start)

		requestCount.WithLabelValues(tag).Inc()
		requestLatency.WithLabelValues(tag).Observe(float64(latency.Milliseconds()) / 1000)
	}
}

func (statsCollector) blocking() func() {
	blockCount.Inc()
	return blockCount.Dec
}

func (statsCollector) fetching() func(ok bool) {
	fetchCount.WithLabelValues(fetchActiveTag).Inc()
	return func(ok bool) {
		var tag string
		if ok {
			tag = fetchOkTag
		} else {
			tag = fetchFailedTag
		}

		fetchCount.WithLabelValues(tag).Dec()
	}
}
<commit_msg>token cache: make fetch_total a counter<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"time"

	"k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

var (
	requestLatency = metrics.NewHistogramVec(
		&metrics.HistogramOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "request_duration_seconds",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"status"},
	)
	requestCount = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "request_total",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"status"},
	)
	fetchCount = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "fetch_total",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"status"},
	)
	activeFetchCount = metrics.NewGaugeVec(
		&metrics.GaugeOpts{
			Namespace:      "authentication",
			Subsystem:      "token_cache",
			Name:           "active_fetch_count",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"status"},
	)
)

func init() {
	legacyregistry.MustRegister(
		requestLatency,
		requestCount,
		fetchCount,
		activeFetchCount,
	)
}

const (
	hitTag  = "hit"
	missTag = "miss"

	fetchFailedTag = "error"
	fetchOkTag     = "ok"

	fetchInFlightTag = "in_flight"
	fetchBlockedTag  = "blocked"
)

type statsCollector struct{}

var stats = statsCollector{}

func (statsCollector) authenticating() func(hit bool) {
	start := time.Now()
	return func(hit bool) {
		var tag string
		if hit {
			tag = hitTag
		} else {
			tag = missTag
		}

		latency := time.Since(start)

		requestCount.WithLabelValues(tag).Inc()
		requestLatency.WithLabelValues(tag).Observe(float64(latency.Milliseconds()) / 1000)
	}
}

func (statsCollector) blocking() func() {
	activeFetchCount.WithLabelValues(fetchBlockedTag).Inc()
	return activeFetchCount.WithLabelValues(fetchBlockedTag).Dec
}

func (statsCollector) fetching() func(ok bool) {
	activeFetchCount.WithLabelValues(fetchInFlightTag).Inc()
	return func(ok bool) {
		var tag string
		if ok {
			tag = fetchOkTag
		} else {
			tag = fetchFailedTag
		}

		fetchCount.WithLabelValues(tag).Inc()

		activeFetchCount.WithLabelValues(fetchInFlightTag).Dec()
	}
}
<|endoftext|>


<commit_before>package acme

import (
	"fmt"
	"net"
	"net/http"
	"strings"
)

// HTTPProviderServer implements ChallengeProvider for `http-01` challenge
// It may be instantiated without using the NewHTTPProviderServer function if
// you want only to use the default values.
type HTTPProviderServer struct {
	iface    string
	port     string
	done     chan bool
	listener net.Listener
}

// NewHTTPProviderServer creates a new HTTPProviderServer on the selected interface and port.
// Setting iface and / or port to an empty string will make the server fall back to
// the "any" interface and port 80 respectively.
func NewHTTPProviderServer(iface, port string) *HTTPProviderServer {
	return &HTTPProviderServer{iface: iface, port: port}
}

// Present starts a web server and makes the token available at `HTTP01ChallengePath(token)` for web requests.
func (s *HTTPProviderServer) Present(domain, token, keyAuth string) error {
	if s.port == "" {
		s.port = "80"
	}

	var err error
	s.listener, err = net.Listen("tcp", net.JoinHostPort(s.iface, s.port))
	if err != nil {
		return fmt.Errorf("Could not start HTTP server for challenge -> %v", err)
	}

	s.done = make(chan bool)
	go s.serve(domain, token, keyAuth)
	return nil
}

// CleanUp closes the HTTP server and removes the token from `HTTP01ChallengePath(token)`
func (s *HTTPProviderServer) CleanUp(domain, token, keyAuth string) error {
	if s.listener == nil {
		return nil
	}
	s.listener.Close()
	<-s.done
	return nil
}

func (s *HTTPProviderServer) serve(domain, token, keyAuth string) {
	path := HTTP01ChallengePath(token)

	// The handler validates the HOST header and request type.
	// For validation it then writes the token the server returned with the challenge
	mux := http.NewServeMux()
	mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.Host, domain) && r.Method == "GET" {
			w.Header().Add("Content-Type", "text/plain")
			w.Write([]byte(keyAuth))
			logf("[INFO][%s] Served key authentication", domain)
		} else {
			logf("[INFO] Received request for domain %s with method %s", r.Host, r.Method)
			w.Write([]byte("TEST"))
		}
	})

	httpServer := &http.Server{
		Handler: mux,
	}
	// Once httpServer is shut down we don't want any lingering
	// connections, so disable KeepAlives.
	httpServer.SetKeepAlivesEnabled(false)
	httpServer.Serve(s.listener)
	s.done <- true
}
<commit_msg>Tweak log message for a wrong host in HTTP-01<commit_after>package acme

import (
	"fmt"
	"net"
	"net/http"
	"strings"
)

// HTTPProviderServer implements ChallengeProvider for `http-01` challenge
// It may be instantiated without using the NewHTTPProviderServer function if
// you want only to use the default values.
type HTTPProviderServer struct {
	iface    string
	port     string
	done     chan bool
	listener net.Listener
}

// NewHTTPProviderServer creates a new HTTPProviderServer on the selected interface and port.
// Setting iface and / or port to an empty string will make the server fall back to
// the "any" interface and port 80 respectively.
func NewHTTPProviderServer(iface, port string) *HTTPProviderServer {
	return &HTTPProviderServer{iface: iface, port: port}
}

// Present starts a web server and makes the token available at `HTTP01ChallengePath(token)` for web requests.
func (s *HTTPProviderServer) Present(domain, token, keyAuth string) error {
	if s.port == "" {
		s.port = "80"
	}

	var err error
	s.listener, err = net.Listen("tcp", net.JoinHostPort(s.iface, s.port))
	if err != nil {
		return fmt.Errorf("Could not start HTTP server for challenge -> %v", err)
	}

	s.done = make(chan bool)
	go s.serve(domain, token, keyAuth)
	return nil
}

// CleanUp closes the HTTP server and removes the token from `HTTP01ChallengePath(token)`
func (s *HTTPProviderServer) CleanUp(domain, token, keyAuth string) error {
	if s.listener == nil {
		return nil
	}
	s.listener.Close()
	<-s.done
	return nil
}

func (s *HTTPProviderServer) serve(domain, token, keyAuth string) {
	path := HTTP01ChallengePath(token)

	// The handler validates the HOST header and request type.
	// For validation it then writes the token the server returned with the challenge
	mux := http.NewServeMux()
	mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.Host, domain) && r.Method == "GET" {
			w.Header().Add("Content-Type", "text/plain")
			w.Write([]byte(keyAuth))
			logf("[INFO][%s] Served key authentication", domain)
		} else {
			logf("[WARN] Received request for domain %s with method %s but the domain did not match any challenge. Please ensure your are passing the HOST header properly.", r.Host, r.Method)
			w.Write([]byte("TEST"))
		}
	})

	httpServer := &http.Server{
		Handler: mux,
	}
	// Once httpServer is shut down we don't want any lingering
	// connections, so disable KeepAlives.
	httpServer.SetKeepAlivesEnabled(false)
	httpServer.Serve(s.listener)
	s.done <- true
}
<|endoftext|>


<commit_before>package scmauth

import (
	"path/filepath"
)

const GitConfigName = "gitconfig"

// GitConfig implements SCMAuth interface for using a custom .gitconfig file
type GitConfig struct{}

// Setup adds the secret .gitconfig as an include to the .gitconfig file to be used in the build
func (_ GitConfig) Setup(baseDir string) error {
	return ensureGitConfigIncludes(filepath.Join(baseDir, GitConfigName))
}

// Name returns the name of this auth method.
func (_ GitConfig) Name() string {
	return GitConfigName
}

// Handles returns true if the secret file is a gitconfig
func (_ GitConfig) Handles(name string) bool {
	return name == GitConfigName
}
<commit_msg>Rename gitconfig secret key to .gitconfig<commit_after>package scmauth

import (
	"path/filepath"
)

const GitConfigName = ".gitconfig"

// GitConfig implements SCMAuth interface for using a custom .gitconfig file
type GitConfig struct{}

// Setup adds the secret .gitconfig as an include to the .gitconfig file to be used in the build
func (_ GitConfig) Setup(baseDir string) error {
	return ensureGitConfigIncludes(filepath.Join(baseDir, GitConfigName))
}

// Name returns the name of this auth method.
func (_ GitConfig) Name() string {
	return GitConfigName
}

// Handles returns true if the secret file is a gitconfig
func (_ GitConfig) Handles(name string) bool {
	return name == GitConfigName
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/config"
)

type EventType int

const (
	ADD_EVENT EventType = iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry struct {
	Value UniqueCopyable
	Event EventType
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *Entry) Copy() Copyable {
	return &Entry{Value: e.Value.Copy().(UniqueCopyable), Event: e.Event}
}

// deliver a message
type pigeon func(msg *Entry)

func dead(msg *Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
// TODO(jdef): I used to think that I'd need history of state changes recorded
// in `history` but it's turning out that I really only need to maintain the
// the current state, so `history` should revert back into `items`
type HistoricalFIFO struct {
	lock    sync.RWMutex
	cond    sync.Cond
	history map[string][]*Entry // We depend on the property that items in the queue are in the set.
	queue   []string
	carrier pigeon // may be dead, but never nil
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entries, exists := f.history[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		head := entries[len(entries)-1]
		if head.Event == DELETE_EVENT || head.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj, nil)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entries, exists := f.history[id]; exists {
		head := entries[len(entries)-1]
		if head.Event != t {
			return
		} else if head.Event == DELETE_EVENT || head.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj, nil)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (*Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	entries, exists := f.history[id]
	if exists {
		//TODO(jdef): set a timer to expunge the history for this object
		//or else, simply do garbage collection every n'th merge(), removing
		//expired DELETE entries
		head := entries[len(entries)-1]
		deleteEvent = &Entry{Value: head.Value, Event: DELETE_EVENT}
		f.history[id] = append(entries, deleteEvent)
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entries := range f.history {
		head := entries[len(entries)-1]
		if head.Event == DELETE_EVENT || head.Event == POP_EVENT {
			continue
		}
		list = append(list, head.Value.Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entries := range c.history {
		head := entries[len(entries)-1]
		if head.Event == DELETE_EVENT || head.Event == POP_EVENT {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entries, exists := f.history[id]
	head := entries[len(entries)-1]
	if exists && !(head.Event == DELETE_EVENT || head.Event == POP_EVENT) {
		return head.Value.Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (*Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		entries, ok := f.history[id]
		head := entries[len(entries)-1]
		if !ok || head.Event == DELETE_EVENT || head.Event == POP_EVENT {
			// Item may have been deleted subsequently.
			continue
		}
		value := head.Value
		popEvent = &Entry{Value: value, Event: POP_EVENT}
		f.history[id] = append(entries, popEvent)
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]*Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj, nil)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

type mergeFilter func(older, newer *Entry) bool

// expects that caller has already locked around state
//TODO(jdef): eliminate the use of mergeFilter if we don't end up needing it
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable, accepts mergeFilter) (notifications []*Entry) {
	entries, exists := f.history[id]
	if !exists {
		entries = make([]*Entry, 0, 3)
		e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
		if accepts == nil || accepts(nil, e) {
			f.history[id] = append(entries, e)
			notifications = append(notifications, e)
		}
	} else {
		head := entries[len(entries)-1]
		if head.Event != DELETE_EVENT && head.Value.GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			e1 := &Entry{Value: head.Value, Event: DELETE_EVENT}
			e2 := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
			if accepts == nil || accepts(e1, e2) {
				f.history[id] = append(entries, e1, e2)
				notifications = append(notifications, e1, e2)
			}
		} else if !reflect.DeepEqual(obj, head.Value) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: UPDATE_EVENT}
			if accepts == nil || accepts(head, e) {
				f.history[id] = append(entries, e)
				notifications = append(notifications, e)
			}
		}
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(mux *config.Mux) *HistoricalFIFO {
	carrier := dead
	if mux != nil {
		//TODO(jdef): append a UUID to "fifo" here?
		ch := mux.Channel("fifo")
		carrier = func(msg *Entry) {
			if msg != nil {
				ch <- msg.Copy()
			}
		}
	}
	f := &HistoricalFIFO{
		history: map[string][]*Entry{},
		queue:   []string{},
		carrier: carrier,
	}
	f.cond.L = &f.lock
	return f
}
<commit_msg>revert history to items; update Replace to consider items not present in the replacment map as deleted<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/config"
)

type EventType int

const (
	ADD_EVENT EventType = iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry struct {
	Value UniqueCopyable
	Event EventType
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *Entry) Copy() Copyable {
	return &Entry{Value: e.Value.Copy().(UniqueCopyable), Event: e.Event}
}

// deliver a message
type pigeon func(msg *Entry)

func dead(msg *Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
type HistoricalFIFO struct {
	lock    sync.RWMutex
	cond    sync.Cond
	items   map[string]*Entry // We depend on the property that items in the queue are in the set.
	queue   []string
	carrier pigeon // may be dead, but never nil
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; exists {
		if entry.Event != t {
			return
		} else if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (*Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	entry, exists := f.items[id]
	if exists {
		//TODO(jdef): set a timer to expunge the history for this object
		//or else, simply do garbage collection every n'th merge(), removing
		//expired DELETE entries
		deleteEvent = &Entry{Value: entry.Value, Event: DELETE_EVENT}
		f.items[id] = deleteEvent
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entry := range f.items {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			continue
		}
		list = append(list, entry.Value.Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entry := range c.items {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entry, exists := f.items[id]
	if exists && !(entry.Event == DELETE_EVENT || entry.Event == POP_EVENT) {
		return entry.Value.Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (*Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		entry, ok := f.items[id]
		if !ok || entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			// Item may have been deleted subsequently.
			continue
		}
		value := entry.Value
		popEvent = &Entry{Value: value, Event: POP_EVENT}
		f.items[id] = popEvent
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]*Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	for id, v := range f.items {
		if _, exists := idToObj[id]; !exists && v.Event != DELETE_EVENT {
			// a non-deleted entry in the items list that doesn't show up in the
			// new list: mark it as deleted
			e := &Entry{Value: v.Value, Event: DELETE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

// expects that caller has already locked around state
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []*Entry) {
	entry, exists := f.items[id]
	if !exists {
		e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
		f.items[id] = e
		notifications = append(notifications, e)
	} else {
		if entry.Event != DELETE_EVENT && entry.Value.GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			e1 := &Entry{Value: entry.Value, Event: DELETE_EVENT}
			e2 := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
			f.items[id] = e2
			notifications = append(notifications, e1, e2)
		} else if !reflect.DeepEqual(obj, entry.Value) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: UPDATE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(mux *config.Mux) *HistoricalFIFO {
	carrier := dead
	if mux != nil {
		//TODO(jdef): append a UUID to "fifo" here?
		ch := mux.Channel("fifo")
		carrier = func(msg *Entry) {
			if msg != nil {
				ch <- msg.Copy()
			}
		}
	}
	f := &HistoricalFIFO{
		items:   map[string]*Entry{},
		queue:   []string{},
		carrier: carrier,
	}
	f.cond.L = &f.lock
	return f
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/apigateway"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsApiGatewayRestApi() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsApiGatewayRestApiCreate,
		Read:   resourceAwsApiGatewayRestApiRead,
		Update: resourceAwsApiGatewayRestApiUpdate,
		Delete: resourceAwsApiGatewayRestApiDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},

			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"policy": {
				Type:             schema.TypeString,
				Computed:         true,
				Optional:         true,
				ValidateFunc:     validateJsonString,
				DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs,
			},

			"binary_media_types": {
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},

			"body": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"minimum_compression_size": {
				Type:         schema.TypeInt,
				Optional:     true,
				Default:      -1,
				ValidateFunc: validateIntegerInRange(-1, 10485760),
			},

			"root_resource_id": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"created_date": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsApiGatewayRestApiCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Creating API Gateway")

	var description *string
	if d.Get("description").(string) != "" {
		description = aws.String(d.Get("description").(string))
	}

	var policy *string
	if d.Get("policy").(string) != "" {
		policy = aws.String(d.Get("policy").(string))
	}

	params := &apigateway.CreateRestApiInput{
		Name:        aws.String(d.Get("name").(string)),
		Description: description,
		Policy:      policy,
	}

	binaryMediaTypes, binaryMediaTypesOk := d.GetOk("binary_media_types")
	if binaryMediaTypesOk {
		params.BinaryMediaTypes = expandStringList(binaryMediaTypes.([]interface{}))
	}

	minimumCompressionSize := d.Get("minimum_compression_size").(int)
	if minimumCompressionSize > -1 {
		params.MinimumCompressionSize = aws.Int64(int64(minimumCompressionSize))
	}

	gateway, err := conn.CreateRestApi(params)
	if err != nil {
		return fmt.Errorf("Error creating API Gateway: %s", err)
	}

	d.SetId(*gateway.Id)

	if body, ok := d.GetOk("body"); ok {
		log.Printf("[DEBUG] Initializing API Gateway from OpenAPI spec %s", d.Id())
		_, err := conn.PutRestApi(&apigateway.PutRestApiInput{
			RestApiId: gateway.Id,
			Mode:      aws.String(apigateway.PutModeOverwrite),
			Body:      []byte(body.(string)),
		})
		if err != nil {
			return errwrap.Wrapf("Error creating API Gateway specification: {{err}}", err)
		}
	}

	if err = resourceAwsApiGatewayRestApiRefreshResources(d, meta); err != nil {
		return err
	}

	return resourceAwsApiGatewayRestApiRead(d, meta)
}

func resourceAwsApiGatewayRestApiRefreshResources(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway

	resp, err := conn.GetResources(&apigateway.GetResourcesInput{
		RestApiId: aws.String(d.Id()),
	})
	if err != nil {
		return err
	}

	for _, item := range resp.Items {
		if *item.Path == "/" {
			d.Set("root_resource_id", item.Id)
			break
		}
	}

	return nil
}

func resourceAwsApiGatewayRestApiRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Reading API Gateway %s", d.Id())

	api, err := conn.GetRestApi(&apigateway.GetRestApiInput{
		RestApiId: aws.String(d.Id()),
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NotFoundException" {
			log.Printf("[WARN] API Gateway (%s) not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("name", api.Name)
	d.Set("description", api.Description)
	d.Set("policy", api.Policy)

	d.Set("binary_media_types", api.BinaryMediaTypes)
	if api.MinimumCompressionSize == nil {
		d.Set("minimum_compression_size", -1)
	} else {
		d.Set("minimum_compression_size", api.MinimumCompressionSize)
	}
	if err := d.Set("created_date", api.CreatedDate.Format(time.RFC3339)); err != nil {
		log.Printf("[DEBUG] Error setting created_date: %s", err)
	}

	return nil
}

func resourceAwsApiGatewayRestApiUpdateOperations(d *schema.ResourceData) []*apigateway.PatchOperation {
	operations := make([]*apigateway.PatchOperation, 0)

	if d.HasChange("name") {
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/name"),
			Value: aws.String(d.Get("name").(string)),
		})
	}

	if d.HasChange("description") {
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/description"),
			Value: aws.String(d.Get("description").(string)),
		})
	}

	if d.HasChange("policy") {
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/policy"),
			Value: aws.String(d.Get("policy").(string)),
		})
	}

	if d.HasChange("minimum_compression_size") {
		minimumCompressionSize := d.Get("minimum_compression_size").(int)
		var value string
		if minimumCompressionSize > -1 {
			value = strconv.Itoa(minimumCompressionSize)
		}
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/minimumCompressionSize"),
			Value: aws.String(value),
		})
	}

	if d.HasChange("binary_media_types") {
		o, n := d.GetChange("binary_media_types")
		prefix := "binaryMediaTypes"

		old := o.([]interface{})
		new := n.([]interface{})

		// Remove every binary media types. Simpler to remove and add new ones,
		// since there are no replacings.
		for _, v := range old {
			operations = append(operations, &apigateway.PatchOperation{
				Op:   aws.String("remove"),
				Path: aws.String(fmt.Sprintf("/%s/%s", prefix, escapeJsonPointer(v.(string)))),
			})
		}

		// Handle additions
		if len(new) > 0 {
			for _, v := range new {
				operations = append(operations, &apigateway.PatchOperation{
					Op:   aws.String("add"),
					Path: aws.String(fmt.Sprintf("/%s/%s", prefix, escapeJsonPointer(v.(string)))),
				})
			}
		}
	}

	return operations
}

func resourceAwsApiGatewayRestApiUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Updating API Gateway %s", d.Id())

	if d.HasChange("body") {
		if body, ok := d.GetOk("body"); ok {
			log.Printf("[DEBUG] Updating API Gateway from OpenAPI spec: %s", d.Id())
			_, err := conn.PutRestApi(&apigateway.PutRestApiInput{
				RestApiId: aws.String(d.Id()),
				Mode:      aws.String(apigateway.PutModeOverwrite),
				Body:      []byte(body.(string)),
			})
			if err != nil {
				return errwrap.Wrapf("Error updating API Gateway specification: {{err}}", err)
			}
		}
	}

	_, err := conn.UpdateRestApi(&apigateway.UpdateRestApiInput{
		RestApiId:       aws.String(d.Id()),
		PatchOperations: resourceAwsApiGatewayRestApiUpdateOperations(d),
	})

	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Updated API Gateway %s", d.Id())

	return resourceAwsApiGatewayRestApiRead(d, meta)
}

func resourceAwsApiGatewayRestApiDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Deleting API Gateway: %s", d.Id())

	return resource.Retry(10*time.Minute, func() *resource.RetryError {
		_, err := conn.DeleteRestApi(&apigateway.DeleteRestApiInput{
			RestApiId: aws.String(d.Id()),
		})
		if err == nil {
			return nil
		}

		if apigatewayErr, ok := err.(awserr.Error); ok && apigatewayErr.Code() == "NotFoundException" {
			return nil
		}

		return resource.NonRetryableError(err)
	})
}
<commit_msg>Remove computed attribute<commit_after>package aws

import (
	"fmt"
	"log"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/apigateway"
	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsApiGatewayRestApi() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsApiGatewayRestApiCreate,
		Read:   resourceAwsApiGatewayRestApiRead,
		Update: resourceAwsApiGatewayRestApiUpdate,
		Delete: resourceAwsApiGatewayRestApiDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
			},

			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"policy": {
				Type:             schema.TypeString,
				Optional:         true,
				ValidateFunc:     validateJsonString,
				DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs,
			},

			"binary_media_types": {
				Type:     schema.TypeList,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},

			"body": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"minimum_compression_size": {
				Type:         schema.TypeInt,
				Optional:     true,
				Default:      -1,
				ValidateFunc: validateIntegerInRange(-1, 10485760),
			},

			"root_resource_id": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"created_date": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsApiGatewayRestApiCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Creating API Gateway")

	var description *string
	if d.Get("description").(string) != "" {
		description = aws.String(d.Get("description").(string))
	}

	var policy *string
	if d.Get("policy").(string) != "" {
		policy = aws.String(d.Get("policy").(string))
	}

	params := &apigateway.CreateRestApiInput{
		Name:        aws.String(d.Get("name").(string)),
		Description: description,
		Policy:      policy,
	}

	binaryMediaTypes, binaryMediaTypesOk := d.GetOk("binary_media_types")
	if binaryMediaTypesOk {
		params.BinaryMediaTypes = expandStringList(binaryMediaTypes.([]interface{}))
	}

	minimumCompressionSize := d.Get("minimum_compression_size").(int)
	if minimumCompressionSize > -1 {
		params.MinimumCompressionSize = aws.Int64(int64(minimumCompressionSize))
	}

	gateway, err := conn.CreateRestApi(params)
	if err != nil {
		return fmt.Errorf("Error creating API Gateway: %s", err)
	}

	d.SetId(*gateway.Id)

	if body, ok := d.GetOk("body"); ok {
		log.Printf("[DEBUG] Initializing API Gateway from OpenAPI spec %s", d.Id())
		_, err := conn.PutRestApi(&apigateway.PutRestApiInput{
			RestApiId: gateway.Id,
			Mode:      aws.String(apigateway.PutModeOverwrite),
			Body:      []byte(body.(string)),
		})
		if err != nil {
			return errwrap.Wrapf("Error creating API Gateway specification: {{err}}", err)
		}
	}

	if err = resourceAwsApiGatewayRestApiRefreshResources(d, meta); err != nil {
		return err
	}

	return resourceAwsApiGatewayRestApiRead(d, meta)
}

func resourceAwsApiGatewayRestApiRefreshResources(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway

	resp, err := conn.GetResources(&apigateway.GetResourcesInput{
		RestApiId: aws.String(d.Id()),
	})
	if err != nil {
		return err
	}

	for _, item := range resp.Items {
		if *item.Path == "/" {
			d.Set("root_resource_id", item.Id)
			break
		}
	}

	return nil
}

func resourceAwsApiGatewayRestApiRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Reading API Gateway %s", d.Id())

	api, err := conn.GetRestApi(&apigateway.GetRestApiInput{
		RestApiId: aws.String(d.Id()),
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NotFoundException" {
			log.Printf("[WARN] API Gateway (%s) not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("name", api.Name)
	d.Set("description", api.Description)
	d.Set("policy", api.Policy)

	d.Set("binary_media_types", api.BinaryMediaTypes)
	if api.MinimumCompressionSize == nil {
		d.Set("minimum_compression_size", -1)
	} else {
		d.Set("minimum_compression_size", api.MinimumCompressionSize)
	}
	if err := d.Set("created_date", api.CreatedDate.Format(time.RFC3339)); err != nil {
		log.Printf("[DEBUG] Error setting created_date: %s", err)
	}

	return nil
}

func resourceAwsApiGatewayRestApiUpdateOperations(d *schema.ResourceData) []*apigateway.PatchOperation {
	operations := make([]*apigateway.PatchOperation, 0)

	if d.HasChange("name") {
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/name"),
			Value: aws.String(d.Get("name").(string)),
		})
	}

	if d.HasChange("description") {
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/description"),
			Value: aws.String(d.Get("description").(string)),
		})
	}

	if d.HasChange("policy") {
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/policy"),
			Value: aws.String(d.Get("policy").(string)),
		})
	}

	if d.HasChange("minimum_compression_size") {
		minimumCompressionSize := d.Get("minimum_compression_size").(int)
		var value string
		if minimumCompressionSize > -1 {
			value = strconv.Itoa(minimumCompressionSize)
		}
		operations = append(operations, &apigateway.PatchOperation{
			Op:    aws.String("replace"),
			Path:  aws.String("/minimumCompressionSize"),
			Value: aws.String(value),
		})
	}

	if d.HasChange("binary_media_types") {
		o, n := d.GetChange("binary_media_types")
		prefix := "binaryMediaTypes"

		old := o.([]interface{})
		new := n.([]interface{})

		// Remove every binary media types. Simpler to remove and add new ones,
		// since there are no replacings.
		for _, v := range old {
			operations = append(operations, &apigateway.PatchOperation{
				Op:   aws.String("remove"),
				Path: aws.String(fmt.Sprintf("/%s/%s", prefix, escapeJsonPointer(v.(string)))),
			})
		}

		// Handle additions
		if len(new) > 0 {
			for _, v := range new {
				operations = append(operations, &apigateway.PatchOperation{
					Op:   aws.String("add"),
					Path: aws.String(fmt.Sprintf("/%s/%s", prefix, escapeJsonPointer(v.(string)))),
				})
			}
		}
	}

	return operations
}

func resourceAwsApiGatewayRestApiUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Updating API Gateway %s", d.Id())

	if d.HasChange("body") {
		if body, ok := d.GetOk("body"); ok {
			log.Printf("[DEBUG] Updating API Gateway from OpenAPI spec: %s", d.Id())
			_, err := conn.PutRestApi(&apigateway.PutRestApiInput{
				RestApiId: aws.String(d.Id()),
				Mode:      aws.String(apigateway.PutModeOverwrite),
				Body:      []byte(body.(string)),
			})
			if err != nil {
				return errwrap.Wrapf("Error updating API Gateway specification: {{err}}", err)
			}
		}
	}

	_, err := conn.UpdateRestApi(&apigateway.UpdateRestApiInput{
		RestApiId:       aws.String(d.Id()),
		PatchOperations: resourceAwsApiGatewayRestApiUpdateOperations(d),
	})

	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Updated API Gateway %s", d.Id())

	return resourceAwsApiGatewayRestApiRead(d, meta)
}

func resourceAwsApiGatewayRestApiDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).apigateway
	log.Printf("[DEBUG] Deleting API Gateway: %s", d.Id())

	return resource.Retry(10*time.Minute, func() *resource.RetryError {
		_, err := conn.DeleteRestApi(&apigateway.DeleteRestApiInput{
			RestApiId: aws.String(d.Id()),
		})
		if err == nil {
			return nil
		}

		if apigatewayErr, ok := err.(awserr.Error); ok && apigatewayErr.Code() == "NotFoundException" {
			return nil
		}

		return resource.NonRetryableError(err)
	})
}
<|endoftext|>


<commit_before>// +build windows,amd64
package winapi

import (
    "os"
    "fmt"
    "sort"
    "strings"
    "syscall"
    "unsafe"
    "reflect"

    so "github.com/iamacarpet/go-win64api/shared"
)

var (
    modSecur32                          = syscall.NewLazyDLL("secur32.dll")
    sessLsaFreeReturnBuffer             = modSecur32.NewProc("LsaFreeReturnBuffer")
    sessLsaEnumerateLogonSessions       = modSecur32.NewProc("LsaEnumerateLogonSessions")
    sessLsaGetLogonSessionData          = modSecur32.NewProc("LsaGetLogonSessionData")
)

const (
    SESS_INTERACTIVE_LOGON              = 2
)

type LUID struct {
    LowPart     uint32
    HighPart    int32
}

type SECURITY_LOGON_SESSION_DATA struct {
    Size                    uint32
    LogonId                 LUID
    UserName                LSA_UNICODE_STRING
    LogonDomain             LSA_UNICODE_STRING
    AuthenticationPackage   LSA_UNICODE_STRING
    LogonType               uint32
    Session                 uint32
    Sid                     uintptr
    LogonTime               uint64
    LogonServer             LSA_UNICODE_STRING
    DnsDomainName           LSA_UNICODE_STRING
    Upn                     LSA_UNICODE_STRING
}

type LSA_UNICODE_STRING struct {
    Length                  uint16
    MaximumLength           uint16
    buffer                  uintptr
}

func ListLoggedInUsers() ([]so.SessionDetails, error) {
    var (
        logonSessionCount       uint64
        loginSessionList        uintptr
        sizeTest                LUID
        uList                   []string            = make([]string, 0)
        uSessList               []so.SessionDetails    = make([]so.SessionDetails, 0)
        PidLUIDList             map[uint32]LUID
    )
    PidLUIDList, err := ProcessLUIDList()
    if err != nil {
        return nil, fmt.Errorf("Error getting process list, %s.", err.Error())
    }

    _, _, _ = sessLsaEnumerateLogonSessions.Call(
        uintptr(unsafe.Pointer(&logonSessionCount)),
        uintptr(unsafe.Pointer(&loginSessionList)),
    )
    defer sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(&loginSessionList)))

    var iter uintptr = uintptr(unsafe.Pointer(loginSessionList))

    for i := uint64(0); i < logonSessionCount; i++ {
        var sessionData uintptr
        _, _, _ = sessLsaGetLogonSessionData.Call(uintptr(iter), uintptr(unsafe.Pointer(&sessionData)))
        if sessionData != uintptr(0){
            var data *SECURITY_LOGON_SESSION_DATA = (*SECURITY_LOGON_SESSION_DATA)(unsafe.Pointer(sessionData))

            if data.Sid != uintptr(0) {
                if data.LogonType == SESS_INTERACTIVE_LOGON {
                    if LsatoString(data.LogonDomain) != "Window Manager" {
                        sUser := fmt.Sprintf("%s\\%s", strings.ToUpper(LsatoString(data.LogonDomain)), strings.ToLower(LsatoString(data.UserName)))
                        sort.Strings(uList)
                        i := sort.Search(len(uList), func(i int) bool { return uList[i] >= sUser })
                        if !(i < len(uList) && uList[i] == sUser) {
                            if luidinmap(&data.LogonId, &PidLUIDList) {
                                uList = append(uList, sUser)
                                ud := so.SessionDetails{
                                    Username: strings.ToLower(LsatoString(data.UserName)),
                                    Domain: strings.ToUpper(LsatoString(data.LogonDomain)),
                                }
                                hn, _ := os.Hostname()
                                if strings.ToUpper(ud.Domain) == strings.ToUpper(hn) {
                                    ud.LocalUser = true
                                    if isAdmin, _ := IsLocalUserAdmin(ud.Username); isAdmin {
                                        ud.LocalAdmin = true
                                    }
                                }
                                uSessList = append(uSessList, ud)
                            }
                        }
                    }
                }
            }
        }

        iter = uintptr(unsafe.Pointer(iter + unsafe.Sizeof(sizeTest)))
        _, _, _ = sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(sessionData)))
    }

    return uSessList, nil
}

func sessUserLUIDs() (map[LUID]string, error) {
    var (
        logonSessionCount       uint64
        loginSessionList        uintptr
        sizeTest                LUID
        uList                   map[LUID]string     = make(map[LUID]string)
    )

    _, _, _ = sessLsaEnumerateLogonSessions.Call(
        uintptr(unsafe.Pointer(&logonSessionCount)),
        uintptr(unsafe.Pointer(&loginSessionList)),
    )
    defer sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(&loginSessionList)))

    var iter uintptr = uintptr(unsafe.Pointer(loginSessionList))

    for i := uint64(0); i < logonSessionCount; i++ {
        var sessionData uintptr
        _, _, _ = sessLsaGetLogonSessionData.Call(uintptr(iter), uintptr(unsafe.Pointer(&sessionData)))
        if sessionData != uintptr(0){
            var data *SECURITY_LOGON_SESSION_DATA = (*SECURITY_LOGON_SESSION_DATA)(unsafe.Pointer(sessionData))

            if data.Sid != uintptr(0) {
                uList[data.LogonId] = fmt.Sprintf("%s\\%s", strings.ToUpper(LsatoString(data.LogonDomain)), strings.ToLower(LsatoString(data.UserName)))
            }
        }

        iter = uintptr(unsafe.Pointer(iter + unsafe.Sizeof(sizeTest)))
        _, _, _ = sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(sessionData)))
    }

    return uList, nil
}

func luidinmap(needle *LUID, haystack *map[uint32]LUID) (bool) {
    for _, l := range *haystack {
        if reflect.DeepEqual(l, *needle) {
            return true
        }
    }
    return false
}

func LsatoString(p LSA_UNICODE_STRING) string {
    return syscall.UTF16ToString((*[4096]uint16)(unsafe.Pointer(p.buffer))[:p.Length])
}
<commit_msg>Include Cached & Remote Interactive Sessions<commit_after>// +build windows,amd64
package winapi

import (
    "os"
    "fmt"
    "sort"
    "strings"
    "syscall"
    "unsafe"
    "reflect"

    so "github.com/iamacarpet/go-win64api/shared"
)

var (
    modSecur32                          = syscall.NewLazyDLL("secur32.dll")
    sessLsaFreeReturnBuffer             = modSecur32.NewProc("LsaFreeReturnBuffer")
    sessLsaEnumerateLogonSessions       = modSecur32.NewProc("LsaEnumerateLogonSessions")
    sessLsaGetLogonSessionData          = modSecur32.NewProc("LsaGetLogonSessionData")
)

const (
    SESS_INTERACTIVE_LOGON              = 2
    SESS_REMOTE_INTERACTIVE_LOGON       = 10
    SESS_CACHED_INTERACTIVE_LOGON       = 11
)

type LUID struct {
    LowPart     uint32
    HighPart    int32
}

type SECURITY_LOGON_SESSION_DATA struct {
    Size                    uint32
    LogonId                 LUID
    UserName                LSA_UNICODE_STRING
    LogonDomain             LSA_UNICODE_STRING
    AuthenticationPackage   LSA_UNICODE_STRING
    LogonType               uint32
    Session                 uint32
    Sid                     uintptr
    LogonTime               uint64
    LogonServer             LSA_UNICODE_STRING
    DnsDomainName           LSA_UNICODE_STRING
    Upn                     LSA_UNICODE_STRING
}

type LSA_UNICODE_STRING struct {
    Length                  uint16
    MaximumLength           uint16
    buffer                  uintptr
}

func ListLoggedInUsers() ([]so.SessionDetails, error) {
    var (
        logonSessionCount       uint64
        loginSessionList        uintptr
        sizeTest                LUID
        uList                   []string            = make([]string, 0)
        uSessList               []so.SessionDetails    = make([]so.SessionDetails, 0)
        PidLUIDList             map[uint32]LUID
    )
    PidLUIDList, err := ProcessLUIDList()
    if err != nil {
        return nil, fmt.Errorf("Error getting process list, %s.", err.Error())
    }

    _, _, _ = sessLsaEnumerateLogonSessions.Call(
        uintptr(unsafe.Pointer(&logonSessionCount)),
        uintptr(unsafe.Pointer(&loginSessionList)),
    )
    defer sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(&loginSessionList)))

    var iter uintptr = uintptr(unsafe.Pointer(loginSessionList))

    for i := uint64(0); i < logonSessionCount; i++ {
        var sessionData uintptr
        _, _, _ = sessLsaGetLogonSessionData.Call(uintptr(iter), uintptr(unsafe.Pointer(&sessionData)))
        if sessionData != uintptr(0){
            var data *SECURITY_LOGON_SESSION_DATA = (*SECURITY_LOGON_SESSION_DATA)(unsafe.Pointer(sessionData))

            if data.Sid != uintptr(0) {
                //fmt.Printf("%s\\%s Type: %d\r\n", strings.ToUpper(LsatoString(data.LogonDomain)), strings.ToLower(LsatoString(data.UserName)), data.LogonType)
                validTypes := []uint32{SESS_INTERACTIVE_LOGON, SESS_CACHED_INTERACTIVE_LOGON, SESS_REMOTE_INTERACTIVE_LOGON}
                if in_array(data.LogonType, validTypes) {
                    if LsatoString(data.LogonDomain) != "Window Manager" {
                        sUser := fmt.Sprintf("%s\\%s", strings.ToUpper(LsatoString(data.LogonDomain)), strings.ToLower(LsatoString(data.UserName)))
                        sort.Strings(uList)
                        i := sort.Search(len(uList), func(i int) bool { return uList[i] >= sUser })
                        if !(i < len(uList) && uList[i] == sUser) {
                            if luidinmap(&data.LogonId, &PidLUIDList) {
                                uList = append(uList, sUser)
                                ud := so.SessionDetails{
                                    Username: strings.ToLower(LsatoString(data.UserName)),
                                    Domain: strings.ToUpper(LsatoString(data.LogonDomain)),
                                }
                                hn, _ := os.Hostname()
                                if strings.ToUpper(ud.Domain) == strings.ToUpper(hn) {
                                    ud.LocalUser = true
                                    if isAdmin, _ := IsLocalUserAdmin(ud.Username); isAdmin {
                                        ud.LocalAdmin = true
                                    }
                                }
                                uSessList = append(uSessList, ud)
                            }
                        }
                    }
                }
            }
        }

        iter = uintptr(unsafe.Pointer(iter + unsafe.Sizeof(sizeTest)))
        _, _, _ = sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(sessionData)))
    }

    return uSessList, nil
}

func sessUserLUIDs() (map[LUID]string, error) {
    var (
        logonSessionCount       uint64
        loginSessionList        uintptr
        sizeTest                LUID
        uList                   map[LUID]string     = make(map[LUID]string)
    )

    _, _, _ = sessLsaEnumerateLogonSessions.Call(
        uintptr(unsafe.Pointer(&logonSessionCount)),
        uintptr(unsafe.Pointer(&loginSessionList)),
    )
    defer sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(&loginSessionList)))

    var iter uintptr = uintptr(unsafe.Pointer(loginSessionList))

    for i := uint64(0); i < logonSessionCount; i++ {
        var sessionData uintptr
        _, _, _ = sessLsaGetLogonSessionData.Call(uintptr(iter), uintptr(unsafe.Pointer(&sessionData)))
        if sessionData != uintptr(0){
            var data *SECURITY_LOGON_SESSION_DATA = (*SECURITY_LOGON_SESSION_DATA)(unsafe.Pointer(sessionData))

            if data.Sid != uintptr(0) {
                uList[data.LogonId] = fmt.Sprintf("%s\\%s", strings.ToUpper(LsatoString(data.LogonDomain)), strings.ToLower(LsatoString(data.UserName)))
            }
        }

        iter = uintptr(unsafe.Pointer(iter + unsafe.Sizeof(sizeTest)))
        _, _, _ = sessLsaFreeReturnBuffer.Call(uintptr(unsafe.Pointer(sessionData)))
    }

    return uList, nil
}

func luidinmap(needle *LUID, haystack *map[uint32]LUID) (bool) {
    for _, l := range *haystack {
        if reflect.DeepEqual(l, *needle) {
            return true
        }
    }
    return false
}

func LsatoString(p LSA_UNICODE_STRING) string {
    return syscall.UTF16ToString((*[4096]uint16)(unsafe.Pointer(p.buffer))[:p.Length])
}

func in_array(val interface{}, array interface{}) (exists bool) {
    exists = false

    switch reflect.TypeOf(array).Kind() {
    case reflect.Slice:
        s := reflect.ValueOf(array)

        for i := 0; i < s.Len(); i++ {
            if reflect.DeepEqual(val, s.Index(i).Interface()) == true {
                exists = true
                return
            }
        }
    }

    return
}
<|endoftext|>