<commit_before>package util

import (
  "boshlite/configuration"
  termcolor "boshlite/terminalcolor"
  "fmt"
  "log"
  "os/exec"
  "os/user"
  "os"
  "bufio"
  "strconv"
  "strings"
)

func Execute(bash string, sudo bool) (out []byte, err error) {
  if sudo {
    out, err = exec.Command("sudo", "bash", "-c", bash).Output()
  } else {
    out, err = exec.Command("bash", "-c", bash).Output()
  }
  return
}

func RouteCmd(config configuration.Configuration) (routecmd string) {
  switch config.OStype {
  case "Darwin":
    routecmd = "route delete -net " + config.IpRange + " " + config.Gateway + " > /dev/null 2>&1;"
    routecmd += "route add -net " + config.IpRange + " " + config.Gateway
  case "Linux":
    routecmd = "route add -net " + config.IpRange + " gw " + config.Gateway
  default:
    log.Fatal("Not supported OS detected")
  }
  return
}

func Addroute(config configuration.Configuration) {
  routecmd := RouteCmd(config)
  out, err := Execute(routecmd, true)
  if err != nil {
    log.Fatal(err)
  }
  fmt.Printf("%s", out)
}

type CheckVersion struct {
  name           string
  cmd_exist      string
  cmd_version    string
  expect_version string
}

func BuildSanitymap() []CheckVersion {
  check_map := []CheckVersion{
    {
      "Vagrant",
      "which vagrant",
      "vagrant -v|cut -d' ' -f2",
      "1.3.4",
    },
    {
      "Vagrant fusion plugin",
      "vagrant plugin list|grep vagrant-vmware-fusion",
      "vagrant plugin list|grep vagrant-vmware-fusion|cut -d' ' -f2|cut -d'(' -f2|cut -d')' -f1",
      "2.1.0",
    },
    {
      "BOSH CLI",
      "which bosh",
      "bosh -v|cut -d' ' -f2",
      "1.5.0.pre.1525",
    },
    {
      "Spiff",
      "which spiff",
      "spiff --version|cut -d' ' -f3",
      "0.0.0",
    },
    {
      "Go CF Cli",
      "which gcf",
      "gcf --version|cut -d' ' -f3",
      "6.0.0.rc1-SHA",
    },
    {
      "Virtual Box",
      "which VirtualBox",
      "VirtualBox --help|head -1|cut -d' ' -f 5",
      "4.2.18",
    },
    {
      "VMware Fusion",
      "ls /Applications/VMware\\ Fusion.app/",
      "cat /Applications/VMware\\ Fusion.app/Contents/Info.plist|grep -A1 CFBundleShortVersionString|tail -1|cut -d'>' -f2|cut -d'<' -f1",
      "6.0.2",
    },
  }
  return check_map
}

func IsVersionNewer(current string, expect string) bool {
  cur_num := strings.Split(current, ".")
  exp_num := strings.Split(expect, ".")
  var cmp1,cmp2 int
  var err error
  for index, sub_num := range exp_num {
    if index >= len(cur_num) {
      return false
    } else {
      switch cur_num[index] {
      case "alpha":
        cmp1 = -3
      case "beta":
        cmp1 = -2
      case "pre":
        cmp1 = -1
      default:
        cmp1, err = strconv.Atoi(cur_num[index])
        if err != nil {
          cmp1 = -9
        }
      }

      switch sub_num {
      case "alpha":
        cmp2 = -3
      case "beta":
        cmp2 = -2
      case "pre":
        cmp2 = -1
      default:
        cmp2, err = strconv.Atoi(sub_num)
        if err != nil {
          cmp2 = -9
        }
      }

      if cmp1 > cmp2 {
        return true
      } else if cmp1 < cmp2 {
        return false
      } else {
        continue
      }
    }
  }
  return true
}

func SoftCheck() {
  sanity_map := BuildSanitymap()
  for _, check := range sanity_map {
    fmt.Printf("Checking %s...\n", termcolor.Colorize(check.name, termcolor.Cyan, false))
    _, err := Execute(check.cmd_exist, false)
    if err != nil {
      fmt.Printf("%s\n", termcolor.FailureColor("  [ERROR] No "+check.name+" found in your path"))
    } else {
      fmt.Printf("%s\n", termcolor.SuccessColor("  Found "+check.name+" installed"))
      out, err := Execute(check.cmd_version, false)
      if err != nil {
        fmt.Printf("%s\n", termcolor.FailureColor("  [ERROR] Not able to determine your "+check.name+" version"))
      }

      var cur_version string
      if len(string(out)) > 1 {
        cur_version = string(out[:len(string(out))-1])
      } else {
        cur_version = "NIL"
      }
      if cur_version == "NIL" {
        fmt.Printf("%s\n", termcolor.WarnColor("  [Warnning] "+check.name+" version unknown, try install "+check.expect_version+" or newer"))
      } else if IsVersionNewer(cur_version, check.expect_version) {
        fmt.Printf("%s\n", termcolor.SuccessColor("  Detect "+check.name+" version ("+cur_version+") fulfill expected version ("+check.expect_version+")"))
      } else {
        fmt.Printf("%s\n", termcolor.WarnColor("  [Warnning] Detect "+check.name+" version ("+cur_version+") lower than expected ("+check.expect_version+")"))
      }
    }
  }
}

func writeLine(line string, file *os.File) error {
  w := bufio.NewWriter(file)
  fmt.Fprintln(w, line)
  return w.Flush()
}

func GenStub(uuid string) error {
  stub := `---
  name: cf-warden
  director_uuid: ` +  uuid + `
  releases:
  - name: cf
    version: latest
  properties:
  loggregator_endpoint:
  shared_secret: PLACEHOLDER-LOGGREGATOR-SECRET
  `
  afile, err := os.Create("/tmp/bosh-lite-manifest-stub")
  if err != nil {
    return err
  }
  if err := writeLine(stub, afile); err !=nil {
    return err
  }
  return err
}

func SetupManifest() error {
  cf_release_dir := os.Getenv("CF_RELEASE_DIR")

  usr, err := user.Current()
  if err != nil {
    log.Fatal( err )
  }

  if cf_release_dir == "" {
    cf_release_dir = usr.HomeDir + "/workspace/cf-release"
  }
  _, err = os.Stat(cf_release_dir)
  if err !=nil {
    fmt.Printf("%s\n", termcolor.FailureColor("[ERROR] cf-release dir: " + cf_release_dir + " not found, set CF_RELEASE_DIR env first"))
    return err
  }

  _, err = Execute("which bosh", false)
  if err != nil {
    fmt.Printf("%s\n", termcolor.FailureColor("[ERROR] No bosh found, install bosh cli first"))
    return err
  }
  _, err = Execute("bosh status|grep 'Bosh Lite Director'", false)
  if err != nil {
    fmt.Printf("%s\n", termcolor.WarnColor("[Warnning] Can only target Bosh Lite Director, Please use 'bosh target' before running this script."))
    return err
  }
  uuid, err := Execute("bosh status | grep UUID | awk '{print $2}'", false)
  if err != nil {
    log.Fatal(err)
  }
  fmt.Printf("%s", termcolor.SuccessColor("Generating Stub file with uuid:" + string(uuid)))
  err = GenStub(string(uuid))
  if err != nil {
    log.Fatal(err)
  }

  cmd := cf_release_dir+"/generate_deployment_manifest warden /tmp/bosh-lite-manifest-stub " + cf_release_dir +"/templates/cf-minimal-dev.yml"
  fmt.Printf("%s\n", termcolor.SuccessColor("Generating deployment file ./bosh-lite-cf-manifest.yml"))
  _, err = Execute(cmd + " > ./bosh-lite-cf-manifest.yml", false)
  if err != nil {
    log.Fatal(err)
  }
  fmt.Printf("%s\n", termcolor.SuccessColor("Seting bosh deployment ./bosh-lite-cf-manifest.yml"))
  _, err = Execute("bosh deployment ./bosh-lite-cf-manifest.yml", false)
  if err != nil {
    log.Fatal(err)
  }
  return err
}
<commit_msg>verify whether we are targeting director with Warden CPI before generating manifest<commit_after>package util

import (
  "boshlite/configuration"
  termcolor "boshlite/terminalcolor"
  "fmt"
  "log"
  "os/exec"
  "os/user"
  "os"
  "bufio"
  "strconv"
  "strings"
)

func Execute(bash string, sudo bool) (out []byte, err error) {
  if sudo {
    out, err = exec.Command("sudo", "bash", "-c", bash).Output()
  } else {
    out, err = exec.Command("bash", "-c", bash).Output()
  }
  return
}

func RouteCmd(config configuration.Configuration) (routecmd string) {
  switch config.OStype {
  case "Darwin":
    routecmd = "route delete -net " + config.IpRange + " " + config.Gateway + " > /dev/null 2>&1;"
    routecmd += "route add -net " + config.IpRange + " " + config.Gateway
  case "Linux":
    routecmd = "route add -net " + config.IpRange + " gw " + config.Gateway
  default:
    log.Fatal("Not supported OS detected")
  }
  return
}

func Addroute(config configuration.Configuration) {
  routecmd := RouteCmd(config)
  out, err := Execute(routecmd, true)
  if err != nil {
    log.Fatal(err)
  }
  fmt.Printf("%s", out)
}

type CheckVersion struct {
  name           string
  cmd_exist      string
  cmd_version    string
  expect_version string
}

func BuildSanitymap() []CheckVersion {
  check_map := []CheckVersion{
    {
      "Vagrant",
      "which vagrant",
      "vagrant -v|cut -d' ' -f2",
      "1.3.4",
    },
    {
      "Vagrant fusion plugin",
      "vagrant plugin list|grep vagrant-vmware-fusion",
      "vagrant plugin list|grep vagrant-vmware-fusion|cut -d' ' -f2|cut -d'(' -f2|cut -d')' -f1",
      "2.1.0",
    },
    {
      "BOSH CLI",
      "which bosh",
      "bosh -v|cut -d' ' -f2",
      "1.5.0.pre.1525",
    },
    {
      "Spiff",
      "which spiff",
      "spiff --version|cut -d' ' -f3",
      "0.0.0",
    },
    {
      "Go CF Cli",
      "which gcf",
      "gcf --version|cut -d' ' -f3",
      "6.0.0.rc1-SHA",
    },
    {
      "Virtual Box",
      "which VirtualBox",
      "VirtualBox --help|head -1|cut -d' ' -f 5",
      "4.2.18",
    },
    {
      "VMware Fusion",
      "ls /Applications/VMware\\ Fusion.app/",
      "cat /Applications/VMware\\ Fusion.app/Contents/Info.plist|grep -A1 CFBundleShortVersionString|tail -1|cut -d'>' -f2|cut -d'<' -f1",
      "6.0.2",
    },
  }
  return check_map
}

func IsVersionNewer(current string, expect string) bool {
  cur_num := strings.Split(current, ".")
  exp_num := strings.Split(expect, ".")
  var cmp1,cmp2 int
  var err error
  for index, sub_num := range exp_num {
    if index >= len(cur_num) {
      return false
    } else {
      switch cur_num[index] {
      case "alpha":
        cmp1 = -3
      case "beta":
        cmp1 = -2
      case "pre":
        cmp1 = -1
      default:
        cmp1, err = strconv.Atoi(cur_num[index])
        if err != nil {
          cmp1 = -9
        }
      }

      switch sub_num {
      case "alpha":
        cmp2 = -3
      case "beta":
        cmp2 = -2
      case "pre":
        cmp2 = -1
      default:
        cmp2, err = strconv.Atoi(sub_num)
        if err != nil {
          cmp2 = -9
        }
      }

      if cmp1 > cmp2 {
        return true
      } else if cmp1 < cmp2 {
        return false
      } else {
        continue
      }
    }
  }
  return true
}

func SoftCheck() {
  sanity_map := BuildSanitymap()
  for _, check := range sanity_map {
    fmt.Printf("Checking %s...\n", termcolor.Colorize(check.name, termcolor.Cyan, false))
    _, err := Execute(check.cmd_exist, false)
    if err != nil {
      fmt.Printf("%s\n", termcolor.FailureColor("  [ERROR] No "+check.name+" found in your path"))
    } else {
      fmt.Printf("%s\n", termcolor.SuccessColor("  Found "+check.name+" installed"))
      out, err := Execute(check.cmd_version, false)
      if err != nil {
        fmt.Printf("%s\n", termcolor.FailureColor("  [ERROR] Not able to determine your "+check.name+" version"))
      }

      var cur_version string
      if len(string(out)) > 1 {
        cur_version = string(out[:len(string(out))-1])
      } else {
        cur_version = "NIL"
      }
      if cur_version == "NIL" {
        fmt.Printf("%s\n", termcolor.WarnColor("  [Warnning] "+check.name+" version unknown, try install "+check.expect_version+" or newer"))
      } else if IsVersionNewer(cur_version, check.expect_version) {
        fmt.Printf("%s\n", termcolor.SuccessColor("  Detect "+check.name+" version ("+cur_version+") fulfill expected version ("+check.expect_version+")"))
      } else {
        fmt.Printf("%s\n", termcolor.WarnColor("  [Warnning] Detect "+check.name+" version ("+cur_version+") lower than expected ("+check.expect_version+")"))
      }
    }
  }
}

func writeLine(line string, file *os.File) error {
  w := bufio.NewWriter(file)
  fmt.Fprintln(w, line)
  return w.Flush()
}

func GenStub(uuid string) error {
  stub := `---
  name: cf-warden
  director_uuid: ` +  uuid + `
  releases:
  - name: cf
    version: latest
  properties:
  loggregator_endpoint:
  shared_secret: PLACEHOLDER-LOGGREGATOR-SECRET
  `
  afile, err := os.Create("/tmp/bosh-lite-manifest-stub")
  if err != nil {
    return err
  }
  if err := writeLine(stub, afile); err !=nil {
    return err
  }
  return err
}

func SetupManifest() error {
  cf_release_dir := os.Getenv("CF_RELEASE_DIR")

  usr, err := user.Current()
  if err != nil {
    log.Fatal( err )
  }

  if cf_release_dir == "" {
    cf_release_dir = usr.HomeDir + "/workspace/cf-release"
  }
  _, err = os.Stat(cf_release_dir)
  if err !=nil {
    fmt.Printf("%s\n", termcolor.FailureColor("[ERROR] cf-release dir: " + cf_release_dir + " not found, set CF_RELEASE_DIR env first"))
    return err
  }

  _, err = Execute("which bosh", false)
  if err != nil {
    fmt.Printf("%s\n", termcolor.FailureColor("[ERROR] No bosh found, install bosh cli first"))
    return err
  }
  _, err = Execute("bosh status|grep 'Bosh Lite Director'", false)
  if err != nil {
    fmt.Printf("%s\n", termcolor.WarnColor("[Warnning] Can only target Bosh Lite Director, Please use 'bosh target' before running this script."))
    return err
  }
  uuid, err := Execute("bosh status | grep UUID | awk '{print $2}'", false)
  if err != nil {
    log.Fatal(err)
  }
  fmt.Printf("%s", termcolor.SuccessColor("Generating Stub file with uuid:" + string(uuid)))
  cpi, err := Execute("bosh status | grep CPI | awk '{print $2}'", false)
  if err != nil {
    log.Fatal(err)
  }
  if (string(cpi) != "warden\n") {
    fmt.Printf("%s\n", termcolor.WarnColor("[Warnning] You are targeting NONE Bosh Lite Director, aborting..."))
    return nil
  }
  err = GenStub(string(uuid))
  if err != nil {
    log.Fatal(err)
  }

  cmd := cf_release_dir+"/generate_deployment_manifest warden /tmp/bosh-lite-manifest-stub " + cf_release_dir +"/templates/cf-minimal-dev.yml"
  fmt.Printf("%s\n", termcolor.SuccessColor("Generating deployment file ./bosh-lite-cf-manifest.yml"))
  _, err = Execute(cmd + " > ./bosh-lite-cf-manifest.yml", false)
  if err != nil {
    log.Fatal(err)
  }
  fmt.Printf("%s\n", termcolor.SuccessColor("Seting bosh deployment ./bosh-lite-cf-manifest.yml"))
  _, err = Execute("bosh deployment ./bosh-lite-cf-manifest.yml", false)
  if err != nil {
    log.Fatal(err)
  }
  return err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/iiif"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/openjpeg"
	"image"
	"image/jpeg"
	"log"
	"net/http"
	"os"
	"regexp"
)

var iiifInfoPathRegex = regexp.MustCompile(`^/images/iiif/(.+)/info.json$`)

func IIIFHandler(w http.ResponseWriter, req *http.Request) {
	p := req.URL.Path

	// Check for info path first, and dispatch if it matches
	if parts := iiifInfoPathRegex.FindStringSubmatch(p); parts != nil {
		iiifInfoHandler(w, req, parts[1])
		return
	}

	// No info path should mean a full command path
	if u := iiif.NewURL(p); u.Valid() {
		iiifCommandHandler(w, req, u)
		return
	}

	// No info or command?  400 error according to IIIF spec.
	http.Error(w, "Invalid IIIF request", 400)
}

func iiifInfoHandler(w http.ResponseWriter, req *http.Request, id string) {
	fmt.Println(req.URL)

	filepath := tilePath + "/" + id
	if _, err := os.Stat(filepath); err != nil {
		http.Error(w, "Image resource does not exist", 404)
		return
	}

	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image %#v", id), 500)
		return
	}

	if err := jp2.ReadHeader(); err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image dimensions for %#v", id), 500)
		return
	}

	info := NewIIIFInfo()
	rect := jp2.Dimensions()
	info.Width = rect.Dx()
	info.Height = rect.Dy()

	// The info id is actually the full path to the resource, not just its ID
	info.ID = iiifBase.String() + "/" + id

	json, err := json.Marshal(info)
	if err != nil {
		log.Printf("ERROR!  Unable to marshal IIIFInfo response: %s", err)
		http.Error(w, "Server error", 500)
		return
	}

	// Set headers - TODO: check for Accept header with jsonld content type!
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(json)
}

// Handles crop/resize operations for JP2s.  Note that this is the *wrong* way
// to handle most image formats.  JP2s are encoded as multi-resolution images,
// so the resize information actually has to be known before a given region can
// be cropped.  Otherwise we'd have to decode the whole image instead of just
// the minimum necessary "layer".
func iiifCommandHandler(w http.ResponseWriter, req *http.Request, u *iiif.URL) {
	// Get file's last modified time, returning a 404 if we can't stat the file
	filepath := tilePath + "/" + u.ID
	if err := sendHeaders(w, req, filepath); err != nil {
		return
	}

	// Do we support this request?  If not, return a 501
	if !iiif.FeaturesLevel1.Supported(u) {
		http.Error(w, "Feature not supported", 501)
		return
	}

	// Create JP2 structure - if we can't, the image must be corrupt or otherwise
	// broken, since we already checked for existence
	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, "Unable to read source image", 500)
		log.Println("Unable to read source image: ", err)
		return
	}
	defer jp2.CleanupResources()

	if u.Region.Type == iiif.RTPixel {
		r := image.Rect(
			int(u.Region.X),
			int(u.Region.Y),
			int(u.Region.X+u.Region.W),
			int(u.Region.Y+u.Region.H),
		)
		jp2.SetCrop(r)
	}

	// TODO: convert size to w/h
	//jp2.SetResize(width, height)
	img, err := jp2.DecodeImage()
	if err != nil {
		http.Error(w, "Unable to decode image", 500)
		log.Println("Unable to decode image: ", err)
		return
	}

	// Encode as JPEG straight to the client
	if err = jpeg.Encode(w, img, &jpeg.Options{Quality: 80}); err != nil {
		http.Error(w, "Unable to encode jpeg", 500)
		log.Println("Unable to encode JPEG:", err)
		return
	}
}
<commit_msg>Remove debug output<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/iiif"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/openjpeg"
	"image"
	"image/jpeg"
	"log"
	"net/http"
	"os"
	"regexp"
)

var iiifInfoPathRegex = regexp.MustCompile(`^/images/iiif/(.+)/info.json$`)

func IIIFHandler(w http.ResponseWriter, req *http.Request) {
	p := req.URL.Path

	// Check for info path first, and dispatch if it matches
	if parts := iiifInfoPathRegex.FindStringSubmatch(p); parts != nil {
		iiifInfoHandler(w, req, parts[1])
		return
	}

	// No info path should mean a full command path
	if u := iiif.NewURL(p); u.Valid() {
		iiifCommandHandler(w, req, u)
		return
	}

	// No info or command?  400 error according to IIIF spec.
	http.Error(w, "Invalid IIIF request", 400)
}

func iiifInfoHandler(w http.ResponseWriter, req *http.Request, id string) {
	filepath := tilePath + "/" + id
	if _, err := os.Stat(filepath); err != nil {
		http.Error(w, "Image resource does not exist", 404)
		return
	}

	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image %#v", id), 500)
		return
	}

	if err := jp2.ReadHeader(); err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image dimensions for %#v", id), 500)
		return
	}

	info := NewIIIFInfo()
	rect := jp2.Dimensions()
	info.Width = rect.Dx()
	info.Height = rect.Dy()

	// The info id is actually the full path to the resource, not just its ID
	info.ID = iiifBase.String() + "/" + id

	json, err := json.Marshal(info)
	if err != nil {
		log.Printf("ERROR!  Unable to marshal IIIFInfo response: %s", err)
		http.Error(w, "Server error", 500)
		return
	}

	// Set headers - TODO: check for Accept header with jsonld content type!
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(json)
}

// Handles crop/resize operations for JP2s.  Note that this is the *wrong* way
// to handle most image formats.  JP2s are encoded as multi-resolution images,
// so the resize information actually has to be known before a given region can
// be cropped.  Otherwise we'd have to decode the whole image instead of just
// the minimum necessary "layer".
func iiifCommandHandler(w http.ResponseWriter, req *http.Request, u *iiif.URL) {
	// Get file's last modified time, returning a 404 if we can't stat the file
	filepath := tilePath + "/" + u.ID
	if err := sendHeaders(w, req, filepath); err != nil {
		return
	}

	// Do we support this request?  If not, return a 501
	if !iiif.FeaturesLevel1.Supported(u) {
		http.Error(w, "Feature not supported", 501)
		return
	}

	// Create JP2 structure - if we can't, the image must be corrupt or otherwise
	// broken, since we already checked for existence
	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, "Unable to read source image", 500)
		log.Println("Unable to read source image: ", err)
		return
	}
	defer jp2.CleanupResources()

	if u.Region.Type == iiif.RTPixel {
		r := image.Rect(
			int(u.Region.X),
			int(u.Region.Y),
			int(u.Region.X+u.Region.W),
			int(u.Region.Y+u.Region.H),
		)
		jp2.SetCrop(r)
	}

	// TODO: convert size to w/h
	//jp2.SetResize(width, height)
	img, err := jp2.DecodeImage()
	if err != nil {
		http.Error(w, "Unable to decode image", 500)
		log.Println("Unable to decode image: ", err)
		return
	}

	// Encode as JPEG straight to the client
	if err = jpeg.Encode(w, img, &jpeg.Options{Quality: 80}); err != nil {
		http.Error(w, "Unable to encode jpeg", 500)
		log.Println("Unable to encode JPEG:", err)
		return
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"net"
	"net/http"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

type webServer struct {
	name       string
	maxClients int
	gw         *Gateway

	httpListener net.Listener
	httpServer   *http.Server

	httpsListener net.Listener
	httpsServer   *http.Server

	router *httprouter.Router

	waitExitFunc    waitExitFunc
	connStateFunc   connStateFunc
	onConnNewFunc   onConnNewFunc
	onConnCloseFunc onConnCloseFunc

	onStop   func()
	onceStop sync.Once

	activeConnN int32
}

func newWebServer(name string, httpAddr, httpsAddr string, maxClients int,
	gw *Gateway) *webServer {
	this := &webServer{
		name:       name,
		gw:         gw,
		maxClients: maxClients,
		router:     httprouter.New(),
	}

	if Options.EnableHttpPanicRecover {
		this.router.PanicHandler = func(w http.ResponseWriter,
			r *http.Request, err interface{}) {
			log.Error("%s[%s] %s %s: %+v", this.name, r.RemoteAddr, r.Method, r.RequestURI, err)
		}
	}

	if httpAddr != "" {
		this.httpServer = &http.Server{
			Addr:           httpAddr,
			Handler:        this.router,
			ReadTimeout:    Options.HttpReadTimeout,
			WriteTimeout:   Options.HttpWriteTimeout,
			MaxHeaderBytes: Options.HttpHeaderMaxBytes,
		}
	}

	if httpsAddr != "" {
		this.httpsServer = &http.Server{
			Addr:           httpsAddr,
			Handler:        this.router,
			ReadTimeout:    Options.HttpReadTimeout,
			WriteTimeout:   Options.HttpWriteTimeout,
			MaxHeaderBytes: Options.HttpHeaderMaxBytes,
		}
	}

	return this
}

func (this *webServer) Start() {
	if this.waitExitFunc == nil {
		this.waitExitFunc = this.waitExit
	}
	if this.connStateFunc == nil {
		this.connStateFunc = this.connStateHandler
	}

	if this.httpsServer != nil {
		this.httpsServer.ConnState = this.connStateFunc
		this.startServer(true)
	}

	if this.httpServer != nil {
		this.httpServer.ConnState = this.connStateFunc
		this.startServer(false)
	}

}

func (this *webServer) Router() *httprouter.Router {
	return this.router
}

func (this *webServer) startServer(https bool) {
	var err error
	waitListenerUp := make(chan struct{})
	go func() {
		if Options.CpuAffinity {
			runtime.LockOSThread()
		}

		var (
			retryDelay         time.Duration
			theListener        net.Listener
			waitListenerUpOnce sync.Once
			addr               string
		)
		for {
			if https {
				this.httpsListener, err = net.Listen("tcp", this.httpsServer.Addr)
				if err != nil {
					if retryDelay == 0 {
						retryDelay = 50 * time.Millisecond
					} else {
						retryDelay = 2 * retryDelay
					}
					if maxDelay := time.Second; retryDelay > maxDelay {
						retryDelay = maxDelay
					}
					log.Error("%s listener %v, retry in %v", this.name, err, retryDelay)
					time.Sleep(retryDelay)
					continue
				}

				this.httpsListener, err = setupHttpsListener(this.httpsListener,
					this.gw.certFile, this.gw.keyFile)
				if err != nil {
					panic(err)
				}

				theListener = this.httpsListener
				addr = this.httpsServer.Addr
			} else {
				this.httpListener, err = net.Listen("tcp", this.httpServer.Addr)
				theListener = this.httpListener
				addr = this.httpServer.Addr
			}

			if err != nil {
				if retryDelay == 0 {
					retryDelay = 50 * time.Millisecond
				} else {
					retryDelay = 2 * retryDelay
				}
				if maxDelay := time.Second; retryDelay > maxDelay {
					retryDelay = maxDelay
				}
				log.Error("%s listener %v, retry in %v", this.name, err, retryDelay)
				time.Sleep(retryDelay)
				continue
			}

			theListener = LimitListener(this.name, this.gw, theListener, this.maxClients)
			waitListenerUpOnce.Do(func() {
				close(waitListenerUp)
			})

			// on non-temporary err, net/http will close the listener
			if https {
				err = this.httpsServer.Serve(theListener)
			} else {
				err = this.httpServer.Serve(theListener)
			}

			select {
			case <-this.gw.shutdownCh:
				log.Trace("%s server stopped on %s", this.name, addr)
				return

			default:
				log.Error("%s server: %v", this.name, err)
			}
		}
	}()

	// FIXME if net.Listen fails, kateway will not be able to stop
	// e,g. start a kateway, then start another, the 2nd will not be able to stop

	// wait for the listener up
	<-waitListenerUp

	this.gw.wg.Add(1)
	if https {
		go this.waitExitFunc(this.httpsServer, this.httpsListener, this.gw.shutdownCh)
	} else {
		go this.waitExitFunc(this.httpServer, this.httpListener, this.gw.shutdownCh)
	}

	if https {
		log.Info("%s https server ready on %s", this.name, this.httpsServer.Addr)
	} else {
		log.Info("%s http server ready on %s", this.name, this.httpServer.Addr)
	}
}

func (this *webServer) connStateHandler(c net.Conn, cs http.ConnState) {
	switch cs {
	case http.StateNew:
		atomic.AddInt32(&this.activeConnN, 1)

		if this.onConnNewFunc != nil {
			this.onConnNewFunc(c)
		}

	case http.StateActive, http.StateIdle:
		// do nothing by default

	case http.StateClosed, http.StateHijacked:
		atomic.AddInt32(&this.activeConnN, -1)

		if this.onConnCloseFunc != nil {
			this.onConnCloseFunc(c)
		}

	}
}

func (this *webServer) waitExit(server *http.Server, listener net.Listener, exit <-chan struct{}) {
	<-exit

	// HTTP response will have "Connection: close"
	server.SetKeepAlivesEnabled(false)

	// avoid new connections
	if err := listener.Close(); err != nil {
		log.Error(err.Error())
	}

	log.Trace("%s on %s listener closed", this.name, server.Addr)

	waitStart := time.Now()
	for {
		activeConnN := atomic.LoadInt32(&this.activeConnN)
		if activeConnN == 0 {
			// good luck, all connections finished
			break
		}

		// timeout mechanism
		if time.Since(waitStart) > Options.SubTimeout+time.Second {
			log.Warn("%s on %s still left %d conns, but forced to shutdown after %s",
				this.name, server.Addr, activeConnN, Options.SubTimeout)
			break
		}

		time.Sleep(time.Millisecond * 50)
	}
	log.Trace("%s on %s all connections finished", this.name, server.Addr)

	if this.onStop != nil {
		// if both http and https, without sync once, onStop will be called twice
		this.onceStop.Do(func() {
			this.onStop()
		})
	}

	this.gw.wg.Done()
}

func (this *webServer) notFoundHandler(w http.ResponseWriter, r *http.Request) {
	log.Error("%s: not found %s", this.name, r.RequestURI)

	writeNotFound(w)
}
<commit_msg>discard the duplicated log messages<commit_after>package gateway

import (
	"net"
	"net/http"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

type webServer struct {
	name       string
	maxClients int
	gw         *Gateway

	httpListener net.Listener
	httpServer   *http.Server

	httpsListener net.Listener
	httpsServer   *http.Server

	router *httprouter.Router

	waitExitFunc    waitExitFunc
	connStateFunc   connStateFunc
	onConnNewFunc   onConnNewFunc
	onConnCloseFunc onConnCloseFunc

	onStop   func()
	onceStop sync.Once

	activeConnN int32
}

func newWebServer(name string, httpAddr, httpsAddr string, maxClients int,
	gw *Gateway) *webServer {
	this := &webServer{
		name:       name,
		gw:         gw,
		maxClients: maxClients,
		router:     httprouter.New(),
	}

	if Options.EnableHttpPanicRecover {
		this.router.PanicHandler = func(w http.ResponseWriter,
			r *http.Request, err interface{}) {
			log.Error("%s[%s] %s %s: %+v", this.name, r.RemoteAddr, r.Method, r.RequestURI, err)
		}
	}

	if httpAddr != "" {
		this.httpServer = &http.Server{
			Addr:           httpAddr,
			Handler:        this.router,
			ReadTimeout:    Options.HttpReadTimeout,
			WriteTimeout:   Options.HttpWriteTimeout,
			MaxHeaderBytes: Options.HttpHeaderMaxBytes,
		}
	}

	if httpsAddr != "" {
		this.httpsServer = &http.Server{
			Addr:           httpsAddr,
			Handler:        this.router,
			ReadTimeout:    Options.HttpReadTimeout,
			WriteTimeout:   Options.HttpWriteTimeout,
			MaxHeaderBytes: Options.HttpHeaderMaxBytes,
		}
	}

	return this
}

func (this *webServer) Start() {
	if this.waitExitFunc == nil {
		this.waitExitFunc = this.waitExit
	}
	if this.connStateFunc == nil {
		this.connStateFunc = this.connStateHandler
	}

	if this.httpsServer != nil {
		this.httpsServer.ConnState = this.connStateFunc
		this.startServer(true)
	}

	if this.httpServer != nil {
		this.httpServer.ConnState = this.connStateFunc
		this.startServer(false)
	}

}

func (this *webServer) Router() *httprouter.Router {
	return this.router
}

func (this *webServer) startServer(https bool) {
	var err error
	waitListenerUp := make(chan struct{})
	go func() {
		if Options.CpuAffinity {
			runtime.LockOSThread()
		}

		var (
			retryDelay         time.Duration
			theListener        net.Listener
			waitListenerUpOnce sync.Once
		)
		for {
			if https {
				this.httpsListener, err = net.Listen("tcp", this.httpsServer.Addr)
				if err != nil {
					if retryDelay == 0 {
						retryDelay = 50 * time.Millisecond
					} else {
						retryDelay = 2 * retryDelay
					}
					if maxDelay := time.Second; retryDelay > maxDelay {
						retryDelay = maxDelay
					}
					log.Error("%s listener %v, retry in %v", this.name, err, retryDelay)
					time.Sleep(retryDelay)
					continue
				}

				this.httpsListener, err = setupHttpsListener(this.httpsListener,
					this.gw.certFile, this.gw.keyFile)
				if err != nil {
					panic(err)
				}

				theListener = this.httpsListener
			} else {
				this.httpListener, err = net.Listen("tcp", this.httpServer.Addr)
				theListener = this.httpListener
			}

			if err != nil {
				if retryDelay == 0 {
					retryDelay = 50 * time.Millisecond
				} else {
					retryDelay = 2 * retryDelay
				}
				if maxDelay := time.Second; retryDelay > maxDelay {
					retryDelay = maxDelay
				}
				log.Error("%s listener %v, retry in %v", this.name, err, retryDelay)
				time.Sleep(retryDelay)
				continue
			}

			theListener = LimitListener(this.name, this.gw, theListener, this.maxClients)
			waitListenerUpOnce.Do(func() {
				close(waitListenerUp)
			})

			// on non-temporary err, net/http will close the listener
			if https {
				err = this.httpsServer.Serve(theListener)
			} else {
				err = this.httpServer.Serve(theListener)
			}

			select {
			case <-this.gw.shutdownCh:
				return

			default:
				log.Error("%s server: %v", this.name, err)
			}
		}
	}()

	// FIXME if net.Listen fails, kateway will not be able to stop
	// e,g. start a kateway, then start another, the 2nd will not be able to stop

	// wait for the listener up
	<-waitListenerUp

	this.gw.wg.Add(1)
	if https {
		go this.waitExitFunc(this.httpsServer, this.httpsListener, this.gw.shutdownCh)
	} else {
		go this.waitExitFunc(this.httpServer, this.httpListener, this.gw.shutdownCh)
	}

	if https {
		log.Info("%s https server ready on %s", this.name, this.httpsServer.Addr)
	} else {
		log.Info("%s http server ready on %s", this.name, this.httpServer.Addr)
	}
}

func (this *webServer) connStateHandler(c net.Conn, cs http.ConnState) {
	switch cs {
	case http.StateNew:
		atomic.AddInt32(&this.activeConnN, 1)

		if this.onConnNewFunc != nil {
			this.onConnNewFunc(c)
		}

	case http.StateActive, http.StateIdle:
		// do nothing by default

	case http.StateClosed, http.StateHijacked:
		atomic.AddInt32(&this.activeConnN, -1)

		if this.onConnCloseFunc != nil {
			this.onConnCloseFunc(c)
		}

	}
}

func (this *webServer) waitExit(server *http.Server, listener net.Listener, exit <-chan struct{}) {
	<-exit

	// HTTP response will have "Connection: close"
	server.SetKeepAlivesEnabled(false)

	// avoid new connections
	if err := listener.Close(); err != nil {
		log.Error(err.Error())
	}

	log.Trace("%s on %s listener closed", this.name, server.Addr)

	waitStart := time.Now()
	for {
		activeConnN := atomic.LoadInt32(&this.activeConnN)
		if activeConnN == 0 {
			// good luck, all connections finished
			break
		}

		// timeout mechanism
		if time.Since(waitStart) > Options.SubTimeout+time.Second {
			log.Warn("%s on %s still left %d conns, but forced to shutdown after %s",
				this.name, server.Addr, activeConnN, Options.SubTimeout)
			break
		}

		time.Sleep(time.Millisecond * 50)
	}
	log.Trace("%s on %s all connections finished", this.name, server.Addr)

	if this.onStop != nil {
		// if both http and https, without sync once, onStop will be called twice
		this.onceStop.Do(func() {
			this.onStop()
		})
	}

	this.gw.wg.Done()
}

func (this *webServer) notFoundHandler(w http.ResponseWriter, r *http.Request) {
	log.Error("%s: not found %s", this.name, r.RequestURI)

	writeNotFound(w)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"time"

	"bitbucket.org/brunetto/sltools/standAlone/simman"

	"github.com/brunetto/goutils/debug"
	"github.com/brunetto/goutils/notify"
)

func main () () {
	defer debug.TimeMe(time.Now())

	var (
		waitingTime time.Duration = time.Duration(10) * time.Minute
		wakeUp = make(chan map[string]string, 1)
		messageChan = make(chan string, 1)
		jobInfoChan = make(chan simman.JobMap, 1)
		conn = map[string]string{
			"usr": "bziosi00",
			"server": "login.eurora.cineca.it:22",
			"pathToKey": "",
			}

		userChan = make(chan bool, 1)
		running int = 0
	)

	go simman.MainLoop(wakeUp, messageChan, jobInfoChan)
	go printMessages(messageChan)
// 	go timer
	go usrInput(userChan)

	for {
		wakeUp <- conn

		queueLines := <- jobInfoChan

		fmt.Println()

		if len(queueLines) == 0 {
			log.Println("All job finished")
// 			break
			// Start again

		}

		running = 0
		for _, value := range queueLines {
			if	value["status"] == "R" {
				running++
			}
		}

		fmt.Println("------------------------")
		log.Printf("%v jobs, %v running\n", len(queueLines), running)
		fmt.Println("------------------------\n")

		log.Println("Next check in ", waitingTime)
		fmt.Println("If you want details about the last check, write 'details'.")

		select {
			case <-userChan:
				queueLines.Print()
			case <-time.Tick(waitingTime):
				continue
		}

	}

	close(wakeUp)
	close(messageChan)
	close(jobInfoChan)

	notify.Notify("EURORA", "All job are done!!!")
	fmt.Print("\x07") // Beep when finish!!:D
}

func printMessages (messageChan chan string) () {
	for message := range messageChan {
		log.Printf(message)
	}
}

func usrInput (userChan chan bool) () {
	var userInput string
	for {
		_, _ = fmt.Scan(&userInput)
		if userInput == "details" {
			userChan <- true
		} else {
			fmt.Println("Unknown command: ", userInput)
		}
	}

}
<commit_msg>Update simmanCli.go<commit_after>package main

import (
	"fmt"
	"log"
	"time"

	"github.com/brunetto/sltools/cmd/simman"

	"github.com/brunetto/goutils/debug"
	"github.com/brunetto/goutils/notify"
)

func main () () {
	defer debug.TimeMe(time.Now())

	var (
		waitingTime time.Duration = time.Duration(10) * time.Minute
		wakeUp = make(chan map[string]string, 1)
		messageChan = make(chan string, 1)
		jobInfoChan = make(chan simman.JobMap, 1)
		conn = map[string]string{
			"usr": "bziosi00",
			"server": "login.eurora.cineca.it:22",
			"pathToKey": "",
			}

		userChan = make(chan bool, 1)
		running int = 0
	)

	go simman.MainLoop(wakeUp, messageChan, jobInfoChan)
	go printMessages(messageChan)
// 	go timer
	go usrInput(userChan)

	for {
		wakeUp <- conn

		queueLines := <- jobInfoChan

		fmt.Println()

		if len(queueLines) == 0 {
			log.Println("All job finished")
// 			break
			// Start again

		}

		running = 0
		for _, value := range queueLines {
			if	value["status"] == "R" {
				running++
			}
		}

		fmt.Println("------------------------")
		log.Printf("%v jobs, %v running\n", len(queueLines), running)
		fmt.Println("------------------------\n")

		log.Println("Next check in ", waitingTime)
		fmt.Println("If you want details about the last check, write 'details'.")

		select {
			case <-userChan:
				queueLines.Print()
			case <-time.Tick(waitingTime):
				continue
		}

	}

	close(wakeUp)
	close(messageChan)
	close(jobInfoChan)

	notify.Notify("EURORA", "All job are done!!!")
	fmt.Print("\x07") // Beep when finish!!:D
}

func printMessages (messageChan chan string) () {
	for message := range messageChan {
		log.Printf(message)
	}
}

func usrInput (userChan chan bool) () {
	var userInput string
	for {
		_, _ = fmt.Scan(&userInput)
		if userInput == "details" {
			userChan <- true
		} else {
			fmt.Println("Unknown command: ", userInput)
		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n")
	fmt.Fprintf(os.Stderr, "\tgoinstall -a\n")
	flag.PrintDefaults()
	os.Exit(2)
}

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	logfile       = filepath.Join(runtime.GOROOT(), "goinstall.log")
	installedPkgs = make(map[string]bool)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	logPkgs           = flag.Bool("log", true, "log installed packages to $GOROOT/goinstall.log for use by -a")
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg || *logPkgs {
		readPackageList()
	}
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		if len(installedPkgs) == 0 {
			fmt.Fprintf(os.Stderr, "%s: no installed packages\n", argv0)
			os.Exit(1)
		}
		args = make([]string, len(installedPkgs), len(installedPkgs))
		i := 0
		for pkg := range installedPkgs {
			args[i] = pkg
			i++
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if strings.HasPrefix(path, "http://") {
			errorf("'http://' used in remote path, try '%s'\n", path[7:])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from goinstall.log
func readPackageList() {
	pkglistdata, _ := ioutil.ReadFile(logfile)
	pkglist := strings.Fields(string(pkglistdata))
	for _, pkg := range pkglist {
		installedPkgs[pkg] = true
	}
}

// logPackage logs the named package as installed in goinstall.log, if the package is not found in there
func logPackage(pkg string) {
	if installedPkgs[pkg] {
		return
	}
	fout, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", argv0, err)
		return
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) (built bool) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote := isRemote(pkg)
	if remote && (err == build.ErrNotFound || (err == nil && *update)) {
		printf("%s: download\n", pkg)
		err = download(pkg, tree.SrcDir())
	}
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		errorf("%s: package has no files\n", pkg)
		return
	}
	var depBuilt bool
	for _, p := range dirInfo.Imports {
		if p != "C" {
			if install(p, pkg) {
				depBuilt = true
			}
		}
	}
	if errors {
		return
	}

	// Install this package.
	script, err := build.Build(tree, pkg, dirInfo)
	if err != nil {
		errorf("%s: install: %v\n", pkg, err)
		return
	}
	if *nuke {
		printf("%s: nuke\n", pkg)
		script.Nuke()
	} else if *clean {
		printf("%s: clean\n", pkg)
		script.Clean()
	}
	if *doInstall {
		if depBuilt || script.Stale() {
			printf("%s: install\n", pkg)
			if err := script.Run(); err != nil {
				errorf("%s: install: %v\n", pkg, err)
				return
			}
			built = true
		} else {
			printf("%s: up-to-date\n", pkg)
		}
	}
	if remote {
		// mark package as installed in $GOROOT/goinstall.log
		logPackage(pkg)
	}
	return
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.ErrorString("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<commit_msg>goinstall: undo 1ad616fb313d (always rebuild...)<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n")
	fmt.Fprintf(os.Stderr, "\tgoinstall -a\n")
	flag.PrintDefaults()
	os.Exit(2)
}

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	logfile       = filepath.Join(runtime.GOROOT(), "goinstall.log")
	installedPkgs = make(map[string]bool)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	logPkgs           = flag.Bool("log", true, "log installed packages to $GOROOT/goinstall.log for use by -a")
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg || *logPkgs {
		readPackageList()
	}
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		if len(installedPkgs) == 0 {
			fmt.Fprintf(os.Stderr, "%s: no installed packages\n", argv0)
			os.Exit(1)
		}
		args = make([]string, len(installedPkgs), len(installedPkgs))
		i := 0
		for pkg := range installedPkgs {
			args[i] = pkg
			i++
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if strings.HasPrefix(path, "http://") {
			errorf("'http://' used in remote path, try '%s'\n", path[7:])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from goinstall.log
func readPackageList() {
	pkglistdata, _ := ioutil.ReadFile(logfile)
	pkglist := strings.Fields(string(pkglistdata))
	for _, pkg := range pkglist {
		installedPkgs[pkg] = true
	}
}

// logPackage logs the named package as installed in goinstall.log, if the package is not found in there
func logPackage(pkg string) {
	if installedPkgs[pkg] {
		return
	}
	fout, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", argv0, err)
		return
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote := isRemote(pkg)
	if remote && (err == build.ErrNotFound || (err == nil && *update)) {
		printf("%s: download\n", pkg)
		err = download(pkg, tree.SrcDir())
	}
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		errorf("%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		errorf("%s: package has no files\n", pkg)
		return
	}
	for _, p := range dirInfo.Imports {
		if p != "C" {
			install(p, pkg)
		}
	}
	if errors {
		return
	}

	// Install this package.
	script, err := build.Build(tree, pkg, dirInfo)
	if err != nil {
		errorf("%s: install: %v\n", pkg, err)
		return
	}
	if *nuke {
		printf("%s: nuke\n", pkg)
		script.Nuke()
	} else if *clean {
		printf("%s: clean\n", pkg)
		script.Clean()
	}
	if *doInstall {
		if script.Stale() {
			printf("%s: install\n", pkg)
			if err := script.Run(); err != nil {
				errorf("%s: install: %v\n", pkg, err)
				return
			}
		} else {
			printf("%s: up-to-date\n", pkg)
		}
	}
	if remote {
		// mark package as installed in $GOROOT/goinstall.log
		logPackage(pkg)
	}
	return
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.ErrorString("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, local bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(dir + "/Makefile")
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string) ([]byte, os.Error) {
	dirInfo, err := scanDir(dir, false)
	if err != nil {
		return nil, err
	}

	if len(dirInfo.cgoFiles) == 0 && len(dirInfo.cFiles) > 0 {
		// When using cgo, .c files are compiled with gcc.  Without cgo,
		// they may be intended for 6c.  Just error out for now.
		return nil, os.ErrorString("C files found in non-cgo package")
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		isCgo[file] = true
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		oFiles = append(oFiles, file[:len(file)-2]+".o")
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	var buf bytes.Buffer
	md := makedata{pkg, goFiles, cgoFiles, oFiles}
	if err := makefileTemplate.Execute(&buf, &md); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	Pkg      string   // package import path
	GoFiles  []string // list of non-cgo .go files
	CgoFiles []string // list of cgo .go files
	OFiles   []string // list of ofiles for cgo
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={Pkg}

{.section GoFiles}
GOFILES=\
{.repeated section GoFiles}
	{@}\
{.end}

{.end}
{.section CgoFiles}
CGOFILES=\
{.repeated section CgoFiles}
	{@}\
{.end}

{.end}
{.section OFiles}
CGO_OFILES=\
{.repeated section OFiles}
	{@}\
{.end}

{.end}
include $(GOROOT)/src/Make.pkg
`,
	nil)
<commit_msg>goinstall: protect against malicious filenames. It was possible to make package run arbitrary commands when installing if its filenames contained make metacharacters.<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, local bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(dir + "/Makefile")
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string) ([]byte, os.Error) {
	if !safeName(pkg) {
		return nil, os.ErrorString("unsafe name: " + pkg)
	}
	dirInfo, err := scanDir(dir, false)
	if err != nil {
		return nil, err
	}

	if len(dirInfo.cgoFiles) == 0 && len(dirInfo.cFiles) > 0 {
		// When using cgo, .c files are compiled with gcc.  Without cgo,
		// they may be intended for 6c.  Just error out for now.
		return nil, os.ErrorString("C files found in non-cgo package")
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		if !safeName(file) {
			return nil, os.ErrorString("bad name: " + file)
		}
		isCgo[file] = true
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		oFiles = append(oFiles, file[:len(file)-2]+".o")
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !safeName(file) {
			return nil, os.ErrorString("unsafe name: " + file)
		}
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	var buf bytes.Buffer
	md := makedata{pkg, goFiles, cgoFiles, oFiles}
	if err := makefileTemplate.Execute(&buf, &md); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

var safeBytes = []byte("+-./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz")

func safeName(s string) bool {
	for i := 0; i < len(s); i++ {
		if c := s[i]; c < 0x80 && bytes.IndexByte(safeBytes, c) < 0 {
			return false
		}
	}
	return true
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	Pkg      string   // package import path
	GoFiles  []string // list of non-cgo .go files
	CgoFiles []string // list of cgo .go files
	OFiles   []string // list of ofiles for cgo
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={Pkg}

{.section GoFiles}
GOFILES=\
{.repeated section GoFiles}
	{@}\
{.end}

{.end}
{.section CgoFiles}
CGOFILES=\
{.repeated section CgoFiles}
	{@}\
{.end}

{.end}
{.section OFiles}
CGO_OFILES=\
{.repeated section OFiles}
	{@}\
{.end}

{.end}
include $(GOROOT)/src/Make.pkg
`,
	nil)
<|endoftext|>


<commit_before>package torrent

import (
	"net"
	"strconv"
	"time"

	"github.com/anacrolix/torrent/webtorrent"
	"github.com/pion/datachannel"
	"github.com/pion/webrtc/v3"
)

const webrtcNetwork = "webrtc"

type webrtcNetConn struct {
	datachannel.ReadWriteCloser
	webtorrent.DataChannelContext
}

type webrtcNetAddr struct {
	*webrtc.ICECandidate
}

var _ net.Addr = webrtcNetAddr{}

func (webrtcNetAddr) Network() string {
	// Now that we have the ICE candidate, we can tell if it's over udp or tcp. But should we use
	// that for the network?
	return webrtcNetwork
}

func (me webrtcNetAddr) String() string {
	return net.JoinHostPort(me.Address, strconv.FormatUint(uint64(me.Port), 10))
}

func (me webrtcNetConn) LocalAddr() net.Addr {
	// I'm not sure if this evolves over time. It might also be unavailable if the PeerConnection is
	// closed or closes itself. The same concern applies to RemoteAddr.
	pair, err := me.DataChannelContext.GetSelectedIceCandidatePair()
	if err != nil {
		panic(err)
	}
	return webrtcNetAddr{pair.Local}
}

func (me webrtcNetConn) RemoteAddr() net.Addr {
	// See comments on LocalAddr.
	pair, err := me.DataChannelContext.GetSelectedIceCandidatePair()
	if err != nil {
		panic(err)
	}
	return webrtcNetAddr{pair.Remote}
}

// Do we need these for WebRTC connections exposed as net.Conns? Can we set them somewhere inside
// PeerConnection or on the channel or some transport?

func (w webrtcNetConn) SetDeadline(t time.Time) error {
	return nil
}

func (w webrtcNetConn) SetReadDeadline(t time.Time) error {
	return nil
}

func (w webrtcNetConn) SetWriteDeadline(t time.Time) error {
	return nil
}
<commit_msg>Trace some webrtcNetConn use<commit_after>package torrent

import (
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/trace"
	"net"
	"strconv"
	"time"

	"github.com/anacrolix/torrent/webtorrent"
	"github.com/pion/datachannel"
	"github.com/pion/webrtc/v3"
)

const webrtcNetwork = "webrtc"

type webrtcNetConn struct {
	datachannel.ReadWriteCloser
	webtorrent.DataChannelContext
}

type webrtcNetAddr struct {
	*webrtc.ICECandidate
}

var _ net.Addr = webrtcNetAddr{}

func (webrtcNetAddr) Network() string {
	// Now that we have the ICE candidate, we can tell if it's over udp or tcp. But should we use
	// that for the network?
	return webrtcNetwork
}

func (me webrtcNetAddr) String() string {
	return net.JoinHostPort(me.Address, strconv.FormatUint(uint64(me.Port), 10))
}

func (me webrtcNetConn) LocalAddr() net.Addr {
	// I'm not sure if this evolves over time. It might also be unavailable if the PeerConnection is
	// closed or closes itself. The same concern applies to RemoteAddr.
	pair, err := me.DataChannelContext.GetSelectedIceCandidatePair()
	if err != nil {
		panic(err)
	}
	return webrtcNetAddr{pair.Local}
}

func (me webrtcNetConn) RemoteAddr() net.Addr {
	// See comments on LocalAddr.
	pair, err := me.DataChannelContext.GetSelectedIceCandidatePair()
	if err != nil {
		panic(err)
	}
	return webrtcNetAddr{pair.Remote}
}

// Do we need these for WebRTC connections exposed as net.Conns? Can we set them somewhere inside
// PeerConnection or on the channel or some transport?

func (w webrtcNetConn) SetDeadline(t time.Time) error {
	w.Span.AddEvent("SetDeadline", trace.WithAttributes(attribute.String("time", t.String())))
	return nil
}

func (w webrtcNetConn) SetReadDeadline(t time.Time) error {
	w.Span.AddEvent("SetReadDeadline", trace.WithAttributes(attribute.String("time", t.String())))
	return nil
}

func (w webrtcNetConn) SetWriteDeadline(t time.Time) error {
	w.Span.AddEvent("SetWriteDeadline", trace.WithAttributes(attribute.String("time", t.String())))
	return nil
}

func (w webrtcNetConn) Read(b []byte) (n int, err error) {
	_, span := otel.Tracer(tracerName).Start(w.Context, "Read")
	defer span.End()
	span.SetAttributes(attribute.Int("buf_len", len(b)))
	n, err = w.ReadWriteCloser.Read(b)
	span.RecordError(err)
	span.SetAttributes(attribute.Int("bytes_read", n))
	return
}
<|endoftext|>


<commit_before>package widget

import (
	"fmt"
	"html/template"
	"os"
	"path"

	"github.com/jinzhu/gorm"
	"github.com/qor/admin"
	"github.com/qor/assetfs"
	"github.com/qor/qor/resource"
	"github.com/qor/roles"
)

var (
	root, _                = os.Getwd()
	viewPaths              []string
	registeredWidgets      []*Widget
	registeredWidgetsGroup []*WidgetsGroup
)

// Config widget config
type Config struct {
	DB            *gorm.DB
	Admin         *admin.Admin
	PreviewAssets []string
}

func init() {
	if path := os.Getenv("WEB_ROOT"); path != "" {
		root = path
	}
}

// New new widgets container
func New(config *Config) *Widgets {
	widgets := &Widgets{Config: config, funcMaps: template.FuncMap{}, AssetFS: assetfs.AssetFS.NameSpace("widgets")}

	if root != "" {
		widgets.RegisterViewPath(path.Join(root, "app/views/widgets"))
	}
	widgets.RegisterViewPath("app/views/widgets")
	return widgets
}

// Widgets widgets container
type Widgets struct {
	funcMaps              template.FuncMap
	Config                *Config
	Resource              *admin.Resource
	AssetFS               assetfs.Interface
	WidgetSettingResource *admin.Resource
}

// SetAssetFS set asset fs for render
func (widgets *Widgets) SetAssetFS(assetFS assetfs.Interface) {
	for _, viewPath := range viewPaths {
		assetFS.RegisterPath(viewPath)
	}

	widgets.AssetFS = assetFS
}

// RegisterWidget register a new widget
func (widgets *Widgets) RegisterWidget(w *Widget) {
	registeredWidgets = append(registeredWidgets, w)
}

// RegisterWidgetsGroup register widgets group
func (widgets *Widgets) RegisterWidgetsGroup(group *WidgetsGroup) {
	registeredWidgetsGroup = append(registeredWidgetsGroup, group)
}

// RegisterFuncMap register view funcs, it could be used when render templates
func (widgets *Widgets) RegisterFuncMap(name string, fc interface{}) {
	widgets.funcMaps[name] = fc
}

// ConfigureQorResourceBeforeInitialize a method used to config Widget for qor admin
func (widgets *Widgets) ConfigureQorResourceBeforeInitialize(res resource.Resourcer) {
	if res, ok := res.(*admin.Resource); ok {
		// register view paths
		res.GetAdmin().RegisterViewPath("github.com/qor/widget/views")

		// set resources
		widgets.Resource = res

		// set setting resource
		if widgets.WidgetSettingResource == nil {
			widgets.WidgetSettingResource = res.GetAdmin().NewResource(&QorWidgetSetting{}, &admin.Config{Name: res.Name})
		}

		res.Name = widgets.WidgetSettingResource.Name

		for funcName, fc := range funcMap {
			res.GetAdmin().RegisterFuncMap(funcName, fc)
		}

		// configure routes
		controller := widgetController{Widgets: widgets}
		router := res.GetAdmin().GetRouter()
		router.Get(widgets.WidgetSettingResource.ToParam(), controller.Index, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/new", widgets.WidgetSettingResource.ToParam()), controller.New, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/!setting", widgets.WidgetSettingResource.ToParam()), controller.Setting, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/%v", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Edit, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/%v/!preview", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Preview, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/%v/edit", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Edit, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Put(fmt.Sprintf("%v/%v", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Update, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Post(widgets.WidgetSettingResource.ToParam(), controller.Update, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/inline-edit", res.ToParam()), controller.InlineEdit, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
	}
}

// Widget widget struct
type Widget struct {
	Name        string
	PreviewIcon string
	Group       string
	Templates   []string
	Setting     *admin.Resource
	Permission  *roles.Permission
	Context     func(context *Context, setting interface{}) *Context
}

// WidgetsGroup widgets Group
type WidgetsGroup struct {
	Name    string
	Widgets []string
}

// GetWidget get widget by name
func GetWidget(name string) *Widget {
	for _, w := range registeredWidgets {
		if w.Name == name {
			return w
		}
	}

	for _, g := range registeredWidgetsGroup {
		if g.Name == name {
			for _, widgetName := range g.Widgets {
				return GetWidget(widgetName)
			}
		}
	}
	return nil
}
<commit_msg>Change AssetFS to method<commit_after>package widget

import (
	"fmt"
	"html/template"
	"os"
	"path"

	"github.com/jinzhu/gorm"
	"github.com/qor/admin"
	"github.com/qor/assetfs"
	"github.com/qor/qor/resource"
	"github.com/qor/roles"
)

var (
	root, _                = os.Getwd()
	viewPaths              []string
	registeredWidgets      []*Widget
	registeredWidgetsGroup []*WidgetsGroup
)

// Config widget config
type Config struct {
	DB            *gorm.DB
	Admin         *admin.Admin
	PreviewAssets []string
}

func init() {
	if path := os.Getenv("WEB_ROOT"); path != "" {
		root = path
	}
}

// New new widgets container
func New(config *Config) *Widgets {
	widgets := &Widgets{Config: config, funcMaps: template.FuncMap{}, AssetFS: assetfs.AssetFS().NameSpace("widgets")}

	if root != "" {
		widgets.RegisterViewPath(path.Join(root, "app/views/widgets"))
	}
	widgets.RegisterViewPath("app/views/widgets")
	return widgets
}

// Widgets widgets container
type Widgets struct {
	funcMaps              template.FuncMap
	Config                *Config
	Resource              *admin.Resource
	AssetFS               assetfs.Interface
	WidgetSettingResource *admin.Resource
}

// SetAssetFS set asset fs for render
func (widgets *Widgets) SetAssetFS(assetFS assetfs.Interface) {
	for _, viewPath := range viewPaths {
		assetFS.RegisterPath(viewPath)
	}

	widgets.AssetFS = assetFS
}

// RegisterWidget register a new widget
func (widgets *Widgets) RegisterWidget(w *Widget) {
	registeredWidgets = append(registeredWidgets, w)
}

// RegisterWidgetsGroup register widgets group
func (widgets *Widgets) RegisterWidgetsGroup(group *WidgetsGroup) {
	registeredWidgetsGroup = append(registeredWidgetsGroup, group)
}

// RegisterFuncMap register view funcs, it could be used when render templates
func (widgets *Widgets) RegisterFuncMap(name string, fc interface{}) {
	widgets.funcMaps[name] = fc
}

// ConfigureQorResourceBeforeInitialize a method used to config Widget for qor admin
func (widgets *Widgets) ConfigureQorResourceBeforeInitialize(res resource.Resourcer) {
	if res, ok := res.(*admin.Resource); ok {
		// register view paths
		res.GetAdmin().RegisterViewPath("github.com/qor/widget/views")

		// set resources
		widgets.Resource = res

		// set setting resource
		if widgets.WidgetSettingResource == nil {
			widgets.WidgetSettingResource = res.GetAdmin().NewResource(&QorWidgetSetting{}, &admin.Config{Name: res.Name})
		}

		res.Name = widgets.WidgetSettingResource.Name

		for funcName, fc := range funcMap {
			res.GetAdmin().RegisterFuncMap(funcName, fc)
		}

		// configure routes
		controller := widgetController{Widgets: widgets}
		router := res.GetAdmin().GetRouter()
		router.Get(widgets.WidgetSettingResource.ToParam(), controller.Index, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/new", widgets.WidgetSettingResource.ToParam()), controller.New, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/!setting", widgets.WidgetSettingResource.ToParam()), controller.Setting, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/%v", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Edit, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/%v/!preview", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Preview, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/%v/edit", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Edit, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Put(fmt.Sprintf("%v/%v", widgets.WidgetSettingResource.ToParam(), widgets.WidgetSettingResource.ParamIDName()), controller.Update, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Post(widgets.WidgetSettingResource.ToParam(), controller.Update, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
		router.Get(fmt.Sprintf("%v/inline-edit", res.ToParam()), controller.InlineEdit, &admin.RouteConfig{Resource: widgets.WidgetSettingResource})
	}
}

// Widget widget struct
type Widget struct {
	Name        string
	PreviewIcon string
	Group       string
	Templates   []string
	Setting     *admin.Resource
	Permission  *roles.Permission
	Context     func(context *Context, setting interface{}) *Context
}

// WidgetsGroup widgets Group
type WidgetsGroup struct {
	Name    string
	Widgets []string
}

// GetWidget get widget by name
func GetWidget(name string) *Widget {
	for _, w := range registeredWidgets {
		if w.Name == name {
			return w
		}
	}

	for _, g := range registeredWidgetsGroup {
		if g.Name == name {
			for _, widgetName := range g.Widgets {
				return GetWidget(widgetName)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package glimmer

import (
	"github.com/go-gl/gl/v3.3-core/gl"
	"github.com/go-gl/glfw/v3.1/glfw"
)

type Profile interface {
	InitialSize() (width int, height int)
	InitialTitle() string
	PreCreation()
	PostCreation(w *glfw.Window)
	EventFocus(w *glfw.Window, focused bool)
	EventResize(w *glfw.Window, width int, height int)
	EventMousePos(w *glfw.Window, x float64, y float64)
	EventMouseKey(w *glfw.Window, button glfw.MouseButton, action glfw.Action, mods glfw.ModifierKey)
	EventKey(w *glfw.Window, key glfw.Key, scancode int, action glfw.Action, mods glfw.ModifierKey)
	EventRune(w *glfw.Window, char rune)
	Draw(w *glfw.Window)
	Cycle(w *glfw.Window)
}

type DefaultProfile struct {
}

func (p DefaultProfile) InitialSize() (int, int) {
	return 640, 480
}
func (p DefaultProfile) InitialTitle() string {
	return "Default Title"
}
func (p DefaultProfile) PreCreation() {
	glfw.WindowHint(glfw.Resizable, glfw.True)
	glfw.WindowHint(glfw.ContextVersionMajor, 3)
	glfw.WindowHint(glfw.ContextVersionMinor, 3)
}
func (p DefaultProfile) PostCreation(w *glfw.Window) {
	glfw.SwapInterval(1)
}
func (p DefaultProfile) EventFocus(w *glfw.Window, focused bool)            {}
func (p DefaultProfile) EventResize(w *glfw.Window, width int, height int)  {}
func (p DefaultProfile) EventMousePos(w *glfw.Window, x float64, y float64) {}
func (p DefaultProfile) EventMouseKey(w *glfw.Window, key glfw.MouseButton, act glfw.Action, mod glfw.ModifierKey) {
}
func (p DefaultProfile) EventKey(w *glfw.Window, key glfw.Key, scancode int, action glfw.Action, mods glfw.ModifierKey) {
}
func (p DefaultProfile) EventRune(w *glfw.Window, char rune) {}
func (p DefaultProfile) Draw(w *glfw.Window)                 {}
func (p DefaultProfile) Cycle(w *glfw.Window) {
	p.Draw(w)
	w.SwapBuffers()
	glfw.PollEvents()
}

func Run(p Profile) error {
	err := glfw.Init()
	if err != nil {
		return err
	}
	defer glfw.Terminate()

	p.PreCreation()

	width, height := p.InitialSize()
	title := p.InitialTitle()
	w, err := glfw.CreateWindow(width, height, title, nil, nil)
	if err != nil {
		return err
	}
	defer w.Destroy()

	w.MakeContextCurrent()

	p.PostCreation(w)

	w.SetFocusCallback(p.EventFocus)
	w.SetSizeCallback(p.EventResize)
	w.SetCursorPosCallback(p.EventMousePos)
	w.SetMouseButtonCallback(p.EventMouseKey)
	w.SetKeyCallback(p.EventKey)
	w.SetCharCallback(p.EventRune)

	err = gl.Init()
	if err != nil {
		return err
	}

	for !w.ShouldClose() {
		p.Cycle(w)
	}
	return nil
}
<commit_msg>lock os thread in library<commit_after>package glimmer

import (
	"runtime"

	"github.com/go-gl/gl/v3.3-core/gl"
	"github.com/go-gl/glfw/v3.1/glfw"
)

func init() {
	runtime.LockOSThread()
}

type Profile interface {
	InitialSize() (width int, height int)
	InitialTitle() string
	PreCreation()
	PostCreation(w *glfw.Window)
	EventFocus(w *glfw.Window, focused bool)
	EventResize(w *glfw.Window, width int, height int)
	EventMousePos(w *glfw.Window, x float64, y float64)
	EventMouseKey(w *glfw.Window, button glfw.MouseButton, action glfw.Action, mods glfw.ModifierKey)
	EventKey(w *glfw.Window, key glfw.Key, scancode int, action glfw.Action, mods glfw.ModifierKey)
	EventRune(w *glfw.Window, char rune)
	Draw(w *glfw.Window)
	Cycle(w *glfw.Window)
}

type DefaultProfile struct {
}

func (p DefaultProfile) InitialSize() (int, int) {
	return 640, 480
}
func (p DefaultProfile) InitialTitle() string {
	return "Default Title"
}
func (p DefaultProfile) PreCreation() {
	glfw.WindowHint(glfw.Resizable, glfw.True)
	glfw.WindowHint(glfw.ContextVersionMajor, 3)
	glfw.WindowHint(glfw.ContextVersionMinor, 3)
}
func (p DefaultProfile) PostCreation(w *glfw.Window) {
	glfw.SwapInterval(1)
}
func (p DefaultProfile) EventFocus(w *glfw.Window, focused bool)            {}
func (p DefaultProfile) EventResize(w *glfw.Window, width int, height int)  {}
func (p DefaultProfile) EventMousePos(w *glfw.Window, x float64, y float64) {}
func (p DefaultProfile) EventMouseKey(w *glfw.Window, key glfw.MouseButton, act glfw.Action, mod glfw.ModifierKey) {
}
func (p DefaultProfile) EventKey(w *glfw.Window, key glfw.Key, scancode int, action glfw.Action, mods glfw.ModifierKey) {
}
func (p DefaultProfile) EventRune(w *glfw.Window, char rune) {}
func (p DefaultProfile) Draw(w *glfw.Window)                 {}
func (p DefaultProfile) Cycle(w *glfw.Window) {
	p.Draw(w)
	w.SwapBuffers()
	glfw.PollEvents()
}

func Run(p Profile) error {
	err := glfw.Init()
	if err != nil {
		return err
	}
	defer glfw.Terminate()

	p.PreCreation()

	width, height := p.InitialSize()
	title := p.InitialTitle()
	w, err := glfw.CreateWindow(width, height, title, nil, nil)
	if err != nil {
		return err
	}
	defer w.Destroy()

	w.MakeContextCurrent()

	p.PostCreation(w)

	w.SetFocusCallback(p.EventFocus)
	w.SetSizeCallback(p.EventResize)
	w.SetCursorPosCallback(p.EventMousePos)
	w.SetMouseButtonCallback(p.EventMouseKey)
	w.SetKeyCallback(p.EventKey)
	w.SetCharCallback(p.EventRune)

	err = gl.Init()
	if err != nil {
		return err
	}

	for !w.ShouldClose() {
		p.Cycle(w)
	}
	return nil
}
<|endoftext|>


<commit_before>package gokiq

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"math/rand"
	"os"
	"os/signal"
	"reflect"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
)

type Job struct {
	Type  string        `json:"class"`
	Args  []interface{} `json:"args"`
	Queue string        `json:"queue,omitempty"`
	ID    string        `json:"jid"`

	Retry interface{} `json:"retry"` // can be int (number of retries) or bool (true means default)

	MaxRetries   int    `json:"-"`
	RetryCount   int    `json:"retry_count"`
	ErrorMessage string `json:"error_message,omitempty"`
	ErrorType    string `json:"error_class,omitempty"`
	RetriedAt    string `json:"retried_at,omitempty"`
	FailedAt     string `json:"failed_at,omitempty"`

	StartTime time.Time `json:"-"`
}

func (job *Job) FromJSON(data []byte) error {
	err := json.Unmarshal(data, job)
	if err != nil {
		return err
	}
	if max, ok := job.Retry.(float64); ok {
		job.MaxRetries = int(max)
	} else if r, ok := job.Retry.(bool); ok && !r {
	} else {
		job.MaxRetries = defaultMaxRetries
	}
	return nil
}

func (job *Job) JSON() []byte {
	res, _ := json.Marshal(job)
	return res
}

type message struct {
	job *Job
	die bool
}

const (
	TimestampFormat     = "2006-01-02 15:04:05 MST"
	redisTimeout        = 1
	defaultMaxRetries   = 25
	defaultPollInterval = 5
	defaultWorkerCount  = 25
	defaultRedisServer  = "127.0.0.1:6379"
	keyExpiry           = 86400 // one day
)

type QueueConfig map[string]int

func (q QueueConfig) String() string {
	str := ""
	for queue, priority := range q {
		str += fmt.Sprintf("%s=%d,", queue, priority)
	}
	return str[:len(str)-1]
}

type Worker interface {
	Perform([]interface{}) error
}

var Workers = NewWorkerConfig()

type WorkerConfig struct {
	RedisServer    string // TODO: allow specifying redis db
	RedisNamespace string
	Queues         QueueConfig
	WorkerCount    int
	PollInterval   int
	ReportError    func(error, *Job) // TODO: pass in a stack trace for context

	workerMapping map[string]reflect.Type
	randomQueues  []string
	redisPool     *redis.Pool
	workQueue     chan message
	done          sync.WaitGroup
	sync.RWMutex  // R is locked by Run() and scheduler(), W is locked by quitHandler() when it receives a signal
}

func NewWorkerConfig() *WorkerConfig {
	return &WorkerConfig{
		RedisServer:   defaultRedisServer,
		PollInterval:  defaultPollInterval,
		WorkerCount:   defaultWorkerCount,
		Queues:        QueueConfig{"default": 1},
		ReportError:   func(error, *Job) {},
		workerMapping: make(map[string]reflect.Type),
		workQueue:     make(chan message),
	}
}

func (w *WorkerConfig) Register(name string, worker Worker) {
	w.workerMapping[name] = workerType(worker)
}

func (w *WorkerConfig) Run() {
	log.Printf(`state=starting worker_count=%d redis=%s/0/%s queues="%s" pid=%d`, w.WorkerCount, w.RedisServer, w.RedisNamespace, w.Queues, pid)
	w.denormalizeQueues()
	w.connectRedis()

	for i := 0; i < w.WorkerCount; i++ {
		go w.worker(workerID(i))
	}

	go w.scheduler()
	go w.quitHandler()

	log.Printf(`state=started pid=%d`, pid)
	for {
		w.run()
	}
}

func (w *WorkerConfig) run() {
	w.RLock() // don't let quitHandler() stop us in the middle of a job
	defer w.RUnlock()

	msg, err := redis.Values(w.redisQuery("BLPOP", append(w.queueList(), redisTimeout)...))
	if err == redis.ErrNil {
		return
	}
	if err != nil {
		w.handleError(err)
		time.Sleep(redisTimeout * time.Second) // likely a transient redis error, sleep before retrying
		return
	}

	job := &Job{}
	err = job.FromJSON(msg[1].([]byte))
	if err != nil {
		w.handleError(err)
		return
	}
	job.Queue = string(msg[0].([]byte)[len(w.nsKey("queue:")):])
	w.workQueue <- message{job: job}
}

// create a slice of queues with duplicates using the assigned frequencies
func (w *WorkerConfig) denormalizeQueues() {
	for queue, x := range w.Queues {
		for i := 0; i < x; i++ {
			w.randomQueues = append(w.randomQueues, w.nsKey("queue:"+queue))
		}
	}
}

// get a random slice of unique queues from the slice of denormalized queues
func (w *WorkerConfig) queueList() []interface{} {
	size := len(w.Queues)
	res := make([]interface{}, 0, size)
	queues := make(map[string]struct{}, size)

	indices := rand.Perm(len(w.randomQueues))[:size]
	for _, i := range indices {
		queue := w.randomQueues[i]
		if _, ok := queues[queue]; !ok {
			queues[queue] = struct{}{}
			res = append(res, queue)
		}
	}

	return res
}

func (w *WorkerConfig) handleError(err error) {
	log.Printf(`event=error error_type=%T error_message="%s" pid=%d`, err, err, pid)
	w.ReportError(err, nil)
}

// checks the sorted set of scheduled jobs and retries and queues them when it's time
// TODO: move this to a Lua script
func (w *WorkerConfig) scheduler() {
	pollSets := []string{w.nsKey("retry"), w.nsKey("schedule")}

	for _ = range time.Tick(time.Duration(w.PollInterval) * time.Second) {
		w.RLock() // don't let quitHandler() stop us in the middle of a run
		conn := w.redisPool.Get()
		now := fmt.Sprintf("%f", currentTimeFloat())
		for _, set := range pollSets {
			conn.Send("MULTI")
			conn.Send("ZRANGEBYSCORE", set, "-inf", now)
			conn.Send("ZREMRANGEBYSCORE", set, "-inf", now)
			res, err := redis.Values(conn.Do("EXEC"))
			if err != nil {
				w.handleError(err)
				continue
			}

			for _, msg := range res[0].([]interface{}) {
				parsedMsg := &struct {
					Queue string `json:"queue"`
				}{}
				msgBytes := msg.([]byte)
				err := json.Unmarshal(msgBytes, parsedMsg)
				if err != nil {
					w.handleError(err)
					continue
				}
				conn.Send("MULTI")
				conn.Send("SADD", w.nsKey("queues"), parsedMsg.Queue)
				conn.Send("RPUSH", w.nsKey("queue:"+parsedMsg.Queue), msgBytes)
				_, err = conn.Do("EXEC")
				if err != nil {
					w.handleError(err)
				}
			}
		}
		conn.Close()
		w.RUnlock()
	}
}

// listens for SIGINT, SIGTERM, and SIGQUIT to perform a clean shutdown
func (w *WorkerConfig) quitHandler() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	signal.Notify(c, syscall.SIGQUIT)

	for sig := range c {
		log.Printf("state=stopping signal=%s pid=%d", sig, pid)
		w.Lock()           // wait for the current run loop and scheduler iterations to finish
		close(w.workQueue) // tell worker goroutines to stop after they finish their current job
		for i := 0; i < w.WorkerCount; i++ {
			w.done.Wait()
		}
		log.Printf("state=stopped pid=%d", pid)
		os.Exit(0)
	}
}

func (w *WorkerConfig) connectRedis() {
	// TODO: add a mutex for the redis pool
	if w.redisPool != nil {
		w.redisPool.Close()
	}
	w.redisPool = redis.NewPool(func() (redis.Conn, error) {
		return redis.Dial("tcp", w.RedisServer)
	}, w.WorkerCount+1)
}

func (w *WorkerConfig) redisQuery(command string, args ...interface{}) (interface{}, error) {
	conn := w.redisPool.Get()
	defer conn.Close()
	return conn.Do(command, args...)
}

func (w *WorkerConfig) worker(id string) {
	w.done.Add(1)
	for msg := range w.workQueue {
		if msg.die {
			return
		}

		job := msg.job
		typ, ok := w.workerMapping[msg.job.Type]
		if !ok {
			err := UnknownWorkerError{job.Type}
			w.scheduleRetry(job, err)
			continue
		}

		w.logJobStart(job, id)

		// wrap Perform() in a function so that we can recover from panics
		var err error
		func() {
			defer func() {
				if r := recover(); r != nil {
					// TODO: log stack trace
					err = panicToError(r)
				}
			}()
			err = reflect.New(typ).Interface().(Worker).Perform(msg.job.Args)
		}()
		if err != nil {
			w.scheduleRetry(job, err)
		}
		w.logJobFinish(job, id, err == nil)
	}
	w.done.Done()
}

func (w *WorkerConfig) scheduleRetry(job *Job, err error) {
	w.ReportError(err, job)

	now := time.Now().UTC().Format(TimestampFormat)
	if job.FailedAt == "" {
		job.FailedAt = now
	} else {
		job.RetryCount += 1
	}
	if job.RetryCount > 0 {
		job.RetriedAt = now
	}

	log.Printf(`event=job_error job_id=%s job_type=%s queue=%s retries=%d max_retries=%d error_type=%T error_message="%s" pid=%d`, job.ID, job.Type, job.Queue, job.RetryCount, job.MaxRetries, err, err, pid)

	if job.RetryCount < job.MaxRetries {
		job.ErrorType = fmt.Sprintf("%T", err)
		job.ErrorMessage = err.Error()

		nextRetry := currentTimeFloat() + retryDelay(job.RetryCount)

		w.redisQuery("ZADD", w.nsKey("retry"), strconv.FormatFloat(nextRetry, 'f', -1, 64), job.JSON())
	}
}

type runningJob struct {
	Queue     string `json:"queue"`
	Job       *Job   `json:"payload"`
	Timestamp int64  `json:"run_at"`
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobStart(job *Job, workerID string) {
	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SADD", w.nsKey("workers"), workerID)
	conn.Send("SETEX", w.nsKey("worker:"+workerID+":started"), keyExpiry, time.Now().UTC().String())
	payload := &runningJob{job.Queue, job, time.Now().Unix()}
	json, _ := json.Marshal(payload)
	conn.Send("SETEX", w.nsKey("worker:"+workerID), keyExpiry, json)
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}

	job.StartTime = time.Now()
	log.Printf("event=job_start job_id=%s job_type=%s queue=%s worker_id=%s pid=%d", job.ID, job.Type, job.Queue, workerID, pid)
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobFinish(job *Job, workerID string, success bool) {
	log.Printf("event=job_finish job_id=%s job_type=%s queue=%s duration=%v success=%t worker_id=%s pid=%d", job.ID, job.Type, job.Queue, time.Since(job.StartTime), success, workerID, pid)

	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SREM", w.nsKey("workers"), workerID)
	conn.Send("DEL", w.nsKey("worker:"+workerID+":started"))
	conn.Send("DEL", w.nsKey("worker:"+workerID))
	conn.Send("INCR", w.nsKey("stat:processed"))
	if !success {
		conn.Send("INCR", w.nsKey("stat:failed"))
	}
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}
}

func (w *WorkerConfig) nsKey(key string) string {
	if w.RedisNamespace != "" {
		return w.RedisNamespace + ":" + key
	}
	return key
}

// formula from Sidekiq (originally from delayed_job)
func retryDelay(count int) float64 {
	return math.Pow(float64(count), 4) + 15 + (float64(rand.Intn(30)) * (float64(count) + 1))
}

func currentTimeFloat() float64 {
	return float64(time.Now().UnixNano()) / float64(time.Second)
}

func panicToError(err interface{}) error {
	if str, ok := err.(string); ok {
		return fmt.Errorf(str)
	}
	return err.(error)
}

var (
	pid         = os.Getpid()
	hostname, _ = os.Hostname()
)

func workerID(i int) string {
	return fmt.Sprintf("%s:%d-%d", hostname, pid, i)
}

func workerType(worker Worker) reflect.Type {
	return reflect.Indirect(reflect.ValueOf(worker)).Type()
}

type UnknownWorkerError struct{ Type string }

func (e UnknownWorkerError) Error() string {
	return "gokiq: Unknown worker type: " + e.Type
}
<commit_msg>Track killed workers properly<commit_after>package gokiq

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"math/rand"
	"os"
	"os/signal"
	"reflect"
	"strconv"
	"sync"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
)

type Job struct {
	Type  string        `json:"class"`
	Args  []interface{} `json:"args"`
	Queue string        `json:"queue,omitempty"`
	ID    string        `json:"jid"`

	Retry interface{} `json:"retry"` // can be int (number of retries) or bool (true means default)

	MaxRetries   int    `json:"-"`
	RetryCount   int    `json:"retry_count"`
	ErrorMessage string `json:"error_message,omitempty"`
	ErrorType    string `json:"error_class,omitempty"`
	RetriedAt    string `json:"retried_at,omitempty"`
	FailedAt     string `json:"failed_at,omitempty"`

	StartTime time.Time `json:"-"`
}

func (job *Job) FromJSON(data []byte) error {
	err := json.Unmarshal(data, job)
	if err != nil {
		return err
	}
	if max, ok := job.Retry.(float64); ok {
		job.MaxRetries = int(max)
	} else if r, ok := job.Retry.(bool); ok && !r {
	} else {
		job.MaxRetries = defaultMaxRetries
	}
	return nil
}

func (job *Job) JSON() []byte {
	res, _ := json.Marshal(job)
	return res
}

type message struct {
	job *Job
	die bool
}

const (
	TimestampFormat     = "2006-01-02 15:04:05 MST"
	redisTimeout        = 1
	defaultMaxRetries   = 25
	defaultPollInterval = 5
	defaultWorkerCount  = 25
	defaultRedisServer  = "127.0.0.1:6379"
	keyExpiry           = 86400 // one day
)

type QueueConfig map[string]int

func (q QueueConfig) String() string {
	str := ""
	for queue, priority := range q {
		str += fmt.Sprintf("%s=%d,", queue, priority)
	}
	return str[:len(str)-1]
}

type Worker interface {
	Perform([]interface{}) error
}

var Workers = NewWorkerConfig()

type WorkerConfig struct {
	RedisServer    string // TODO: allow specifying redis db
	RedisNamespace string
	Queues         QueueConfig
	WorkerCount    int
	PollInterval   int
	ReportError    func(error, *Job) // TODO: pass in a stack trace for context

	workerMapping map[string]reflect.Type
	randomQueues  []string
	redisPool     *redis.Pool
	workQueue     chan message
	done          sync.WaitGroup
	sync.RWMutex  // R is locked by Run() and scheduler(), W is locked by quitHandler() when it receives a signal
}

func NewWorkerConfig() *WorkerConfig {
	return &WorkerConfig{
		RedisServer:   defaultRedisServer,
		PollInterval:  defaultPollInterval,
		WorkerCount:   defaultWorkerCount,
		Queues:        QueueConfig{"default": 1},
		ReportError:   func(error, *Job) {},
		workerMapping: make(map[string]reflect.Type),
		workQueue:     make(chan message),
	}
}

func (w *WorkerConfig) Register(name string, worker Worker) {
	w.workerMapping[name] = workerType(worker)
}

func (w *WorkerConfig) Run() {
	log.Printf(`state=starting worker_count=%d redis=%s/0/%s queues="%s" pid=%d`, w.WorkerCount, w.RedisServer, w.RedisNamespace, w.Queues, pid)
	w.denormalizeQueues()
	w.connectRedis()

	for i := 0; i < w.WorkerCount; i++ {
		go w.worker(workerID(i))
	}

	go w.scheduler()
	go w.quitHandler()

	log.Printf(`state=started pid=%d`, pid)
	for {
		w.run()
	}
}

func (w *WorkerConfig) run() {
	w.RLock() // don't let quitHandler() stop us in the middle of a job
	defer w.RUnlock()

	msg, err := redis.Values(w.redisQuery("BLPOP", append(w.queueList(), redisTimeout)...))
	if err == redis.ErrNil {
		return
	}
	if err != nil {
		w.handleError(err)
		time.Sleep(redisTimeout * time.Second) // likely a transient redis error, sleep before retrying
		return
	}

	job := &Job{}
	err = job.FromJSON(msg[1].([]byte))
	if err != nil {
		w.handleError(err)
		return
	}
	job.Queue = string(msg[0].([]byte)[len(w.nsKey("queue:")):])
	w.workQueue <- message{job: job}
}

// create a slice of queues with duplicates using the assigned frequencies
func (w *WorkerConfig) denormalizeQueues() {
	for queue, x := range w.Queues {
		for i := 0; i < x; i++ {
			w.randomQueues = append(w.randomQueues, w.nsKey("queue:"+queue))
		}
	}
}

// get a random slice of unique queues from the slice of denormalized queues
func (w *WorkerConfig) queueList() []interface{} {
	size := len(w.Queues)
	res := make([]interface{}, 0, size)
	queues := make(map[string]struct{}, size)

	indices := rand.Perm(len(w.randomQueues))[:size]
	for _, i := range indices {
		queue := w.randomQueues[i]
		if _, ok := queues[queue]; !ok {
			queues[queue] = struct{}{}
			res = append(res, queue)
		}
	}

	return res
}

func (w *WorkerConfig) handleError(err error) {
	log.Printf(`event=error error_type=%T error_message="%s" pid=%d`, err, err, pid)
	w.ReportError(err, nil)
}

// checks the sorted set of scheduled jobs and retries and queues them when it's time
// TODO: move this to a Lua script
func (w *WorkerConfig) scheduler() {
	pollSets := []string{w.nsKey("retry"), w.nsKey("schedule")}

	for _ = range time.Tick(time.Duration(w.PollInterval) * time.Second) {
		w.RLock() // don't let quitHandler() stop us in the middle of a run
		conn := w.redisPool.Get()
		now := fmt.Sprintf("%f", currentTimeFloat())
		for _, set := range pollSets {
			conn.Send("MULTI")
			conn.Send("ZRANGEBYSCORE", set, "-inf", now)
			conn.Send("ZREMRANGEBYSCORE", set, "-inf", now)
			res, err := redis.Values(conn.Do("EXEC"))
			if err != nil {
				w.handleError(err)
				continue
			}

			for _, msg := range res[0].([]interface{}) {
				parsedMsg := &struct {
					Queue string `json:"queue"`
				}{}
				msgBytes := msg.([]byte)
				err := json.Unmarshal(msgBytes, parsedMsg)
				if err != nil {
					w.handleError(err)
					continue
				}
				conn.Send("MULTI")
				conn.Send("SADD", w.nsKey("queues"), parsedMsg.Queue)
				conn.Send("RPUSH", w.nsKey("queue:"+parsedMsg.Queue), msgBytes)
				_, err = conn.Do("EXEC")
				if err != nil {
					w.handleError(err)
				}
			}
		}
		conn.Close()
		w.RUnlock()
	}
}

// listens for SIGINT, SIGTERM, and SIGQUIT to perform a clean shutdown
func (w *WorkerConfig) quitHandler() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	signal.Notify(c, syscall.SIGQUIT)

	for sig := range c {
		log.Printf("state=stopping signal=%s pid=%d", sig, pid)
		w.Lock()           // wait for the current run loop and scheduler iterations to finish
		close(w.workQueue) // tell worker goroutines to stop after they finish their current job
		for i := 0; i < w.WorkerCount; i++ {
			w.done.Wait()
		}
		log.Printf("state=stopped pid=%d", pid)
		os.Exit(0)
	}
}

func (w *WorkerConfig) connectRedis() {
	// TODO: add a mutex for the redis pool
	if w.redisPool != nil {
		w.redisPool.Close()
	}
	w.redisPool = redis.NewPool(func() (redis.Conn, error) {
		return redis.Dial("tcp", w.RedisServer)
	}, w.WorkerCount+1)
}

func (w *WorkerConfig) redisQuery(command string, args ...interface{}) (interface{}, error) {
	conn := w.redisPool.Get()
	defer conn.Close()
	return conn.Do(command, args...)
}

func (w *WorkerConfig) worker(id string) {
	w.done.Add(1)
	for msg := range w.workQueue {
		if msg.die {
			break
		}

		job := msg.job
		typ, ok := w.workerMapping[msg.job.Type]
		if !ok {
			err := UnknownWorkerError{job.Type}
			w.scheduleRetry(job, err)
			continue
		}

		w.logJobStart(job, id)

		// wrap Perform() in a function so that we can recover from panics
		var err error
		func() {
			defer func() {
				if r := recover(); r != nil {
					// TODO: log stack trace
					err = panicToError(r)
				}
			}()
			err = reflect.New(typ).Interface().(Worker).Perform(msg.job.Args)
		}()
		if err != nil {
			w.scheduleRetry(job, err)
		}
		w.logJobFinish(job, id, err == nil)
	}
	w.done.Done()
}

func (w *WorkerConfig) scheduleRetry(job *Job, err error) {
	w.ReportError(err, job)

	now := time.Now().UTC().Format(TimestampFormat)
	if job.FailedAt == "" {
		job.FailedAt = now
	} else {
		job.RetryCount += 1
	}
	if job.RetryCount > 0 {
		job.RetriedAt = now
	}

	log.Printf(`event=job_error job_id=%s job_type=%s queue=%s retries=%d max_retries=%d error_type=%T error_message="%s" pid=%d`, job.ID, job.Type, job.Queue, job.RetryCount, job.MaxRetries, err, err, pid)

	if job.RetryCount < job.MaxRetries {
		job.ErrorType = fmt.Sprintf("%T", err)
		job.ErrorMessage = err.Error()

		nextRetry := currentTimeFloat() + retryDelay(job.RetryCount)

		w.redisQuery("ZADD", w.nsKey("retry"), strconv.FormatFloat(nextRetry, 'f', -1, 64), job.JSON())
	}
}

type runningJob struct {
	Queue     string `json:"queue"`
	Job       *Job   `json:"payload"`
	Timestamp int64  `json:"run_at"`
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobStart(job *Job, workerID string) {
	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SADD", w.nsKey("workers"), workerID)
	conn.Send("SETEX", w.nsKey("worker:"+workerID+":started"), keyExpiry, time.Now().UTC().String())
	payload := &runningJob{job.Queue, job, time.Now().Unix()}
	json, _ := json.Marshal(payload)
	conn.Send("SETEX", w.nsKey("worker:"+workerID), keyExpiry, json)
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}

	job.StartTime = time.Now()
	log.Printf("event=job_start job_id=%s job_type=%s queue=%s worker_id=%s pid=%d", job.ID, job.Type, job.Queue, workerID, pid)
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobFinish(job *Job, workerID string, success bool) {
	log.Printf("event=job_finish job_id=%s job_type=%s queue=%s duration=%v success=%t worker_id=%s pid=%d", job.ID, job.Type, job.Queue, time.Since(job.StartTime), success, workerID, pid)

	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SREM", w.nsKey("workers"), workerID)
	conn.Send("DEL", w.nsKey("worker:"+workerID+":started"))
	conn.Send("DEL", w.nsKey("worker:"+workerID))
	conn.Send("INCR", w.nsKey("stat:processed"))
	if !success {
		conn.Send("INCR", w.nsKey("stat:failed"))
	}
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}
}

func (w *WorkerConfig) nsKey(key string) string {
	if w.RedisNamespace != "" {
		return w.RedisNamespace + ":" + key
	}
	return key
}

// formula from Sidekiq (originally from delayed_job)
func retryDelay(count int) float64 {
	return math.Pow(float64(count), 4) + 15 + (float64(rand.Intn(30)) * (float64(count) + 1))
}

func currentTimeFloat() float64 {
	return float64(time.Now().UnixNano()) / float64(time.Second)
}

func panicToError(err interface{}) error {
	if str, ok := err.(string); ok {
		return fmt.Errorf(str)
	}
	return err.(error)
}

var (
	pid         = os.Getpid()
	hostname, _ = os.Hostname()
)

func workerID(i int) string {
	return fmt.Sprintf("%s:%d-%d", hostname, pid, i)
}

func workerType(worker Worker) reflect.Type {
	return reflect.Indirect(reflect.ValueOf(worker)).Type()
}

type UnknownWorkerError struct{ Type string }

func (e UnknownWorkerError) Error() string {
	return "gokiq: Unknown worker type: " + e.Type
}
<|endoftext|>