<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "strconv"
  "strings"
  "net/textproto"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod Moderation
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  // http frontend
  frontend Frontend

  // thumbnail generator for images
  img_thm ThumbnailGenerator

  // nntp feeds map, feed, isoutbound
  feeds map[NNTPConnection]bool
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their message id
  send_all_feeds chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  allow_tor := self.conf.daemon["allow_tor"]
  allow_tor_attachments := self.conf.daemon["allow_tor_attachments"]
  feed := NNTPConnection{conn, textproto.NewConn(conn), inbound, self.debug, new(ConnectionInfo), policy,  make(chan string, 512), self.store, self.store, allow_tor == "1", allow_tor_attachments == "1"}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      // start syncing in background
      go func() {
        if self.sync_on_start {
          log.Println("sync on start")
          // get every article
          articles := self.database.GetAllArticles()
          // wait 5 seconds for feed to handshake
          time.Sleep(5 * time.Second)
          log.Println("outfeed begin sync")
          for _, result := range articles {
            msgid := result[0]
            group := result[1]
            if policy.AllowsNewsgroup(group) {
              //XXX: will this crash if interrupted?
              nntp.sync <- msgid
            }
          }
          log.Println("outfeed end sync")
        }
      }()
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  self.pollfeeds()

}

func (self *NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    select {
    case nntp := <- chnl:
      // new post from frontend
      log.Println("frontend post", nntp.MessageID())
      self.infeed <- nntp
    }
  }
}

func (self *NNTPDaemon) pollfeeds() {
  var chnl chan NNTPMessage
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {
    select {
    case msgid := <- self.send_all_feeds:
      // send all feeds
      nntp := self.store.GetMessage(msgid)
      if nntp == nil {
        log.Printf("failed to load %s for federation", msgid)
      } else {
        for feed , use := range self.feeds {
          if use && feed.policy != nil {
            if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
              feed.sync <- nntp.MessageID()
            } else {
              log.Println("not syncing", msgid)
            }
          }
        }
      }
    case msgid := <- self.infeed_load:
      log.Println("load from infeed", msgid)
      msg := self.store.ReadTempMessage(msgid)
      if msg != nil {
        self.infeed <- msg
      }
    case nntp := <- self.infeed:
      // ammend path
      nntp.AppendPath(self.instance_name)
      msgid := nntp.MessageID()
      log.Println("daemon got", msgid)

      // store article and attachments
      // register with database
      // this also generates thumbnails
      go self.store.StorePost(nntp)

      // prepare for content rollover
      // fallback rollover
      rollover := 100

      group := nntp.Newsgroup()
      tpp, err := self.database.GetThreadsPerPage(group)
      ppb, err := self.database.GetPagesPerBoard(group)
      if err == nil {
        rollover = tpp * ppb
      }

      // roll over old content
      self.expire.ExpireGroup(group, rollover)
      if err == nil {
        // queue to all outfeeds
        self.send_all_feeds <- msgid
        // tell frontend
        if chnl != nil {
          chnl <- nntp
        }
      } else {
        log.Printf("%s failed to store: %s", msgid, err)
      }
    }
  }
}

func (self *NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunInbound(nntp)
  }
}

func (self *NNTPDaemon) RunInbound(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

func (self *NNTPDaemon) Setup() {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {

  // set up daemon configs
  self.Setup()

  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan string)
  self.feeds = make(map[NNTPConnection]bool)

  self.bind_addr = self.conf.daemon["bind"]

  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return false
  }

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  // initialize moderation engine
  self.mod.Init(self)

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    self.frontend = NewHTTPFrontend(self, self.conf.frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }
  return true
}
<commit_msg>use break in select block to try fixing deadlock<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "strconv"
  "strings"
  "net/textproto"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod Moderation
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  // http frontend
  frontend Frontend

  // thumbnail generator for images
  img_thm ThumbnailGenerator

  // nntp feeds map, feed, isoutbound
  feeds map[NNTPConnection]bool
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their message id
  send_all_feeds chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  allow_tor := self.conf.daemon["allow_tor"]
  allow_tor_attachments := self.conf.daemon["allow_tor_attachments"]
  feed := NNTPConnection{conn, textproto.NewConn(conn), inbound, self.debug, new(ConnectionInfo), policy,  make(chan string, 512), self.store, self.store, allow_tor == "1", allow_tor_attachments == "1"}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      // start syncing in background
      go func() {
        if self.sync_on_start {
          log.Println("sync on start")
          // get every article
          articles := self.database.GetAllArticles()
          // wait 5 seconds for feed to handshake
          time.Sleep(5 * time.Second)
          log.Println("outfeed begin sync")
          for _, result := range articles {
            msgid := result[0]
            group := result[1]
            if policy.AllowsNewsgroup(group) {
              //XXX: will this crash if interrupted?
              nntp.sync <- msgid
            }
          }
          log.Println("outfeed end sync")
        }
      }()
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  self.pollfeeds()

}

func (self *NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    select {
    case nntp := <- chnl:
      // new post from frontend
      log.Println("frontend post", nntp.MessageID())
      self.infeed <- nntp
    }
  }
}

func (self *NNTPDaemon) pollfeeds() {
  var chnl chan NNTPMessage
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {
    select {
    case msgid := <- self.send_all_feeds:
      // send all feeds
      nntp := self.store.GetMessage(msgid)
      if nntp == nil {
        log.Printf("failed to load %s for federation", msgid)
      } else {
        for feed , use := range self.feeds {
          if use && feed.policy != nil {
            if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
              feed.sync <- nntp.MessageID()
            } else {
              log.Println("not syncing", msgid)
            }
          }
        }
      }
      break;
    case msgid := <- self.infeed_load:
      log.Println("load from infeed", msgid)
      msg := self.store.ReadTempMessage(msgid)
      if msg != nil {
        self.infeed <- msg
      }
      break;
    case nntp := <- self.infeed:
      // ammend path
      nntp.AppendPath(self.instance_name)
      msgid := nntp.MessageID()
      log.Println("daemon got", msgid)

      // store article and attachments
      // register with database
      // this also generates thumbnails
      go self.store.StorePost(nntp)

      // prepare for content rollover
      // fallback rollover
      rollover := 100

      group := nntp.Newsgroup()
      tpp, err := self.database.GetThreadsPerPage(group)
      ppb, err := self.database.GetPagesPerBoard(group)
      if err == nil {
        rollover = tpp * ppb
      }

      // roll over old content
      self.expire.ExpireGroup(group, rollover)
      if err == nil {
        // queue to all outfeeds
        self.send_all_feeds <- msgid
        // tell frontend
        if chnl != nil {
          chnl <- nntp
        }
      } else {
        log.Printf("%s failed to store: %s", msgid, err)
      }
      break;
    }
  }
}

func (self *NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunInbound(nntp)
  }
}

func (self *NNTPDaemon) RunInbound(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

func (self *NNTPDaemon) Setup() {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {

  // set up daemon configs
  self.Setup()

  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan string)
  self.feeds = make(map[NNTPConnection]bool)

  self.bind_addr = self.conf.daemon["bind"]

  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return false
  }

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  // initialize moderation engine
  self.mod.Init(self)

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    self.frontend = NewHTTPFrontend(self, self.conf.frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }
  return true
}
<|endoftext|>


<commit_before>// Package libdocker provides docker related library functions.
package libdocker

import (
	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/resourced/resourced/libstring"
	"os"
	"path"
	"strconv"
	"sync"
)

var connections map[string]*dockerclient.Client
var connectionsLock = &sync.RWMutex{}

type CompleteDockerContainer struct {
	NiceImageName string `json:"NiceImageName,omitempty" yaml:"NiceImageName,omitempty"`
	Command       string `json:"Command,omitempty" yaml:"Command,omitempty"`
	Status        string `json:"Status,omitempty" yaml:"Status,omitempty"`
	dockerclient.Container
}

type CompleteDockerImage struct {
	RepoTags    []string `json:"RepoTags,omitempty" yaml:"RepoTags,omitempty"`
	VirtualSize int64    `json:"VirtualSize,omitempty" yaml:"VirtualSize,omitempty"`
	ParentID    string   `json:"ParentId,omitempty" yaml:"ParentId,omitempty"`
	dockerclient.Image
}

// DockerClient returns dockerclient.Client which handles Docker connection.
func DockerClient(endpoint string) (*dockerclient.Client, error) {
	var conn *dockerclient.Client
	var err error

	if endpoint == "" {
		endpoint = os.Getenv("DOCKER_HOST")
		if endpoint == "" {
			endpoint = "unix:///var/run/docker.sock"
		}
	}

	connectionsLock.RLock()
	if connections == nil {
		connectionsLock.Lock()
		connections = make(map[string]*dockerclient.Client)
		connectionsLock.Unlock()
	}
	connectionsLock.RUnlock()

	// Do not create connection if one already exist.
	connectionsLock.RLock()
	existingConnection, ok := connections[endpoint]
	connectionsLock.RUnlock()

	if ok && existingConnection != nil {
		return existingConnection, nil
	}

	dockerCertPath := os.Getenv("DOCKER_CERT_PATH")
	if dockerCertPath != "" {
		cert := path.Join(dockerCertPath, "cert.pem")
		key := path.Join(dockerCertPath, "key.pem")
		ca := path.Join(dockerCertPath, "ca.pem")

		conn, err = dockerclient.NewTLSClient(endpoint, cert, key, ca)
	} else {
		conn, err = dockerclient.NewClient(endpoint)
	}

	if err == nil && conn != nil {
		connectionsLock.Lock()
		connections[endpoint] = conn
		connectionsLock.Unlock()
	}

	return conn, err
}

// InfoAndVersion is a convenience function to fetch info and version data.
func InfoAndVersion(endpoint string) (map[string]interface{}, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	version, err := client.Version()
	if err != nil {
		return nil, err
	}

	info, err := client.Info()
	if err != nil {
		return nil, err
	}

	versionAsMap := version.Map()
	infoAsMap := info.Map()

	data := make(map[string]interface{})

	for key, value := range versionAsMap {
		data[key] = value
	}

	data["Driver"] = make(map[string]interface{})

	for key, value := range infoAsMap {
		if libstring.StringInSlice(key, []string{"NGoroutines", "Containers", "Images", "MemTotal"}) {
			data[key] = info.GetInt64(key)

		} else if key == "NFd" {
			data["NumFileDescriptors"] = info.GetInt64(key)

		} else if key == "NEventsListener" {
			data["NumEventsListeners"] = info.GetInt64(key)

		} else if key == "NCPU" {
			data["NumCPUs"] = info.GetInt64(key)

		} else if libstring.StringInSlice(key, []string{"Debug", "IPv4Forwarding", "MemoryLimit", "SwapLimit"}) {
			data[key] = info.GetBool(key)

		} else if key == "Driver" {
			driverMap := data["Driver"].(map[string]interface{})
			driverMap["Name"] = value

		} else if key == "DriverStatus" {
			tupleSlice := make([][]string, 2)
			info.GetJSON(key, &tupleSlice)

			for _, tuple := range tupleSlice {
				tupleKey := tuple[0]
				tupleValue := tuple[1]

				driverMap := data["Driver"].(map[string]interface{})

				if tupleKey == "Root Dir" {
					driverMap["RootDir"] = tupleValue
				}
				if tupleKey == "Dirs" {
					tupleValueInt64, err := strconv.ParseInt(tupleValue, 10, 64)
					if err == nil {
						driverMap[tupleKey] = tupleValueInt64
					}
				}
			}

		} else if key == "RegistryConfig" {
			registryConfig := make(map[string]interface{})
			err := info.GetJSON(key, &registryConfig)
			if err == nil {
				data[key] = registryConfig
			}

		} else {
			data[key] = value
		}
	}

	return data, nil
}

// AllContainers is a convenience function to fetch a slice of all containers data.
func AllContainers(endpoint string) ([]dockerclient.APIContainers, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListContainers(dockerclient.ListContainersOptions{})
}

// AllInspectedContainers is a convenience function to fetch a slice of all inspected containers data.
func AllInspectedContainers(endpoint string) ([]*CompleteDockerContainer, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescContainers, err := client.ListContainers(dockerclient.ListContainersOptions{})
	if err != nil {
		return nil, err
	}

	containersChan := make(chan *CompleteDockerContainer)
	var wg sync.WaitGroup

	for _, shortDescContainer := range shortDescContainers {
		container := &CompleteDockerContainer{}
		container.NiceImageName = shortDescContainer.Image
		container.Command = shortDescContainer.Command
		container.Status = shortDescContainer.Status

		wg.Add(1)

		go func(container *CompleteDockerContainer) {
			defer wg.Done()

			fullDescContainer, err := client.InspectContainer(shortDescContainer.ID)
			if err == nil && fullDescContainer != nil {
				container.ID = fullDescContainer.ID
				container.Created = fullDescContainer.Created
				container.Path = fullDescContainer.Path
				container.Args = fullDescContainer.Args
				container.Config = fullDescContainer.Config
				container.State = fullDescContainer.State
				container.Image = fullDescContainer.Image
				container.NetworkSettings = fullDescContainer.NetworkSettings
				container.SysInitPath = fullDescContainer.SysInitPath
				container.ResolvConfPath = fullDescContainer.ResolvConfPath
				container.HostnamePath = fullDescContainer.HostnamePath
				container.HostsPath = fullDescContainer.HostsPath
				container.Name = fullDescContainer.Name
				container.Driver = fullDescContainer.Driver
				container.Volumes = fullDescContainer.Volumes
				container.VolumesRW = fullDescContainer.VolumesRW
				container.HostConfig = fullDescContainer.HostConfig

				containersChan <- container
			}
		}(container)
	}

	containers := make([]*CompleteDockerContainer, 0)

	go func() {
		for container := range containersChan {
			containers = append(containers, container)
		}
	}()

	wg.Wait()
	close(containersChan)

	return containers, nil
}

// AllImages is a convenience function to fetch a slice of all images data.
func AllImages(endpoint string) ([]dockerclient.APIImages, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListImages(dockerclient.ListImagesOptions{})
}

// AllInspectedImages is a convenience function to fetch a slice of all inspected images data.
func AllInspectedImages(endpoint string) ([]*CompleteDockerImage, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescImages, err := client.ListImages(dockerclient.ListImagesOptions{})
	if err != nil {
		return nil, err
	}

	imagesChan := make(chan *CompleteDockerImage)
	var wg sync.WaitGroup

	for _, shortDescImage := range shortDescImages {
		img := &CompleteDockerImage{}
		img.ID = shortDescImage.ID
		img.RepoTags = shortDescImage.RepoTags
		img.VirtualSize = shortDescImage.VirtualSize
		img.ParentID = shortDescImage.ParentID

		wg.Add(1)

		go func(img *CompleteDockerImage) {
			defer wg.Done()

			fullDescImage, err := client.InspectImage(img.ID)
			if err == nil && fullDescImage != nil {
				img.Parent = fullDescImage.Parent
				img.Comment = fullDescImage.Comment
				img.Created = fullDescImage.Created
				img.Container = fullDescImage.Container
				img.ContainerConfig = fullDescImage.ContainerConfig
				img.DockerVersion = fullDescImage.DockerVersion
				img.Author = fullDescImage.Author
				img.Config = fullDescImage.Config
				img.Architecture = fullDescImage.Architecture
				img.Size = fullDescImage.Size

				imagesChan <- img
			}
		}(img)
	}

	images := make([]*CompleteDockerImage, 0)

	go func() {
		for image := range imagesChan {
			images = append(images, image)
		}
	}()

	wg.Wait()
	close(imagesChan)

	return images, nil
}
<commit_msg>Simplify libdocker locking.<commit_after>// Package libdocker provides docker related library functions.
package libdocker

import (
	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/resourced/resourced/libstring"
	"os"
	"path"
	"strconv"
	"sync"
)

var connections map[string]*dockerclient.Client
var connectionsLock = &sync.RWMutex{}

type CompleteDockerContainer struct {
	NiceImageName string `json:"NiceImageName,omitempty" yaml:"NiceImageName,omitempty"`
	Command       string `json:"Command,omitempty" yaml:"Command,omitempty"`
	Status        string `json:"Status,omitempty" yaml:"Status,omitempty"`
	dockerclient.Container
}

type CompleteDockerImage struct {
	RepoTags    []string `json:"RepoTags,omitempty" yaml:"RepoTags,omitempty"`
	VirtualSize int64    `json:"VirtualSize,omitempty" yaml:"VirtualSize,omitempty"`
	ParentID    string   `json:"ParentId,omitempty" yaml:"ParentId,omitempty"`
	dockerclient.Image
}

// DockerClient returns dockerclient.Client which handles Docker connection.
func DockerClient(endpoint string) (*dockerclient.Client, error) {
	var conn *dockerclient.Client
	var err error

	if endpoint == "" {
		endpoint = os.Getenv("DOCKER_HOST")
		if endpoint == "" {
			endpoint = "unix:///var/run/docker.sock"
		}
	}

	connectionsLock.Lock()
	if connections == nil {
		connections = make(map[string]*dockerclient.Client)
	}
	connectionsLock.Unlock()

	// Do not create connection if one already exist.
	connectionsLock.RLock()
	existingConnection, ok := connections[endpoint]
	connectionsLock.RUnlock()

	if ok && existingConnection != nil {
		return existingConnection, nil
	}

	dockerCertPath := os.Getenv("DOCKER_CERT_PATH")
	if dockerCertPath != "" {
		cert := path.Join(dockerCertPath, "cert.pem")
		key := path.Join(dockerCertPath, "key.pem")
		ca := path.Join(dockerCertPath, "ca.pem")

		conn, err = dockerclient.NewTLSClient(endpoint, cert, key, ca)
	} else {
		conn, err = dockerclient.NewClient(endpoint)
	}

	if err == nil && conn != nil {
		connectionsLock.Lock()
		connections[endpoint] = conn
		connectionsLock.Unlock()
	}

	return conn, err
}

// InfoAndVersion is a convenience function to fetch info and version data.
func InfoAndVersion(endpoint string) (map[string]interface{}, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	version, err := client.Version()
	if err != nil {
		return nil, err
	}

	info, err := client.Info()
	if err != nil {
		return nil, err
	}

	versionAsMap := version.Map()
	infoAsMap := info.Map()

	data := make(map[string]interface{})

	for key, value := range versionAsMap {
		data[key] = value
	}

	data["Driver"] = make(map[string]interface{})

	for key, value := range infoAsMap {
		if libstring.StringInSlice(key, []string{"NGoroutines", "Containers", "Images", "MemTotal"}) {
			data[key] = info.GetInt64(key)

		} else if key == "NFd" {
			data["NumFileDescriptors"] = info.GetInt64(key)

		} else if key == "NEventsListener" {
			data["NumEventsListeners"] = info.GetInt64(key)

		} else if key == "NCPU" {
			data["NumCPUs"] = info.GetInt64(key)

		} else if libstring.StringInSlice(key, []string{"Debug", "IPv4Forwarding", "MemoryLimit", "SwapLimit"}) {
			data[key] = info.GetBool(key)

		} else if key == "Driver" {
			driverMap := data["Driver"].(map[string]interface{})
			driverMap["Name"] = value

		} else if key == "DriverStatus" {
			tupleSlice := make([][]string, 2)
			info.GetJSON(key, &tupleSlice)

			for _, tuple := range tupleSlice {
				tupleKey := tuple[0]
				tupleValue := tuple[1]

				driverMap := data["Driver"].(map[string]interface{})

				if tupleKey == "Root Dir" {
					driverMap["RootDir"] = tupleValue
				}
				if tupleKey == "Dirs" {
					tupleValueInt64, err := strconv.ParseInt(tupleValue, 10, 64)
					if err == nil {
						driverMap[tupleKey] = tupleValueInt64
					}
				}
			}

		} else if key == "RegistryConfig" {
			registryConfig := make(map[string]interface{})
			err := info.GetJSON(key, &registryConfig)
			if err == nil {
				data[key] = registryConfig
			}

		} else {
			data[key] = value
		}
	}

	return data, nil
}

// AllContainers is a convenience function to fetch a slice of all containers data.
func AllContainers(endpoint string) ([]dockerclient.APIContainers, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListContainers(dockerclient.ListContainersOptions{})
}

// AllInspectedContainers is a convenience function to fetch a slice of all inspected containers data.
func AllInspectedContainers(endpoint string) ([]*CompleteDockerContainer, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescContainers, err := client.ListContainers(dockerclient.ListContainersOptions{})
	if err != nil {
		return nil, err
	}

	containersChan := make(chan *CompleteDockerContainer)
	var wg sync.WaitGroup

	for _, shortDescContainer := range shortDescContainers {
		container := &CompleteDockerContainer{}
		container.NiceImageName = shortDescContainer.Image
		container.Command = shortDescContainer.Command
		container.Status = shortDescContainer.Status

		wg.Add(1)

		go func(container *CompleteDockerContainer) {
			defer wg.Done()

			fullDescContainer, err := client.InspectContainer(shortDescContainer.ID)
			if err == nil && fullDescContainer != nil {
				container.ID = fullDescContainer.ID
				container.Created = fullDescContainer.Created
				container.Path = fullDescContainer.Path
				container.Args = fullDescContainer.Args
				container.Config = fullDescContainer.Config
				container.State = fullDescContainer.State
				container.Image = fullDescContainer.Image
				container.NetworkSettings = fullDescContainer.NetworkSettings
				container.SysInitPath = fullDescContainer.SysInitPath
				container.ResolvConfPath = fullDescContainer.ResolvConfPath
				container.HostnamePath = fullDescContainer.HostnamePath
				container.HostsPath = fullDescContainer.HostsPath
				container.Name = fullDescContainer.Name
				container.Driver = fullDescContainer.Driver
				container.Volumes = fullDescContainer.Volumes
				container.VolumesRW = fullDescContainer.VolumesRW
				container.HostConfig = fullDescContainer.HostConfig

				containersChan <- container
			}
		}(container)
	}

	containers := make([]*CompleteDockerContainer, 0)

	go func() {
		for container := range containersChan {
			containers = append(containers, container)
		}
	}()

	wg.Wait()
	close(containersChan)

	return containers, nil
}

// AllImages is a convenience function to fetch a slice of all images data.
func AllImages(endpoint string) ([]dockerclient.APIImages, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	return client.ListImages(dockerclient.ListImagesOptions{})
}

// AllInspectedImages is a convenience function to fetch a slice of all inspected images data.
func AllInspectedImages(endpoint string) ([]*CompleteDockerImage, error) {
	client, err := DockerClient(endpoint)
	if err != nil {
		return nil, err
	}

	shortDescImages, err := client.ListImages(dockerclient.ListImagesOptions{})
	if err != nil {
		return nil, err
	}

	imagesChan := make(chan *CompleteDockerImage)
	var wg sync.WaitGroup

	for _, shortDescImage := range shortDescImages {
		img := &CompleteDockerImage{}
		img.ID = shortDescImage.ID
		img.RepoTags = shortDescImage.RepoTags
		img.VirtualSize = shortDescImage.VirtualSize
		img.ParentID = shortDescImage.ParentID

		wg.Add(1)

		go func(img *CompleteDockerImage) {
			defer wg.Done()

			fullDescImage, err := client.InspectImage(img.ID)
			if err == nil && fullDescImage != nil {
				img.Parent = fullDescImage.Parent
				img.Comment = fullDescImage.Comment
				img.Created = fullDescImage.Created
				img.Container = fullDescImage.Container
				img.ContainerConfig = fullDescImage.ContainerConfig
				img.DockerVersion = fullDescImage.DockerVersion
				img.Author = fullDescImage.Author
				img.Config = fullDescImage.Config
				img.Architecture = fullDescImage.Architecture
				img.Size = fullDescImage.Size

				imagesChan <- img
			}
		}(img)
	}

	images := make([]*CompleteDockerImage, 0)

	go func() {
		for image := range imagesChan {
			images = append(images, image)
		}
	}()

	wg.Wait()
	close(imagesChan)

	return images, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"../../" // Use github.com/kurrik/twittergo for your code.
	"flag"
	"fmt"
	"github.com/kurrik/oauth1a"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const MINWAIT = time.Duration(10) * time.Second

func LoadCredentials() (client *twittergo.Client, err error) {
	credentials, err := ioutil.ReadFile("CREDENTIALS")
	if err != nil {
		return
	}
	lines := strings.Split(string(credentials), "\n")
	config := &oauth1a.ClientConfig{
		ConsumerKey:    lines[0],
		ConsumerSecret: lines[1],
	}
	user := oauth1a.NewAuthorizedConfig(lines[2], lines[3])
	client = twittergo.NewClient(config, user)
	return
}

type Args struct {
	ScreenName string
	Count      string
}

func parseArgs() *Args {
	a := &Args{}
	flag.StringVar(&a.ScreenName, "screen_name", "twitterapi", "Screen name to look up")
	flag.StringVar(&a.Count, "count", "5", "Number of results / page")
	flag.Parse()
	return a
}

func handleRateLimit(err error) error {
	if rle, ok := err.(twittergo.RateLimitError); ok {
		dur := rle.Reset.Sub(time.Now()) + time.Second
		if dur < MINWAIT {
			// Don't wait less than minwait.
			dur = MINWAIT
		}
		msg := "Rate limited. Reset at %v. Waiting for %v\n"
		fmt.Printf(msg, rle.Reset, dur)
		time.Sleep(dur)
		return nil
	}
	return err
}

func printRateLimit(resp *twittergo.APIResponse) {
	if resp.HasRateLimit() {
		fmt.Printf("Rate limit:           %v\n", resp.RateLimit())
		fmt.Printf("Rate limit remaining: %v\n", resp.RateLimitRemaining())
		fmt.Printf("Rate limit reset:     %v\n", resp.RateLimitReset())
	} else {
		fmt.Printf("Could not parse rate limit from response.\n")
	}
}

func printList(list *twittergo.List) {
	user := list.User()
	fmt.Printf("%v\n", list.Name())
	fmt.Printf("Owner: %v (@%v)\n", user.Name(), user.ScreenName())
	fmt.Printf("Members: %v\n", list.MemberCount())
	fmt.Printf("Subscribers: %v\n\n", list.SubscriberCount())
}

func fetchAndPrintList(client *twittergo.Client, path string, query url.Values) (err error) {
	var (
		req     *http.Request
		resp    *twittergo.APIResponse
		results twittergo.Lists
	)
	for {
		url := fmt.Sprintf("%v?%v", path, query.Encode())
		req, err = http.NewRequest("GET", url, nil)
		req.Header.Set("Accept-Encoding", "gzip, deflate")
		if err != nil {
			err = fmt.Errorf("Could not parse request: %v\n", err)
			return
		}
		resp, err = client.SendRequest(req)
		if err != nil {
			err = fmt.Errorf("Could not send request: %v\n", err)
			return
		}
		results = twittergo.Lists{}
		if err = resp.Parse(&results); err != nil {
			if err = handleRateLimit(err); err != nil {
				err = fmt.Errorf("Problem parsing response: %v\n", err)
				return
			} else {
				continue
			}
		}
		fmt.Printf("\n")
		for i, list := range results {
			fmt.Printf("%v.) ", i+1)
			printList(&list)
		}
		printRateLimit(resp)
		return
	}
	return
}

func fetchAndPrintCursoredList(client *twittergo.Client, path string, query url.Values) (err error) {
	var (
		req     *http.Request
		resp    *twittergo.APIResponse
		results twittergo.CursoredLists
		i       int64
	)
	i = 1
	query.Set("cursor", "-1")
	for {
		url := fmt.Sprintf("%v?%v", path, query.Encode())
		req, err = http.NewRequest("GET", url, nil)
		req.Header.Set("Accept-Encoding", "gzip, deflate")
		if err != nil {
			err = fmt.Errorf("Could not parse request: %v\n", err)
			break
		}
		resp, err = client.SendRequest(req)
		if err != nil {
			err = fmt.Errorf("Could not send request: %v\n", err)
			break
		}
		results = twittergo.CursoredLists{}
		if err = resp.Parse(&results); err != nil {
			if err = handleRateLimit(err); err != nil {
				err = fmt.Errorf("Problem parsing response: %v\n", err)
				break
			} else {
				continue
			}
		}
		fmt.Printf("\n")
		for _, list := range results.Lists() {
			fmt.Printf("%v.) ", i)
			printList(&list)
			i += 1
		}
		printRateLimit(resp)
		if results.NextCursorStr() == "0" {
			break
		}
		query.Set("cursor", results.NextCursorStr())
	}
	return
}

func main() {
	var (
		err    error
		args   *Args
		client *twittergo.Client
	)
	args = parseArgs()
	if client, err = LoadCredentials(); err != nil {
		fmt.Printf("Could not parse CREDENTIALS file: %v\n", err)
		os.Exit(1)
	}
	query := url.Values{}
	query.Set("screen_name", args.ScreenName)

	fmt.Printf("Printing up to 100 lists %v owns or is subscribed to:\n", args.ScreenName)
	fmt.Printf("=========================================================\n")
	if err = fetchAndPrintList(client, "/1.1/lists/list.json", query); err != nil {
		fmt.Println("Error: %v\n", err)
	}
	fmt.Printf("\n\n")

	// Add count for future requests
	query.Set("count", args.Count)

	fmt.Printf("Printing the lists %v is a member of:\n", args.ScreenName)
	fmt.Printf("=========================================================\n")
	if err = fetchAndPrintCursoredList(client, "/1.1/lists/memberships.json", query); err != nil {
		fmt.Println("Error: %v\n", err)
	}
	fmt.Printf("\n\n")

	fmt.Printf("Printing the lists %v is subscribed to:\n", args.ScreenName)
	fmt.Printf("=========================================================\n")
	if err = fetchAndPrintCursoredList(client, "/1.1/lists/subscriptions.json", query); err != nil {
		fmt.Println("Error: %v\n", err)
	}
}
<commit_msg>Add support for lists/ownerships endpoint.<commit_after>// Copyright 2013 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"../../" // Use github.com/kurrik/twittergo for your code.
	"flag"
	"fmt"
	"github.com/kurrik/oauth1a"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const MINWAIT = time.Duration(10) * time.Second

func LoadCredentials() (client *twittergo.Client, err error) {
	credentials, err := ioutil.ReadFile("CREDENTIALS")
	if err != nil {
		return
	}
	lines := strings.Split(string(credentials), "\n")
	config := &oauth1a.ClientConfig{
		ConsumerKey:    lines[0],
		ConsumerSecret: lines[1],
	}
	user := oauth1a.NewAuthorizedConfig(lines[2], lines[3])
	client = twittergo.NewClient(config, user)
	return
}

type Args struct {
	ScreenName string
	Count      string
}

func parseArgs() *Args {
	a := &Args{}
	flag.StringVar(&a.ScreenName, "screen_name", "episod", "Screen name to look up")
	flag.StringVar(&a.Count, "count", "100", "Number of results / page")
	flag.Parse()
	return a
}

func handleRateLimit(err error) error {
	if rle, ok := err.(twittergo.RateLimitError); ok {
		dur := rle.Reset.Sub(time.Now()) + time.Second
		if dur < MINWAIT {
			// Don't wait less than minwait.
			dur = MINWAIT
		}
		msg := "Rate limited. Reset at %v. Waiting for %v\n"
		fmt.Printf(msg, rle.Reset, dur)
		time.Sleep(dur)
		return nil
	}
	return err
}

func printRateLimit(resp *twittergo.APIResponse) {
	if resp.HasRateLimit() {
		fmt.Printf("Rate limit:           %v\n", resp.RateLimit())
		fmt.Printf("Rate limit remaining: %v\n", resp.RateLimitRemaining())
		fmt.Printf("Rate limit reset:     %v\n", resp.RateLimitReset())
	} else {
		fmt.Printf("Could not parse rate limit from response.\n")
	}
}

func printList(list *twittergo.List) {
	user := list.User()
	fmt.Printf("%v\n", list.Name())
	fmt.Printf("Owner: %v (@%v)\n", user.Name(), user.ScreenName())
	fmt.Printf("Members: %v\n", list.MemberCount())
	fmt.Printf("Subscribers: %v\n\n", list.SubscriberCount())
}

func fetchAndPrintList(client *twittergo.Client, path string, query url.Values) (err error) {
	var (
		req     *http.Request
		resp    *twittergo.APIResponse
		results twittergo.Lists
	)
	for {
		url := fmt.Sprintf("%v?%v", path, query.Encode())
		req, err = http.NewRequest("GET", url, nil)
		req.Header.Set("Accept-Encoding", "gzip, deflate")
		if err != nil {
			err = fmt.Errorf("Could not parse request: %v\n", err)
			return
		}
		resp, err = client.SendRequest(req)
		if err != nil {
			err = fmt.Errorf("Could not send request: %v\n", err)
			return
		}
		results = twittergo.Lists{}
		if err = resp.Parse(&results); err != nil {
			if err = handleRateLimit(err); err != nil {
				err = fmt.Errorf("Problem parsing response: %v\n", err)
				return
			} else {
				continue
			}
		}
		fmt.Printf("\n")
		for i, list := range results {
			fmt.Printf("%v.) ", i+1)
			printList(&list)
		}
		printRateLimit(resp)
		return
	}
	return
}

func fetchAndPrintCursoredList(client *twittergo.Client, path string, query url.Values) (err error) {
	var (
		req     *http.Request
		resp    *twittergo.APIResponse
		results twittergo.CursoredLists
		i       int64
	)
	i = 1
	query.Set("cursor", "-1")
	for {
		url := fmt.Sprintf("%v?%v", path, query.Encode())
		req, err = http.NewRequest("GET", url, nil)
		req.Header.Set("Accept-Encoding", "gzip, deflate")
		if err != nil {
			err = fmt.Errorf("Could not parse request: %v\n", err)
			break
		}
		resp, err = client.SendRequest(req)
		if err != nil {
			err = fmt.Errorf("Could not send request: %v\n", err)
			break
		}
		results = twittergo.CursoredLists{}
		if err = resp.Parse(&results); err != nil {
			if err = handleRateLimit(err); err != nil {
				err = fmt.Errorf("Problem parsing response: %v\n", err)
				break
			} else {
				continue
			}
		}
		fmt.Printf("\n")
		for _, list := range results.Lists() {
			fmt.Printf("%v.) ", i)
			printList(&list)
			i += 1
		}
		printRateLimit(resp)
		if results.NextCursorStr() == "0" {
			break
		}
		query.Set("cursor", results.NextCursorStr())
	}
	return
}

func main() {
	var (
		err    error
		args   *Args
		client *twittergo.Client
	)
	args = parseArgs()
	if client, err = LoadCredentials(); err != nil {
		fmt.Printf("Could not parse CREDENTIALS file: %v\n", err)
		os.Exit(1)
	}
	query := url.Values{}
	query.Set("screen_name", args.ScreenName)

	fmt.Printf("Printing up to 100 lists %v owns or is subscribed to:\n", args.ScreenName)
	fmt.Printf("=========================================================\n")
	if err = fetchAndPrintList(client, "/1.1/lists/list.json", query); err != nil {
		fmt.Println("Error: %v\n", err)
	}
	fmt.Printf("\n\n")

	// Add count for future requests
	query.Set("count", args.Count)

	fmt.Printf("Printing the lists %v is a member of:\n", args.ScreenName)
	fmt.Printf("=========================================================\n")
	if err = fetchAndPrintCursoredList(client, "/1.1/lists/memberships.json", query); err != nil {
		fmt.Printf("Error: %v\n", err)
	}
	fmt.Printf("\n\n")

	fmt.Printf("Printing the lists %v is subscribed to:\n", args.ScreenName)
	fmt.Printf("=========================================================\n")
	if err = fetchAndPrintCursoredList(client, "/1.1/lists/subscriptions.json", query); err != nil {
		fmt.Printf("Error: %v\n", err)
	}
	fmt.Printf("\n\n")

	fmt.Printf("Printing the lists %v is owner of:\n", args.ScreenName)
	fmt.Printf("=========================================================\n")
	if err = fetchAndPrintCursoredList(client, "/1.1/lists/ownerships.json", query); err != nil {
		fmt.Printf("Error: %v\n", err)
	}
}
<|endoftext|>


<commit_before>package libgodelbrot

import (
    "math/big"
    "log"
    "fmt"
)

// Object to initialize the godelbrot system
type configurator Info

// InitializeContext examines the description, chooses a renderer, numerical system and palette.
func Configure(req *Request) (*Info, error) {
    c := &configurator{}
    c.UserRequest = *req

    nerr := c.chooseNumerics()

    if nerr != nil {
        return nil, nerr
    }

    rerr := c.chooseRenderStrategy()

    if rerr != nil {
        return nil, rerr
    }

    perr := c.choosePalette()

    if perr != nil {
        return nil, perr
    }

    return (*Info)(c), nil
}

// Initialize the render system
func (c *configurator) chooseRenderStrategy() error {
    req := c.UserRequest
    switch req.Renderer {
    case AutoDetectRenderMode:
        c.chooseFastRenderStrategy()
    case SequenceRenderMode:
        c.useSequenceRenderer()
    case RegionRenderMode:
        c.useRegionRenderer()
    default:
        return fmt.Errorf("Unknown render mode: %v", req.Renderer)
    }

    return nil
}

// Initialize the numerics system
func (c *configurator) chooseNumerics() error {
    desc := c.UserRequest
    perr := c.parseUserCoords()

    if perr != nil {
        return perr
    }

    switch desc.Numerics {
    case AutoDetectNumericsMode:
        c.chooseAccurateNumerics()
    case NativeNumericsMode:
        c.useNative()
        c.Precision = 53
        c.usePrec()
    case BigFloatNumericsMode:
        c.selectUserPrec()
        c.usePrec()
        c.useBig()
    default:
        return fmt.Errorf("Unknown numerics mode:", desc.Numerics)
    }

    return nil
}

func (c *configurator) selectUserPrec() {
    userPrec := c.UserRequest.Precision
    if userPrec > 0 {
        c.Precision = userPrec
    } else {
        c.Precision = c.howManyBits()
    }
}

func (c *configurator) chooseAccurateNumerics() {
    // 53 bits precision is available to 64 bit floats
    const prec64 uint = 53

    c.selectUserPrec()
    c.usePrec()
    if c.Precision > prec64 {
        c.useBig()
    } else {
        c.useNative()
    }
}

func (c *configurator) usePrec() {
    bounds := []*big.Float{
        &c.RealMin,
        &c.RealMax,
        &c.ImagMin,
        &c.ImagMax,
    }

    for _, num := range bounds {
        // I say c.Precision rather than bits because I think these should be equal
        // and if there is a bug, this will certainly break quicker.
        num.SetPrec(c.Precision)
    }
}

func (c *configurator) useNative() {
    c.NumericsStrategy = NativeNumericsMode
}

func (c *configurator) useBig() {
    c.NumericsStrategy = BigFloatNumericsMode
}

func (c *configurator) parseUserCoords() error {
    bigActions := []func(*big.Float){
        func(realMin *big.Float) { c.RealMin = *realMin },
        func(realMax *big.Float) { c.RealMax = *realMax },
        func(imagMin *big.Float) { c.ImagMin = *imagMin },
        func(imagMax *big.Float) { c.ImagMax = *imagMax },
    }

    desc := c.UserRequest
    userInput := []string{
        desc.RealMin,
        desc.RealMax,
        desc.ImagMin,
        desc.ImagMax,
    }

    inputNames := []string{"realMin", "realMax", "imagMin", "imagMax"}

    for i, num := range userInput {
        bigFloat, bigErr := parseBig(num)

        if bigErr != nil {
            return fmt.Errorf("Could not parse %v: %v", inputNames[i], bigErr)
        }

        // Handle parse results
        bigActions[i](bigFloat)
    }

    return nil
}

// Choose an optimal strategy for rendering the image
func (c *configurator) chooseFastRenderStrategy() {
    req := c.UserRequest

    area := req.ImageWidth * req.ImageHeight
    numerics := c.NumericsStrategy

    if numerics == AutoDetectNumericsMode {
        log.Panic("Must choose render strategy after numerics system")
    }

    if area < DefaultTinyImageArea && numerics == NativeNumericsMode {
        // Use `SequenceRenderStrategy' when
        // We have native arithmetic and the image is tiny
        c.useSequenceRenderer()
    } else {
        c.useRegionRenderer()
    }
}

func (c *configurator) useSequenceRenderer() {
    c.RenderStrategy = SequenceRenderMode
}

func (c *configurator) useRegionRenderer() {
    c.RenderStrategy = RegionRenderMode
}

// Sample method to discover how many bits needed
func (c *configurator) howManyBits() uint {
    bounds := []big.Float{
        c.RealMin,
        c.RealMax,
        c.ImagMin,
        c.ImagMax,
    }

    bits := uint(0)
    for _, bnd := range bounds {
        prec := bnd.MinPrec()
        if prec > bits {
            bits = prec
        }
    }

    return bits
}

func (c *configurator) choosePalette() error {
    code := c.UserRequest.PaletteCode
    switch code {
    case "pretty":
        c.PaletteType = Pretty
    case "redscale":
        c.PaletteType = Redscale
    case "grayscale":
        c.PaletteType = Grayscale
    default:
        return fmt.Errorf("Invalid palette code: %v", code)
    }

    return nil
}<commit_msg>Fix #9 with configbrot default of region optimisation to square images<commit_after>package libgodelbrot

import (
    "math/big"
    "log"
    "fmt"
)

// Object to initialize the godelbrot system
type configurator Info

// InitializeContext examines the description, chooses a renderer, numerical system and palette.
func Configure(req *Request) (*Info, error) {
    c := &configurator{}
    c.UserRequest = *req

    nerr := c.chooseNumerics()

    if nerr != nil {
        return nil, nerr
    }

    rerr := c.chooseRenderStrategy()

    if rerr != nil {
        return nil, rerr
    }

    perr := c.choosePalette()

    if perr != nil {
        return nil, perr
    }

    return (*Info)(c), nil
}

// Initialize the render system
func (c *configurator) chooseRenderStrategy() error {
    req := c.UserRequest
    switch req.Renderer {
    case AutoDetectRenderMode:
        c.chooseFastRenderStrategy()
    case SequenceRenderMode:
        c.useSequenceRenderer()
    case RegionRenderMode:
        c.useRegionRenderer()
    default:
        return fmt.Errorf("Unknown render mode: %v", req.Renderer)
    }

    return nil
}

// Initialize the numerics system
func (c *configurator) chooseNumerics() error {
    desc := c.UserRequest
    perr := c.parseUserCoords()

    if perr != nil {
        return perr
    }

    switch desc.Numerics {
    case AutoDetectNumericsMode:
        c.chooseAccurateNumerics()
    case NativeNumericsMode:
        c.useNative()
        c.Precision = 53
        c.usePrec()
    case BigFloatNumericsMode:
        c.selectUserPrec()
        c.usePrec()
        c.useBig()
    default:
        return fmt.Errorf("Unknown numerics mode:", desc.Numerics)
    }

    return nil
}

func (c *configurator) selectUserPrec() {
    userPrec := c.UserRequest.Precision
    if userPrec > 0 {
        c.Precision = userPrec
    } else {
        c.Precision = c.howManyBits()
    }
}

func (c *configurator) chooseAccurateNumerics() {
    // 53 bits precision is available to 64 bit floats
    const prec64 uint = 53

    c.selectUserPrec()
    c.usePrec()
    if c.Precision > prec64 {
        c.useBig()
    } else {
        c.useNative()
    }
}

func (c *configurator) usePrec() {
    bounds := []*big.Float{
        &c.RealMin,
        &c.RealMax,
        &c.ImagMin,
        &c.ImagMax,
    }

    for _, num := range bounds {
        // I say c.Precision rather than bits because I think these should be equal
        // and if there is a bug, this will certainly break quicker.
        num.SetPrec(c.Precision)
    }
}

func (c *configurator) useNative() {
    c.NumericsStrategy = NativeNumericsMode
}

func (c *configurator) useBig() {
    c.NumericsStrategy = BigFloatNumericsMode
}

func (c *configurator) parseUserCoords() error {
    bigActions := []func(*big.Float){
        func(realMin *big.Float) { c.RealMin = *realMin },
        func(realMax *big.Float) { c.RealMax = *realMax },
        func(imagMin *big.Float) { c.ImagMin = *imagMin },
        func(imagMax *big.Float) { c.ImagMax = *imagMax },
    }

    desc := c.UserRequest
    userInput := []string{
        desc.RealMin,
        desc.RealMax,
        desc.ImagMin,
        desc.ImagMax,
    }

    inputNames := []string{"realMin", "realMax", "imagMin", "imagMax"}

    for i, num := range userInput {
        bigFloat, bigErr := parseBig(num)

        if bigErr != nil {
            return fmt.Errorf("Could not parse %v: %v", inputNames[i], bigErr)
        }

        // Handle parse results
        bigActions[i](bigFloat)
    }

    return nil
}

// Choose an optimal strategy for rendering the image
func (c *configurator) chooseFastRenderStrategy() {
    req := c.UserRequest

    area := req.ImageWidth * req.ImageHeight
    numerics := c.NumericsStrategy

    if numerics == AutoDetectNumericsMode {
        log.Panic("Must choose render strategy after numerics system")
    }

    bigsz := area > DefaultTinyImageArea
    weirdbase := numerics != NativeNumericsMode
    squarepic := req.ImageWidth == req.ImageHeight

    if (bigsz || weirdbase) && squarepic {
        c.useRegionRenderer()
    } else {
        c.useRegionRenderer()
    }
}

func (c *configurator) useSequenceRenderer() {
    c.RenderStrategy = SequenceRenderMode
}

func (c *configurator) useRegionRenderer() {
    c.RenderStrategy = RegionRenderMode
}

// Sample method to discover how many bits needed
func (c *configurator) howManyBits() uint {
    bounds := []big.Float{
        c.RealMin,
        c.RealMax,
        c.ImagMin,
        c.ImagMax,
    }

    bits := uint(0)
    for _, bnd := range bounds {
        prec := bnd.MinPrec()
        if prec > bits {
            bits = prec
        }
    }

    return bits
}

func (c *configurator) choosePalette() error {
    code := c.UserRequest.PaletteCode
    switch code {
    case "pretty":
        c.PaletteType = Pretty
    case "redscale":
        c.PaletteType = Redscale
    case "grayscale":
        c.PaletteType = Grayscale
    default:
        return fmt.Errorf("Invalid palette code: %v", code)
    }

    return nil
}<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package exec_test

import (
	"fmt"
	"io/ioutil"

	"k8s.io/utils/exec"
)

func ExampleCmd_StderrPipe() {
	cmd := exec.New().Command("/bin/sh", "-c", "echo 'We can read from stderr via pipe!' >&2")

	stderrPipe, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	stderr := make(chan []byte)
	go func() {
		b, err := ioutil.ReadAll(stderrPipe)
		if err != nil {
			panic(err)
		}
		stderr <- b
	}()

	if err := cmd.Start(); err != nil {
		panic(err)
	}

	received := <-stderr

	if err := cmd.Wait(); err != nil {
		panic(err)
	}

	fmt.Println(string(received))
	// Output: We can read from stderr via pipe!
}
<commit_msg>[exec] Fix godoc Example<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package exec_test

import (
	"fmt"
	"io/ioutil"

	"k8s.io/utils/exec"
)

func ExampleNew_stderrPipe() {
	cmd := exec.New().Command("/bin/sh", "-c", "echo 'We can read from stderr via pipe!' >&2")

	stderrPipe, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	stderr := make(chan []byte)
	go func() {
		b, err := ioutil.ReadAll(stderrPipe)
		if err != nil {
			panic(err)
		}
		stderr <- b
	}()

	if err := cmd.Start(); err != nil {
		panic(err)
	}

	received := <-stderr

	if err := cmd.Wait(); err != nil {
		panic(err)
	}

	fmt.Println(string(received))
	// Output: We can read from stderr via pipe!
}
<|endoftext|>


<commit_before>package expr

import (
	"fmt"
	"math"
	"sort"
	"strings"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/errors"
	"github.com/grafana/metrictank/schema"
)

type FuncAsPercent struct {
	in          GraphiteFunc
	totalFloat  float64
	totalSeries GraphiteFunc
	nodes       []expr
}

func NewAsPercent() GraphiteFunc {
	return &FuncAsPercent{totalFloat: math.NaN()}
}

func (s *FuncAsPercent) Signature() ([]Arg, []Arg) {
	return []Arg{
		ArgSeriesList{val: &s.in},
		ArgIn{
			key: "total",
			opt: true,
			args: []Arg{
				ArgFloat{val: &s.totalFloat},
				ArgSeriesList{val: &s.totalSeries},
			},
		},
		ArgStringsOrInts{val: &s.nodes, opt: true, key: "nodes"},
	}, []Arg{ArgSeriesList{}}
}

func (s *FuncAsPercent) Context(context Context) Context {
	return context
}

func (s *FuncAsPercent) Exec(cache map[Req][]models.Series) ([]models.Series, error) {
	series, err := s.in.Exec(cache)
	if err != nil {
		return nil, err
	}

	var outSeries []models.Series
	var totals []models.Series
	if s.totalSeries != nil {
		totals, err = s.totalSeries.Exec(cache)
		if err != nil {
			return nil, err
		}
	}

	if s.nodes != nil {
		if !math.IsNaN(s.totalFloat) {
			return nil, errors.NewBadRequest("total must be None or a seriesList")
		}
		outSeries, err = s.execWithNodes(series, totals, cache)
	} else {
		if totals != nil && len(totals) != 1 && len(totals) != len(series) {
			return nil, errors.NewBadRequest("asPercent second argument (total) must be missing, a single digit, reference exactly 1 series or reference the same number of series as the first argument")
		}
		outSeries, err = s.execWithoutNodes(series, totals, cache)
	}
	return outSeries, err
}

func (s *FuncAsPercent) execWithNodes(series, totals []models.Series, cache map[Req][]models.Series) ([]models.Series, error) {
	var outSeries []models.Series
	// Set of keys
	keys := make(map[string]struct{})
	// Series grouped by key
	metaSeries := groupSeriesByKey(series, s.nodes, &keys)
	// The totals series for each key
	var totalSeries map[string]models.Series

	// calculate the sum
	if math.IsNaN(s.totalFloat) && totals == nil {
		totalSeries = getTotalSeries(metaSeries, metaSeries, cache)
		// calculate sum of totals series
	} else if totals != nil {
		totalSeriesLists := groupSeriesByKey(totals, s.nodes, &keys)
		totalSeries = getTotalSeries(totalSeriesLists, metaSeries, cache)
	}

	var nones []schema.Point

	for key := range keys {
		// No input series for a corresponding total series
		if _, ok := metaSeries[key]; !ok {
			nonesSerie := totalSeries[key]
			nonesSerie.QueryPatt = fmt.Sprintf("asPercent(MISSING,%s)", totalSeries[key].QueryPatt)
			nonesSerie.Target = fmt.Sprintf("asPercent(MISSING,%s)", totalSeries[key].Target)
			nonesSerie.Tags = map[string]string{"name": nonesSerie.Target}

			if nones == nil {
				nones = pointSlicePool.Get().([]schema.Point)
				for _, p := range totalSeries[key].Datapoints {
					p.Val = math.NaN()
					nones = append(nones, p)
				}
				cache[Req{}] = append(cache[Req{}], nonesSerie)
			}

			nonesSerie.Datapoints = nones
			outSeries = append(outSeries, nonesSerie)
			continue
		}

		for _, serie1 := range metaSeries[key] {
			// No total series for a corresponding input series
			if _, ok := totalSeries[key]; !ok {
				nonesSerie := serie1
				nonesSerie.QueryPatt = fmt.Sprintf("asPercent(%s,MISSING)", serie1.QueryPatt)
				nonesSerie.Target = fmt.Sprintf("asPercent(%s,MISSING)", serie1.Target)
				nonesSerie.Tags = map[string]string{"name": nonesSerie.Target}
				nonesSerie.Meta = serie1.Meta.Copy()

				if nones == nil {
					nones = pointSlicePool.Get().([]schema.Point)
					for _, p := range serie1.Datapoints {
						p.Val = math.NaN()
						nones = append(nones, p)
					}
					cache[Req{}] = append(cache[Req{}], nonesSerie)
				}

				nonesSerie.Datapoints = nones
				outSeries = append(outSeries, nonesSerie)
			} else {
				// key found in both metaSeries and totalSeries
				serie1 = serie1.Copy(pointSlicePool.Get().([]schema.Point))
				serie2 := totalSeries[key]
				serie1.QueryPatt = fmt.Sprintf("asPercent(%s,%s)", serie1.QueryPatt, serie2.QueryPatt)
				serie1.Target = fmt.Sprintf("asPercent(%s,%s)", serie1.Target, serie2.Target)
				serie1.Tags = map[string]string{"name": serie1.Target}
				for i := range serie1.Datapoints {
					serie1.Datapoints[i].Val = computeAsPercent(serie1.Datapoints[i].Val, serie2.Datapoints[i].Val)
				}
				outSeries = append(outSeries, serie1)
				cache[Req{}] = append(cache[Req{}], serie1)
			}

		}
	}
	return outSeries, nil
}

func (s *FuncAsPercent) execWithoutNodes(series, totals []models.Series, cache map[Req][]models.Series) ([]models.Series, error) {
	var outSeries []models.Series
	var totalsSerie models.Series
	if math.IsNaN(s.totalFloat) && totals == nil {
		totalsSerie = sumSeries(series, cache)
		if len(series) == 1 {
			totalsSerie.Target = fmt.Sprintf("sumSeries(%s)", totalsSerie.QueryPatt)
			totalsSerie.QueryPatt = fmt.Sprintf("sumSeries(%s)", totalsSerie.QueryPatt)
			totalsSerie.Tags = map[string]string{"name": totalsSerie.Target}
		}
	} else if totals != nil {
		if len(totals) == 1 {
			totalsSerie = totals[0]
		} else if len(totals) == len(series) {
			// Sorted to match the input series with the total series based on Target.
			// Mimics Graphite's implementation
			sort.Slice(series, func(i, j int) bool {
				return series[i].Target < series[j].Target
			})
			sort.Slice(totals, func(i, j int) bool {
				return totals[i].Target < totals[j].Target
			})
		}
	} else {
		totalsSerie.QueryPatt = fmt.Sprint(s.totalFloat)
		totalsSerie.Target = fmt.Sprint(s.totalFloat)
	}

	for i, serie := range series {
		if len(totals) == len(series) {
			totalsSerie = totals[i]
		}
		serie = serie.Copy(pointSlicePool.Get().([]schema.Point))
		serie.QueryPatt = fmt.Sprintf("asPercent(%s,%s)", serie.QueryPatt, totalsSerie.QueryPatt)
		serie.Target = fmt.Sprintf("asPercent(%s,%s)", serie.Target, totalsSerie.Target)
		serie.Tags = map[string]string{"name": serie.Target}
		for i := range serie.Datapoints {
			var totalVal float64
			if len(totalsSerie.Datapoints) > 0 {
				totalVal = totalsSerie.Datapoints[i].Val
			} else {
				totalVal = s.totalFloat
			}
			serie.Datapoints[i].Val = computeAsPercent(serie.Datapoints[i].Val, totalVal)
		}
		serie.Meta = serie.Meta.Merge(totalsSerie.Meta)
		outSeries = append(outSeries, serie)
		cache[Req{}] = append(cache[Req{}], serie)
	}
	return outSeries, nil
}

func computeAsPercent(in, total float64) float64 {
	if math.IsNaN(in) || math.IsNaN(total) {
		return math.NaN()
	}
	if total == 0 {
		return math.NaN()
	}
	return in / total * 100
}

func groupSeriesByKey(series []models.Series, nodes []expr, keys *map[string]struct{}) map[string][]models.Series {
	keyedSeries := make(map[string][]models.Series)
	for _, serie := range series {
		key := aggKey(serie, nodes)
		if _, ok := keyedSeries[key]; !ok {
			keyedSeries[key] = []models.Series{serie}
			(*keys)[key] = struct{}{}
		} else {
			keyedSeries[key] = append(keyedSeries[key], serie)
		}
	}
	return keyedSeries
}

// Sums each seriesList in map of seriesLists
func getTotalSeries(totalSeriesLists, include map[string][]models.Series, cache map[Req][]models.Series) map[string]models.Series {
	totalSeries := make(map[string]models.Series, len(totalSeriesLists))
	for key := range totalSeriesLists {
		if _, ok := include[key]; ok {
			totalSeries[key] = sumSeries(totalSeriesLists[key], cache)
		} else {
			totalSeries[key] = totalSeriesLists[key][0]
		}

	}
	return totalSeries
}

// sumSeries returns a copy-on-write series that is the sum of the inputs
func sumSeries(series []models.Series, cache map[Req][]models.Series) models.Series {
	if len(series) == 1 {
		return series[0]
	}
	out := pointSlicePool.Get().([]schema.Point)
	crossSeriesSum(series, &out)
	var queryPatts []string
	var meta models.SeriesMeta

Loop:
	for _, v := range series {
		meta = meta.Merge(v.Meta)
		// avoid duplicates
		for _, qp := range queryPatts {
			if qp == v.QueryPatt {
				continue Loop
			}
		}
		queryPatts = append(queryPatts, v.QueryPatt)
	}
	name := fmt.Sprintf("sumSeries(%s)", strings.Join(queryPatts, ","))
	cons, queryCons := summarizeCons(series)
	sum := models.Series{
		Target:       name,
		QueryPatt:    name,
		Datapoints:   out,
		Interval:     series[0].Interval,
		Consolidator: cons,
		QueryCons:    queryCons,
		QueryFrom:    series[0].QueryFrom,
		QueryTo:      series[0].QueryTo,
		Tags:         map[string]string{"name": name},
		Meta:         meta,
	}
	cache[Req{}] = append(cache[Req{}], sum)
	return sum
}
<commit_msg>cleanup/clarify asPercent code<commit_after>package expr

import (
	"fmt"
	"math"
	"sort"
	"strings"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/errors"
	"github.com/grafana/metrictank/schema"
)

type FuncAsPercent struct {
	in          GraphiteFunc
	totalFloat  float64
	totalSeries GraphiteFunc
	nodes       []expr
}

func NewAsPercent() GraphiteFunc {
	return &FuncAsPercent{totalFloat: math.NaN()}
}

func (s *FuncAsPercent) Signature() ([]Arg, []Arg) {
	return []Arg{
		ArgSeriesList{val: &s.in},
		ArgIn{
			key: "total",
			opt: true,
			args: []Arg{
				ArgFloat{val: &s.totalFloat},
				ArgSeriesList{val: &s.totalSeries},
			},
		},
		ArgStringsOrInts{val: &s.nodes, opt: true, key: "nodes"},
	}, []Arg{ArgSeriesList{}}
}

func (s *FuncAsPercent) Context(context Context) Context {
	return context
}

func (s *FuncAsPercent) Exec(cache map[Req][]models.Series) ([]models.Series, error) {
	in, err := s.in.Exec(cache)
	if err != nil {
		return nil, err
	}

	var totals []models.Series
	if s.totalSeries != nil {
		totals, err = s.totalSeries.Exec(cache)
		if err != nil {
			return nil, err
		}
	}

	if s.nodes != nil {
		if !math.IsNaN(s.totalFloat) {
			return nil, errors.NewBadRequest("total must be None or a seriesList")
		}
		return s.execWithNodes(in, totals, cache)
	}

	if totals != nil && len(totals) != 1 && len(totals) != len(in) {
		return nil, errors.NewBadRequest("if nodes specified, asPercent second argument (total) must be missing, a single digit, reference exactly 1 series or reference the same number of series as the first argument")
	}
	return s.execWithoutNodes(in, totals, cache)
}

// when nodes are given, totals can be:
// * nil -> in which case we divide by the sum of all input series in the group
// * serieslist -> we will sum the series in the group (or not, if we know that the group won't exist in `in` anyway, we don't need to do this work)
// * NOT a number in this case.
func (s *FuncAsPercent) execWithNodes(in, totals []models.Series, cache map[Req][]models.Series) ([]models.Series, error) {
	var outSeries []models.Series

	keys := make(map[string]struct{}) // will track all aggKeys seen, amongst inputs and totals series
	inByKey := groupSeriesByKey(in, s.nodes, keys)
	var totalSerieByKey map[string]models.Series

	// calculate the sum

	if math.IsNaN(s.totalFloat) && totals == nil {
		totalSerieByKey = getTotalSeries(inByKey, inByKey, cache)
	} else if totals != nil {
		totalSeriesByKey := groupSeriesByKey(totals, s.nodes, keys)
		totalSerieByKey = getTotalSeries(totalSeriesByKey, inByKey, cache)
	}

	var nones []schema.Point

	for key := range keys {
		// No input series for a corresponding total series
		if _, ok := inByKey[key]; !ok {
			nonesSerie := totalSerieByKey[key]
			nonesSerie.QueryPatt = fmt.Sprintf("asPercent(MISSING,%s)", totalSerieByKey[key].QueryPatt)
			nonesSerie.Target = fmt.Sprintf("asPercent(MISSING,%s)", totalSerieByKey[key].Target)
			nonesSerie.Tags = map[string]string{"name": nonesSerie.Target}

			if nones == nil {
				nones = pointSlicePool.Get().([]schema.Point)
				for _, p := range totalSerieByKey[key].Datapoints {
					p.Val = math.NaN()
					nones = append(nones, p)
				}
				cache[Req{}] = append(cache[Req{}], nonesSerie)
			}

			nonesSerie.Datapoints = nones
			outSeries = append(outSeries, nonesSerie)
			continue
		}

		for _, serie1 := range inByKey[key] {
			// No total series for a corresponding input series
			if _, ok := totalSerieByKey[key]; !ok {
				nonesSerie := serie1
				nonesSerie.QueryPatt = fmt.Sprintf("asPercent(%s,MISSING)", serie1.QueryPatt)
				nonesSerie.Target = fmt.Sprintf("asPercent(%s,MISSING)", serie1.Target)
				nonesSerie.Tags = map[string]string{"name": nonesSerie.Target}
				nonesSerie.Meta = serie1.Meta.Copy()

				if nones == nil {
					nones = pointSlicePool.Get().([]schema.Point)
					for _, p := range serie1.Datapoints {
						p.Val = math.NaN()
						nones = append(nones, p)
					}
					cache[Req{}] = append(cache[Req{}], nonesSerie)
				}

				nonesSerie.Datapoints = nones
				outSeries = append(outSeries, nonesSerie)
			} else {
				// key found in both inByKey and totalSerieByKey
				serie1 = serie1.Copy(pointSlicePool.Get().([]schema.Point))
				serie2 := totalSerieByKey[key]
				serie1.QueryPatt = fmt.Sprintf("asPercent(%s,%s)", serie1.QueryPatt, serie2.QueryPatt)
				serie1.Target = fmt.Sprintf("asPercent(%s,%s)", serie1.Target, serie2.Target)
				serie1.Tags = map[string]string{"name": serie1.Target}
				for i := range serie1.Datapoints {
					serie1.Datapoints[i].Val = computeAsPercent(serie1.Datapoints[i].Val, serie2.Datapoints[i].Val)
				}
				outSeries = append(outSeries, serie1)
				cache[Req{}] = append(cache[Req{}], serie1)
			}

		}
	}
	return outSeries, nil
}

// execWithoutNodes returns the asPercent output series for each input series.
// the total (divisor) we use for each input series is based on the totals parameter, which cane be:
// * a number
// * a single series -> used as divisor consistently
// * multiple series -> must match len(series) and matched up in pairs
// * nil -> generate total by summing the inputs
func (s *FuncAsPercent) execWithoutNodes(in, totals []models.Series, cache map[Req][]models.Series) ([]models.Series, error) {
	var outSeries []models.Series
	var totalsSerie models.Series
	if math.IsNaN(s.totalFloat) && totals == nil {
		totalsSerie = sumSeries(in, cache)
		if len(in) == 1 {
			totalsSerie.Target = fmt.Sprintf("sumSeries(%s)", totalsSerie.QueryPatt)
			totalsSerie.QueryPatt = fmt.Sprintf("sumSeries(%s)", totalsSerie.QueryPatt)
			totalsSerie.Tags = map[string]string{"name": totalsSerie.Target}
		}
	} else if totals != nil {
		if len(totals) == 1 {
			totalsSerie = totals[0]
		} else if len(totals) == len(in) {
			// Sorted to match the input series with the total series based on Target.
			// Mimics Graphite's implementation
			sort.Slice(in, func(i, j int) bool {
				return in[i].Target < in[j].Target
			})
			sort.Slice(totals, func(i, j int) bool {
				return totals[i].Target < totals[j].Target
			})
		}
	} else {
		totalsSerie.QueryPatt = fmt.Sprint(s.totalFloat)
		totalsSerie.Target = fmt.Sprint(s.totalFloat)
	}

	for i, serie := range in {
		if len(totals) == len(in) {
			totalsSerie = totals[i]
		}
		serie = serie.Copy(pointSlicePool.Get().([]schema.Point))
		serie.QueryPatt = fmt.Sprintf("asPercent(%s,%s)", serie.QueryPatt, totalsSerie.QueryPatt)
		serie.Target = fmt.Sprintf("asPercent(%s,%s)", serie.Target, totalsSerie.Target)
		serie.Tags = map[string]string{"name": serie.Target}
		for i := range serie.Datapoints {
			var totalVal float64
			if len(totalsSerie.Datapoints) > 0 {
				totalVal = totalsSerie.Datapoints[i].Val
			} else {
				totalVal = s.totalFloat
			}
			serie.Datapoints[i].Val = computeAsPercent(serie.Datapoints[i].Val, totalVal)
		}
		serie.Meta = serie.Meta.Merge(totalsSerie.Meta)
		outSeries = append(outSeries, serie)
		cache[Req{}] = append(cache[Req{}], serie)
	}
	return outSeries, nil
}

func computeAsPercent(in, total float64) float64 {
	if math.IsNaN(in) || math.IsNaN(total) {
		return math.NaN()
	}
	if total == 0 {
		return math.NaN()
	}
	return in / total * 100
}

// groupSeriesByKey groups series by their aggkey which is derived from nodes,
// and adds all seen keys to the pre-existing keys map
func groupSeriesByKey(in []models.Series, nodes []expr, keys map[string]struct{}) map[string][]models.Series {
	inByKey := make(map[string][]models.Series)
	for _, serie := range in {
		key := aggKey(serie, nodes)
		if _, ok := inByKey[key]; !ok {
			inByKey[key] = []models.Series{serie}
			keys[key] = struct{}{}
		} else {
			inByKey[key] = append(inByKey[key], serie)
		}
	}
	return inByKey
}

// getTotalSeries constructs a map with one total serie by key.
// if there is a value for the key in "inByKey", we sum the entries in totalSeriesByKey under that key,
// otherwise we do an optimization: we know that the datapoints for that key won't actually be used,
// in that case we only need to return a series that has the proper fields set like QueryPattern etc.
// note: inByKey is only used for its keys, the values (series slices) are not used.
func getTotalSeries(totalSeriesByKey, inByKey map[string][]models.Series, cache map[Req][]models.Series) map[string]models.Series {
	totalSerieByKey := make(map[string]models.Series, len(totalSeriesByKey))
	for key := range totalSeriesByKey {
		if _, ok := inByKey[key]; ok {
			totalSerieByKey[key] = sumSeries(totalSeriesByKey[key], cache)
		} else {
			totalSerieByKey[key] = totalSeriesByKey[key][0]
		}
	}
	return totalSerieByKey
}

// sumSeries returns a copy-on-write series that is the sum of the inputs
func sumSeries(in []models.Series, cache map[Req][]models.Series) models.Series {
	if len(in) == 1 {
		return in[0]
	}
	out := pointSlicePool.Get().([]schema.Point)
	crossSeriesSum(in, &out)
	var queryPatts []string
	var meta models.SeriesMeta

Loop:
	for _, v := range in {
		meta = meta.Merge(v.Meta)
		// avoid duplicates
		for _, qp := range queryPatts {
			if qp == v.QueryPatt {
				continue Loop
			}
		}
		queryPatts = append(queryPatts, v.QueryPatt)
	}
	name := fmt.Sprintf("sumSeries(%s)", strings.Join(queryPatts, ","))
	cons, queryCons := summarizeCons(in)
	sum := models.Series{
		Target:       name,
		QueryPatt:    name,
		Datapoints:   out,
		Interval:     in[0].Interval,
		Consolidator: cons,
		QueryCons:    queryCons,
		QueryFrom:    in[0].QueryFrom,
		QueryTo:      in[0].QueryTo,
		Tags:         map[string]string{"name": name},
		Meta:         meta,
	}
	cache[Req{}] = append(cache[Req{}], sum)
	return sum
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"
	"regexp"
	"strconv"
)
const VERSION string = "1.0.1"
const SUCCESS_EXIT int = 0
const ERROR_EXIT int = 1
const LIMIT_WAIT_COUNT int = 540 // 20sec * 540 = 3 hours
const API_SERVER string = "https://api.vaddy.net"

type CrawlSearch struct {
	Total int `json:"total"`
	Items []CrawlSearchItem `json:"items"`
}

type CrawlSearchItem struct {
	CrawlId int `json:"id"`
}

type StartScan struct {
	ScanID string `json:"scan_id"`
}

type ScanResult struct {
	Status        string `json:"status"`
	AlertCount    int    `json:"alert_count"`
	ScanResultUrl string `json:"scan_result_url"`
}

func main() {
	fmt.Println("==== Start VAddy Scan (Version " + VERSION + ")====")

	var auth_key, user, fqdn, crawl string = getApiParamsFromArgsOrEnv()

	if checkNeedToGetCrawlId(crawl) {
		fmt.Println("Start to get crawl ID from keyword: " + crawl)
		crawl = getCrawlId(auth_key, user, fqdn, crawl)
	}

	scan_id := startScan(auth_key, user, fqdn, crawl)

	var wait_count int = 0
	for {
		checkScanResult(auth_key, user, fqdn, scan_id)

		time.Sleep(20 * time.Second) //wait 20 second
		wait_count++
		if wait_count > LIMIT_WAIT_COUNT {
			fmt.Println("Error: time out")
			os.Exit(ERROR_EXIT)
		}
	}
}

func getApiParamsFromArgsOrEnv() (string, string, string, string) {
	var auth_key, user, fqdn, crawl string
	if len(os.Args) < 4 {
		return getArgsFromEnv()
	}

	auth_key = os.Args[1]
	user = os.Args[2]
	fqdn = os.Args[3]
	if len(os.Args) >= 5 {
		crawl = os.Args[4]
	}
	return auth_key, user, fqdn, crawl
}

func getArgsFromEnv() (string, string, string, string) {
	var auth_key, user, fqdn, crawl string
	auth_key, ok1 := os.LookupEnv("VADDY_TOKEN")
	user, ok2 := os.LookupEnv("VADDY_USER")
	fqdn, ok3 := os.LookupEnv("VADDY_HOST")
	crawl, _ = os.LookupEnv("VADDY_CRAWL")

	if !ok1 || !ok2 || !ok3 {
		fmt.Println("Missing arguments or system env.")
		fmt.Println("USAGE: vaddy.go ApiKey UserId FQDN CrawlID/Label(optional)")
		os.Exit(ERROR_EXIT)
	}
	return auth_key, user, fqdn, crawl
}

func startScan(auth_key string, user string, fqdn string, crawl string) string {
	values := url.Values{}
	values.Add("auth_key", auth_key)
	values.Add("user", user)
	values.Add("fqdn", fqdn)
	values.Add("action", "start")
	if len(crawl) > 0 {
		values.Add("crawl_id", crawl)
	}

	api_server := getApiServerName()
	res, err := http.PostForm(api_server + "/v1/scan", values)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
	defer res.Body.Close()
	json_response := getResponseData(res)
	scanId := getScanId(json_response)
	//fmt.Println("scanId: " + scanId)
	return scanId
}

func getScanResult(auth_key string, user string, fqdn string, scan_id string) []byte {
	values := url.Values{}
	values.Add("auth_key", auth_key)
	values.Add("user", user)
	values.Add("fqdn", fqdn)
	values.Add("scan_id", scan_id)

	api_server := getApiServerName()
	res, err := http.Get(api_server + "/v1/scan/result?" + values.Encode())
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
	defer res.Body.Close()

	json_response := getResponseData(res)
	return json_response
}

func checkScanResult(auth_key string, user string, fqdn string, scan_id string) {
	json_response := getScanResult(auth_key, user, fqdn, scan_id)

	var scan_result ScanResult
	convertJsonToStruct(json_response, &scan_result)

	status := scan_result.Status
	switch status {
	case "scanning":
		fmt.Println(scan_result.Status)
	case "canceled":
		fmt.Println(scan_result.Status)
		os.Exit(ERROR_EXIT)
	case "finish":
		//fmt.Println(string(json_response) + "\n")
		fmt.Println("Server: " + fqdn)
		fmt.Println("scanId: " + scan_id)
		fmt.Println("Result URL: " + scan_result.ScanResultUrl)

		if scan_result.AlertCount > 0 {
			fmt.Print("Vulnerabilities: ")
			fmt.Println(scan_result.AlertCount)
			fmt.Println("Warning!!!")
			os.Exit(ERROR_EXIT)
		} else {
			fmt.Println("Scan Success. No vulnerabilities!")
			os.Exit(SUCCESS_EXIT)
		}
	}
}

func getCrawlId(auth_key string, user string, fqdn string, search_label string) string {
	json_response := doCrawlSearch(auth_key, user, fqdn, search_label)
	//fmt.Println(string(json_response))

	var crawl_result CrawlSearch
	convertJsonToStruct(json_response, &crawl_result)
	if crawl_result.Total == 0 {
		fmt.Println("can not find crawl id. using latest crawl id.")
		return ""
	}
	var crawl_id int = crawl_result.Items[0].CrawlId
	fmt.Printf("Found %d results. Using CrawlID: %d \n\n", crawl_result.Total, crawl_id)
	return strconv.Itoa(crawl_id)
}

func doCrawlSearch(auth_key string, user string, fqdn string, search_label string) []byte {
	values := url.Values{}
	values.Add("auth_key", auth_key)
	values.Add("user", user)
	values.Add("fqdn", fqdn)
	values.Add("search_label", search_label)

	api_server := getApiServerName()
	res, err := http.Get(api_server + "/v1/crawl?" + values.Encode())
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
	defer res.Body.Close()

	json_response := getResponseData(res)
	return json_response
}

func getResponseData(resp *http.Response) []byte {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(string(body))
		os.Exit(ERROR_EXIT)
	}
	status_code := resp.StatusCode
	//fmt.Println(status_code)
	if status_code != 200 {
		fmt.Println("Network/Auth error\n" + string(body))
		os.Exit(ERROR_EXIT)
	}
	return []byte(body)
}

func getScanId(jsonByteData []byte) string {
	var scan_result StartScan
	convertJsonToStruct(jsonByteData, &scan_result)
	return scan_result.ScanID
}

func convertJsonToStruct(jsonByteData []byte, structData interface{}) {
	err := json.Unmarshal(jsonByteData, structData)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
}

func getApiServerName() string {
	api_server, ok := os.LookupEnv("VADDY_API_SERVER")
	if ok {
		return api_server
	}
	return API_SERVER
}

func checkNeedToGetCrawlId(str string) bool {
	if len(str) == 0 || str == "" {
		return false
	}
	var regex string = `[^0-9]`
	return regexp.MustCompile(regex).Match([]byte(str))
}

<commit_msg>error exit when there is no scan url<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"time"
)

const VERSION string = "1.0.1"
const SUCCESS_EXIT int = 0
const ERROR_EXIT int = 1
const LIMIT_WAIT_COUNT int = 540 // 20sec * 540 = 3 hours
const API_SERVER string = "https://api.vaddy.net"

type CrawlSearch struct {
	Total int               `json:"total"`
	Items []CrawlSearchItem `json:"items"`
}

type CrawlSearchItem struct {
	CrawlId int `json:"id"`
}

type StartScan struct {
	ScanID string `json:"scan_id"`
}

type ScanResult struct {
	Status        string `json:"status"`
	AlertCount    int    `json:"alert_count"`
	ScanCount     int    `json:"scan_count"`
	ScanResultUrl string `json:"scan_result_url"`
}

func main() {
	fmt.Println("==== Start VAddy Scan (Version " + VERSION + ")====")

	var auth_key, user, fqdn, crawl string = getApiParamsFromArgsOrEnv()

	if checkNeedToGetCrawlId(crawl) {
		fmt.Println("Start to get crawl ID from keyword: " + crawl)
		crawl = getCrawlId(auth_key, user, fqdn, crawl)
	}

	scan_id := startScan(auth_key, user, fqdn, crawl)

	var wait_count int = 0
	for {
		checkScanResult(auth_key, user, fqdn, scan_id)

		time.Sleep(20 * time.Second) //wait 20 second
		wait_count++
		if wait_count > LIMIT_WAIT_COUNT {
			fmt.Println("Error: time out")
			os.Exit(ERROR_EXIT)
		}
	}
}

func getApiParamsFromArgsOrEnv() (string, string, string, string) {
	var auth_key, user, fqdn, crawl string
	if len(os.Args) < 4 {
		return getArgsFromEnv()
	}

	auth_key = os.Args[1]
	user = os.Args[2]
	fqdn = os.Args[3]
	if len(os.Args) >= 5 {
		crawl = os.Args[4]
	}
	return auth_key, user, fqdn, crawl
}

func getArgsFromEnv() (string, string, string, string) {
	var auth_key, user, fqdn, crawl string
	auth_key, ok1 := os.LookupEnv("VADDY_TOKEN")
	user, ok2 := os.LookupEnv("VADDY_USER")
	fqdn, ok3 := os.LookupEnv("VADDY_HOST")
	crawl, _ = os.LookupEnv("VADDY_CRAWL")

	if !ok1 || !ok2 || !ok3 {
		fmt.Println("Missing arguments or system env.")
		fmt.Println("USAGE: vaddy.go ApiKey UserId FQDN CrawlID/Label(optional)")
		os.Exit(ERROR_EXIT)
	}
	return auth_key, user, fqdn, crawl
}

func startScan(auth_key string, user string, fqdn string, crawl string) string {
	values := url.Values{}
	values.Add("auth_key", auth_key)
	values.Add("user", user)
	values.Add("fqdn", fqdn)
	values.Add("action", "start")
	if len(crawl) > 0 {
		values.Add("crawl_id", crawl)
	}

	api_server := getApiServerName()
	res, err := http.PostForm(api_server+"/v1/scan", values)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
	defer res.Body.Close()
	json_response := getResponseData(res)
	scanId := getScanId(json_response)
	//fmt.Println("scanId: " + scanId)
	return scanId
}

func getScanResult(auth_key string, user string, fqdn string, scan_id string) []byte {
	values := url.Values{}
	values.Add("auth_key", auth_key)
	values.Add("user", user)
	values.Add("fqdn", fqdn)
	values.Add("scan_id", scan_id)

	api_server := getApiServerName()
	res, err := http.Get(api_server + "/v1/scan/result?" + values.Encode())
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
	defer res.Body.Close()

	json_response := getResponseData(res)
	return json_response
}

func checkScanResult(auth_key string, user string, fqdn string, scan_id string) {
	json_response := getScanResult(auth_key, user, fqdn, scan_id)

	var scan_result ScanResult
	convertJsonToStruct(json_response, &scan_result)

	status := scan_result.Status
	switch status {
	case "scanning":
		fmt.Println(scan_result.Status)
	case "canceled":
		fmt.Println(scan_result.Status)
		os.Exit(ERROR_EXIT)
	case "finish":
		//fmt.Println(string(json_response) + "\n")
		fmt.Println("Server: " + fqdn)
		fmt.Println("scanId: " + scan_id)
		fmt.Println("Result URL: " + scan_result.ScanResultUrl)

		if scan_result.AlertCount > 0 {
			fmt.Print("Vulnerabilities: ")
			fmt.Println(scan_result.AlertCount)
			fmt.Println("Warning!!!")
			os.Exit(ERROR_EXIT)
		} else if scan_result.ScanCount == 0 {
			fmt.Println("ERROR: VAddy was not able to scan your sever. Check the result on the Result URL.")
			os.Exit(ERROR_EXIT)
		} else {
			fmt.Println("Scan Success. No vulnerabilities!")
			os.Exit(SUCCESS_EXIT)
		}
	}
}

func getCrawlId(auth_key string, user string, fqdn string, search_label string) string {
	json_response := doCrawlSearch(auth_key, user, fqdn, search_label)
	//fmt.Println(string(json_response))

	var crawl_result CrawlSearch
	convertJsonToStruct(json_response, &crawl_result)
	if crawl_result.Total == 0 {
		fmt.Println("can not find crawl id. using latest crawl id.")
		return ""
	}
	var crawl_id int = crawl_result.Items[0].CrawlId
	fmt.Printf("Found %d results. Using CrawlID: %d \n\n", crawl_result.Total, crawl_id)
	return strconv.Itoa(crawl_id)
}

func doCrawlSearch(auth_key string, user string, fqdn string, search_label string) []byte {
	values := url.Values{}
	values.Add("auth_key", auth_key)
	values.Add("user", user)
	values.Add("fqdn", fqdn)
	values.Add("search_label", search_label)

	api_server := getApiServerName()
	res, err := http.Get(api_server + "/v1/crawl?" + values.Encode())
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
	defer res.Body.Close()

	json_response := getResponseData(res)
	return json_response
}

func getResponseData(resp *http.Response) []byte {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(string(body))
		os.Exit(ERROR_EXIT)
	}
	status_code := resp.StatusCode
	//fmt.Println(status_code)
	if status_code != 200 {
		fmt.Println("Network/Auth error\n" + string(body))
		os.Exit(ERROR_EXIT)
	}
	return []byte(body)
}

func getScanId(jsonByteData []byte) string {
	var scan_result StartScan
	convertJsonToStruct(jsonByteData, &scan_result)
	return scan_result.ScanID
}

func convertJsonToStruct(jsonByteData []byte, structData interface{}) {
	err := json.Unmarshal(jsonByteData, structData)
	if err != nil {
		fmt.Println(err)
		os.Exit(ERROR_EXIT)
	}
}

func getApiServerName() string {
	api_server, ok := os.LookupEnv("VADDY_API_SERVER")
	if ok {
		return api_server
	}
	return API_SERVER
}

func checkNeedToGetCrawlId(str string) bool {
	if len(str) == 0 || str == "" {
		return false
	}
	var regex string = `[^0-9]`
	return regexp.MustCompile(regex).Match([]byte(str))
}
<|endoftext|>