<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "STREAM" {
          log.Println("send", nntp.MessageID(), "to", feed.name)
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "READER" {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, c, "stream")
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<commit_msg>correct order of param<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "STREAM" {
          log.Println("send", nntp.MessageID(), "to", feed.name)
          feed.check <- nntp.MessageID()
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) && feed.mode == "READER" {
          log.Println("asking", feed.name, "for", nntp.MessageID())
          feed.article <- nntp.MessageID()
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, "stream", c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<|endoftext|>


<commit_before>package concurrency_sqlite_test

import (
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	_ "github.com/mattn/go-sqlite3"

	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	common "github.com/maximilien/bosh-softlayer-cpi/common"
	bslcvmpool "github.com/maximilien/bosh-softlayer-cpi/softlayer/vm/pool"
	testhelpers "github.com/maximilien/softlayer-go/test_helpers"
)

var (
	SQLITE_DB_FOLDER    = "/tmp/concurrency_sqlite_test"
	SQLITE_DB_FILE      = "vm_pool.sqlite"
	SQLITE_DB_FILE_PATH = filepath.Join(SQLITE_DB_FOLDER, SQLITE_DB_FILE)
	stemcellUuid        = "fake_stemcell_uuid"
	domain              = "softlayer.com"
	c                   = make(chan string, 0)
	logger              = boshlog.NewLogger(boshlog.LevelInfo)
)

var _ = Describe("Concurrency test for Sqlite DB", func() {
	var (
		err error
	)

	BeforeEach(func() {
		common.SetOSEnvVariable("OS_RELOAD_ENABLED", "TRUE")
		common.SetOSEnvVariable("SQLITE_DB_FOLDER", SQLITE_DB_FOLDER)
		common.SetOSEnvVariable("SQLITE_DB_FILE", SQLITE_DB_FILE)

		testhelpers.TIMEOUT = 35 * time.Minute
		testhelpers.POLLING_INTERVAL = 10 * time.Second

		err = os.RemoveAll(SQLITE_DB_FOLDER)
		Expect(err).ToNot(HaveOccurred())

		err = bslcvmpool.InitVMPoolDB(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL, logger)
		Expect(err).ToNot(HaveOccurred())

		populateDB()

	})

	AfterEach(func() {
		err = os.RemoveAll(SQLITE_DB_FOLDER)
		Expect(err).ToNot(HaveOccurred())

	})

	Context("Manipulate DB concurrently", func() {

		It("Manipulate DB concurrently", func() {
			runtime.GOMAXPROCS(1)

			for i := 100; i <= 500; i += 100 {
				go insertVMInfo(i)
				go updateVMInfoByID()
				go queryVMInfobyID()
				go queryVMInfobyAgentID()
			}

			time.Sleep(240 * time.Second)

		})
	})

})

func populateDB() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := i
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.InsertVMInfo(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())
	}
}

func insertVMInfo(init int) {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := init; i < init+100; i++ {
		vmID := i
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.InsertVMInfo(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}

func updateVMInfoByID() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := rand.Intn(100)
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "f", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.UpdateVMInfoByID(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}

func queryVMInfobyID() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := rand.Intn(100)
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.QueryVMInfobyID(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}

func queryVMInfobyAgentID() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := rand.Intn(100)
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.QueryVMInfobyAgentID(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}
<commit_msg>increase time out<commit_after>package concurrency_sqlite_test

import (
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	_ "github.com/mattn/go-sqlite3"

	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	common "github.com/maximilien/bosh-softlayer-cpi/common"
	bslcvmpool "github.com/maximilien/bosh-softlayer-cpi/softlayer/vm/pool"
	testhelpers "github.com/maximilien/softlayer-go/test_helpers"
)

var (
	SQLITE_DB_FOLDER    = "/tmp/concurrency_sqlite_test"
	SQLITE_DB_FILE      = "vm_pool.sqlite"
	SQLITE_DB_FILE_PATH = filepath.Join(SQLITE_DB_FOLDER, SQLITE_DB_FILE)
	stemcellUuid        = "fake_stemcell_uuid"
	domain              = "softlayer.com"
	c                   = make(chan string, 0)
	logger              = boshlog.NewLogger(boshlog.LevelInfo)
)

var _ = Describe("Concurrency test for Sqlite DB", func() {
	var (
		err error
	)

	BeforeEach(func() {
		common.SetOSEnvVariable("OS_RELOAD_ENABLED", "TRUE")
		common.SetOSEnvVariable("SQLITE_DB_FOLDER", SQLITE_DB_FOLDER)
		common.SetOSEnvVariable("SQLITE_DB_FILE", SQLITE_DB_FILE)

		testhelpers.TIMEOUT = 35 * time.Minute
		testhelpers.POLLING_INTERVAL = 10 * time.Second

		err = os.RemoveAll(SQLITE_DB_FOLDER)
		Expect(err).ToNot(HaveOccurred())

		err = bslcvmpool.InitVMPoolDB(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL, logger)
		Expect(err).ToNot(HaveOccurred())

		populateDB()

	})

	AfterEach(func() {
		err = os.RemoveAll(SQLITE_DB_FOLDER)
		Expect(err).ToNot(HaveOccurred())

	})

	Context("Manipulate DB concurrently", func() {

		It("Manipulate DB concurrently", func() {
			runtime.GOMAXPROCS(1)

			for i := 100; i <= 500; i += 100 {
				go insertVMInfo(i)
				go updateVMInfoByID()
				go queryVMInfobyID()
				go queryVMInfobyAgentID()
			}

			time.Sleep(480 * time.Second)

		})
	})

})

func populateDB() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := i
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.InsertVMInfo(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())
	}
}

func insertVMInfo(init int) {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := init; i < init+100; i++ {
		vmID := i
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.InsertVMInfo(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}

func updateVMInfoByID() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := rand.Intn(100)
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "f", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.UpdateVMInfoByID(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}

func queryVMInfobyID() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := rand.Intn(100)
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.QueryVMInfobyID(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}

func queryVMInfobyAgentID() {
	db, err := bslcvmpool.OpenDB(SQLITE_DB_FILE_PATH)
	Expect(err).ToNot(HaveOccurred())

	for i := 0; i < 100; i++ {
		vmID := rand.Intn(100)
		hostname := fmt.Sprintf("concurrency_test_%d", vmID)
		agentID := strconv.Itoa(i)
		vmInfoDB := bslcvmpool.NewVMInfoDB(vmID, hostname+"."+domain, "t", stemcellUuid, agentID, logger, db)
		defer vmInfoDB.CloseDB()

		err = vmInfoDB.QueryVMInfobyAgentID(bslcvmpool.DB_RETRY_TIMEOUT, bslcvmpool.DB_RETRY_INTERVAL)
		Expect(err).ToNot(HaveOccurred())

		time.Sleep(time.Duration(rand.Int63n(100)) * time.Millisecond)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
	"sort"
	"strconv"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"
const QUERY_LIMIT = 100

var noLimitFlag bool

func init() {
	flag.BoolVar(&noLimitFlag, "a", false, "Specify to execute the solves query with no limit.")
}

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesPuzzleID, SolvesTotalTime, SolvesUser, PuzzlesTable, PuzzlesID, PuzzlesDifficulty string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
}

type puzzles []puzzle

type byUserRelativeDifficulty struct {
	puzzles
}

func (self puzzles) Len() int {
	return len(self)
}

func (self puzzles) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byUserRelativeDifficulty) Less(i, j int) bool {
	return self.puzzles[i].userRelativeDifficulty < self.puzzles[j].userRelativeDifficulty
}

func (self *userSolvesCollection) addSolve(solve solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

//Whehter or not this should be included in calculation.
//Basically, whether the reltaiveDifficulties will all be valid.
//Normally this returns false if there is only one solve by the user, but could also
//happen when there are multiple solves but (crazily enough) they all have exactly the same solveTime.
//This DOES happen in the production dataset.
func (self *userSolvesCollection) valid() bool {
	return self.max != self.min
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {

	flag.Parse()

	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	var solvesQuery string

	if noLimitFlag {
		fmt.Println("Running without a limit for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s from %s"
	} else {
		fmt.Println("Running with a limit of ", QUERY_LIMIT, " for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s from %s limit " + strconv.Itoa(QUERY_LIMIT)
	}

	res, err := db.Start(solvesQuery, config.SolvesUser, config.SolvesPuzzleID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if !collection.valid() {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	fmt.Println("Skipped ", skippedUsers, " users because they had only had a single unique solve time.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{puzzleID, sum / float32(len(difficulties))}
		index++
	}

	//Sort the puzzles by relative user difficulty
	//We actually don't need the wrapper, since it will modify the underlying slice.
	sort.Sort(byUserRelativeDifficulty{puzzles})
}
<commit_msg>Defined getPuzzleDifficultyRatings, not used yet.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
	"sort"
	"strconv"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"
const QUERY_LIMIT = 100

var noLimitFlag bool

func init() {
	flag.BoolVar(&noLimitFlag, "a", false, "Specify to execute the solves query with no limit.")
}

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesPuzzleID, SolvesTotalTime, SolvesUser, PuzzlesTable, PuzzlesID, PuzzlesDifficulty string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
}

type puzzles []puzzle

type byUserRelativeDifficulty struct {
	puzzles
}

func (self puzzles) Len() int {
	return len(self)
}

func (self puzzles) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byUserRelativeDifficulty) Less(i, j int) bool {
	return self.puzzles[i].userRelativeDifficulty < self.puzzles[j].userRelativeDifficulty
}

func (self *userSolvesCollection) addSolve(solve solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

//Whehter or not this should be included in calculation.
//Basically, whether the reltaiveDifficulties will all be valid.
//Normally this returns false if there is only one solve by the user, but could also
//happen when there are multiple solves but (crazily enough) they all have exactly the same solveTime.
//This DOES happen in the production dataset.
func (self *userSolvesCollection) valid() bool {
	return self.max != self.min
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {

	flag.Parse()

	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	var solvesQuery string

	if noLimitFlag {
		fmt.Println("Running without a limit for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s from %s"
	} else {
		fmt.Println("Running with a limit of ", QUERY_LIMIT, " for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s from %s limit " + strconv.Itoa(QUERY_LIMIT)
	}

	res, err := db.Start(solvesQuery, config.SolvesUser, config.SolvesPuzzleID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if !collection.valid() {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	fmt.Println("Skipped ", skippedUsers, " users because they had only had a single unique solve time.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{puzzleID, sum / float32(len(difficulties))}
		index++
	}

	//Sort the puzzles by relative user difficulty
	//We actually don't need the wrapper, since it will modify the underlying slice.
	sort.Sort(byUserRelativeDifficulty{puzzles})
}

func getPuzzleDifficultyRatings(config *dbConfig, result chan map[int]int) {

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s from %s", config.PuzzlesID, config.PuzzlesDifficulty, config.PuzzlesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	puzzles := make(map[int]int)

	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		puzzles[row.Int(0)] = row.Int(1)
	}

	result <- puzzles

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesTotalTime, SolvesUser string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []*solve
	max    int
	min    int
}

func (self *userSolvesCollection) addSolve(solve *solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {
	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s, %s from %s limit 100", config.SolvesUser, config.SolvesID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(&solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	for _, collection := range solvesByUser {

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	//Now average all of the relative difficulties by puzzle.

}
<commit_msg>Discard users who only solved one unique puzzle.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"

type dbConfig struct {
	Url, Username, Password, DbName, SolvesTable, SolvesID, SolvesTotalTime, SolvesUser string
}

type solve struct {
	puzzleID  int
	totalTime int
}

type userSolvesCollection struct {
	solves []*solve
	max    int
	min    int
}

func (self *userSolvesCollection) addSolve(solve *solve) {
	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {
	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s, %s from %s limit 100", config.SolvesUser, config.SolvesID, config.SolvesTotalTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		userSolves.addSolve(&solve{row.Int(1), row.Int(2)})
		i++
	}

	fmt.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	for _, collection := range solvesByUser {

		if len(collection.solves) < 2 {
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	//Now average all of the relative difficulties by puzzle.

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"os"
	"sort"
	"strconv"
	"strings"
	"github.com/petar/GoDCCP/dccp"
)

type logPipe struct {
	Log  *dccp.LogRecord
	Pipe *emitPipe
}

// htmlize orders a slice of logPipes by time and prints them to standard output with the
// surrounding HTML boilerplate
func htmlize(records []*logPipe, srt bool) {
	if srt {
		sort.Sort(logPipeTimeSort(records))
	}
	fmt.Println(htmlHeader)
	var last int64
	var sec  int64
	var sflag rune = ' '
	for _, r := range records {
		r.Pipe.Time = fmt.Sprintf("%15s %c", dccp.Nstoa(r.Log.Time - last), sflag)
		sflag = ' '
		last = r.Log.Time
		if last / 1e9 > sec {
			sflag = '*'
			sec = last / 1e9
		}
		fmt.Println(htmlizePipe(r.Pipe))
	}
	subSpacer := emitSubPipe{
		State:  "",
		Detail: "",
		Left:   "    ",
		Right:  "    ",
	}
	spacer := &emitPipe{
		Client: subSpacer,
		Pipe:   subSpacer,
		Server: subSpacer,
		Event:  "spacer",
	}
	fmt.Println(htmlizePipe(spacer))
	fmt.Println(htmlFooter)
}

// pipeEmit converts a log record into an HTMLRecord.
// The Time field of the
func pipeEmit(t *dccp.LogRecord) *logPipe {
	var pipe *emitPipe
	switch t.Event {
	case "Write":
		pipe = pipeWrite(t)
	case "Read":
		pipe = pipeRead(t)
	case "Drop":
		pipe = pipeDrop(t)
	case "Idle":
		pipe = pipeIdle(t)
	default:
		pipe = pipeGeneric(t)
	}
	if pipe == nil {
		fmt.Fprintf(os.Stderr, "Dropping: %v\n", t)
		return nil
	}
	pipe.SourceFile = t.SourceFile
	pipe.SourceLine = strconv.Itoa(t.SourceLine)
	pipe.Event = classify(t.Event)
	return &logPipe{ Log: t, Pipe: pipe }
}

func classify(ev string) string {
	var w bytes.Buffer
	for _, b := range ev {
		switch b {
		case '-':
			w.WriteByte('_')
		default:
			w.WriteByte(byte(b))
		}
	}
	return strings.ToLower(string(w.Bytes()))
}

type emitPipe struct {
	Time       string
	SourceFile string
	SourceLine string
	Event      string
	Client     emitSubPipe
	Pipe       emitSubPipe
	Server     emitSubPipe
}

type emitSubPipe struct {
	State       string
	Detail      string
	Left, Right string
}

var (
	htmlHeader =
		`<!doctype html>` +
		`<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->` +
		`<!--[if IE 7]>    <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->` +
		`<!--[if IE 8]>    <html class="no-js lt-ie9" lang="en"> <![endif]-->` +
		`<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->` +
		`<head>` +
			`<meta charset="utf-8">` +
			`<title>DCCP Inspector</title>` +
			`<style>` +
			`table, tr, td { font-family: monospace; }` +
			`td { background: #fdfdfd; }` +
			`td { margin: 0; padding:0px; }` +
			`td.time { width: 100px; text-align: right }` +
			`td.state.client { text-align: right }` +
			`td.state.server { text-align: left }` +
			`td.detail { width: 200px }` +
			`td.left { text-align: right; padding-left: 3px; padding-right: 3px }` +
			`td.right { text-align: left; padding-left: 3px; padding-right: 3px }` +
			`td.file { width: 250px; text-align: right }` +
			`td.sep { width: 10px; text-align: center }` +
			`td.line { width: 30px; text-align: left }` +
			`td.client.nonempty { background: #fff0f0 }` +
			`td.server.nonempty { background: #f0f0ff }` +
			`td.pipe.nonempty { background: #f8e0f8 }` +
			`pre { padding: 0; margin: 0 }` +
			// Event coloring
			`.ev_warn { color: #c00 }` +
			`.client.ev_idle.nonempty { background: #f8e0e0 }` +
			`.server.ev_idle.nonempty { background: #e0e0f8 }` +
			`.ev_event, .ev_idle, .ev_rrtt_h, .ev_rrtt, .ev_wccval, .ev_info { color: #aaa }` +
			`.ev_write, .ev_read, .ev_drop { color: #000 }` +
			`.ev_end { background: #0c0 !important; color: #fff !important }` +
			`.ev_spacer { background: white !important; color: white !important }` +
			`</style>` +
			`<!-- script src="js/libs/modernizr-2.5.3.min.js"></script-->` +
		`</head>` +
		`<body><table cell-spacing="2px">`
	htmlFooter =
		`</table></body></html>`
	emitTmpl = template.Must(template.New("emit").Parse(
		`{{ define "emit" }}` +
			`<tr class="emit">` +
				`{{ $ev := .Event }}` +
				`<td class="time ev_{{ $ev }} "><pre>{{ .Time }}</pre></td>` +
				`{{ with .Client }}` +
					`{{ $n0 := or .State .Left .Right .Detail }}` +
					`{{ $ne := and $n0 "nonempty" }}` +
					`<td class="client state ev_{{ $ev }} {{ $ne }}"><pre>{{ .State }}</pre></td>` +
					`<td class="client left ev_{{ $ev }} {{ $ne }}"><pre>{{ .Left  }}</pre></td>` +
					`<td class="client detail ev_{{ $ev }} {{ $ne }}"><pre>{{ .Detail }}</pre></td>` +
					`<td class="client right ev_{{ $ev }} {{ $ne }}"><pre>{{ .Right }}</pre></td>` +
				`{{ end }}` +
				`{{ with .Pipe }}` +
					`{{ $n0 := or .State .Left .Right .Detail }}` +
					`{{ $ne := and $n0 "nonempty" }}` +
					`<td class="pipe left ev_{{ $ev }} {{ $ne }}"><pre>{{ .Left }}</pre></td>` +
					`<td class="pipe detail ev_{{ $ev }} {{ $ne }}"><pre>{{ .Detail }}</pre></td>` +
					`<td class="pipe right ev_{{ $ev }} {{ $ne }}"><pre>{{ .Right }}</pre></td>` +
				`{{ end }}` +
				`{{ with .Server }}` +
					`{{ $n0 := or .State .Left .Right .Detail }}` +
					`{{ $ne := and $n0 "nonempty" }}` +
					`<td class="server left ev_{{ $ev }} {{ $ne }}"><pre>{{ .Left }}</pre></td>` +
					`<td class="server detail ev_{{ $ev }} {{ $ne }}"><pre>{{ .Detail }}</pre></td>` +
					`<td class="server right ev_{{ $ev }} {{ $ne }}"><pre>{{ .Right }}</pre></td>` +
					`<td class="server state ev_{{ $ev }} {{ $ne }}"><pre>{{ .State }}</pre></td>` +
				`{{ end }}` +
				`<td class="file ev_{{ $ev }} "><pre>{{ .SourceFile }}</pre></td>` +
				`<td class="sep ev_{{ $ev }} "><pre>:</pre></td>` +
				`<td class="line ev_{{ $ev }} "><pre>{{ .SourceLine }}</pre></td>` +
			`</tr>` +
		`{{ end }}`,
	))
)

func htmlizePipe(e *emitPipe) string {
	var w bytes.Buffer
	if err := emitTmpl.Execute(&w, e); err != nil {
		fmt.Fprintf(os.Stderr, "template error (%s)\n", err)
		panic("error htmlizing emit")
	}
	return string(w.Bytes())
}

const htmlPacketWidth = 17

func pipeWrite(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Right:  "<——W",
			},
		}
	case "client":
		return &emitPipe{
			Client: emitSubPipe{
				State : r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Left:   "W——>",
			},
		}
	}
	return nil
}

func pipeRead(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "client":
		return &emitPipe{
			Client: emitSubPipe {
				State:  r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Left:   "R<——",
			},
		}
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Right:  "——>R",
			},
		}
	}
	return nil
}

func pipeIdle(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "client":
		return &emitPipe{
			Client: emitSubPipe {
				State:  r.State,
				Detail: "",
			},
		}
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: "",
			},
		}
	}
	return nil
}

func pipeDrop(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "line":
		switch r.Module {
		case "server":
			return &emitPipe{
				Pipe: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Left:   "D<——",
				},
			}
		case "client":
			return &emitPipe{
				Pipe: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Right:  "——>D",
				},
			}
		}
	case "client":
		switch r.Comment {
		case "Slow app":
			return &emitPipe{
				Client: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Right:  "D<——",
				},
			}
		case "Slow strobe":
			return &emitPipe{
				Client: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Right:  "——>D",
				},
			}
		}
	case "server":
		switch r.Comment {
		case "Slow app":
			return &emitPipe{
				Server: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Left:  "——>D",
				},
			}
		case "Slow strobe":
			return &emitPipe{
				Server: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Left:  "D<——",
				},
			}
		}
	}
	return nil
}

const htmlEventWidth = 30

func sprintPacketEventCommentHTML(r *dccp.LogRecord) string {
	if r.SeqNo == 0 {
		return fmt.Sprintf(" %s ", cut(r.Comment, htmlEventWidth-2))
	}
	return fmt.Sprintf(" %s %06x/%06x ", cut(r.Comment, htmlEventWidth-14-2), r.SeqNo, r.AckNo)
}

func pipeGeneric(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "line":
		return &emitPipe{
			Pipe: emitSubPipe {
				Detail: sprintPacketEventCommentHTML(r),
			},
		}
	case "client":
		return &emitPipe{
			Client: emitSubPipe {
				State:  r.State,
				Detail: sprintPacketEventCommentHTML(r),
			},
		}
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: sprintPacketEventCommentHTML(r),
			},
		}
	}
	return nil
}

// logPipeTimeSort sorts logPipe records by timestamp
type logPipeTimeSort []*logPipe

func (t logPipeTimeSort) Len() int {
	return len(t)
}

func (t logPipeTimeSort) Less(i, j int) bool {
	return t[i].Log.Time < t[j].Log.Time
}

func (t logPipeTimeSort) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
<commit_msg>set font in inspector<commit_after>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"os"
	"sort"
	"strconv"
	"strings"
	"github.com/petar/GoDCCP/dccp"
)

type logPipe struct {
	Log  *dccp.LogRecord
	Pipe *emitPipe
}

// htmlize orders a slice of logPipes by time and prints them to standard output with the
// surrounding HTML boilerplate
func htmlize(records []*logPipe, srt bool) {
	if srt {
		sort.Sort(logPipeTimeSort(records))
	}
	fmt.Println(htmlHeader)
	var last int64
	var sec  int64
	var sflag rune = ' '
	for _, r := range records {
		r.Pipe.Time = fmt.Sprintf("%15s %c", dccp.Nstoa(r.Log.Time - last), sflag)
		sflag = ' '
		last = r.Log.Time
		if last / 1e9 > sec {
			sflag = '*'
			sec = last / 1e9
		}
		fmt.Println(htmlizePipe(r.Pipe))
	}
	subSpacer := emitSubPipe{
		State:  "",
		Detail: "",
		Left:   "    ",
		Right:  "    ",
	}
	spacer := &emitPipe{
		Client: subSpacer,
		Pipe:   subSpacer,
		Server: subSpacer,
		Event:  "spacer",
	}
	fmt.Println(htmlizePipe(spacer))
	fmt.Println(htmlFooter)
}

// pipeEmit converts a log record into an HTMLRecord.
// The Time field of the
func pipeEmit(t *dccp.LogRecord) *logPipe {
	var pipe *emitPipe
	switch t.Event {
	case "Write":
		pipe = pipeWrite(t)
	case "Read":
		pipe = pipeRead(t)
	case "Drop":
		pipe = pipeDrop(t)
	case "Idle":
		pipe = pipeIdle(t)
	default:
		pipe = pipeGeneric(t)
	}
	if pipe == nil {
		fmt.Fprintf(os.Stderr, "Dropping: %v\n", t)
		return nil
	}
	pipe.SourceFile = t.SourceFile
	pipe.SourceLine = strconv.Itoa(t.SourceLine)
	pipe.Event = classify(t.Event)
	return &logPipe{ Log: t, Pipe: pipe }
}

func classify(ev string) string {
	var w bytes.Buffer
	for _, b := range ev {
		switch b {
		case '-':
			w.WriteByte('_')
		default:
			w.WriteByte(byte(b))
		}
	}
	return strings.ToLower(string(w.Bytes()))
}

type emitPipe struct {
	Time       string
	SourceFile string
	SourceLine string
	Event      string
	Client     emitSubPipe
	Pipe       emitSubPipe
	Server     emitSubPipe
}

type emitSubPipe struct {
	State       string
	Detail      string
	Left, Right string
}

var (
	htmlHeader =
		`<!doctype html>` +
		`<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->` +
		`<!--[if IE 7]>    <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->` +
		`<!--[if IE 8]>    <html class="no-js lt-ie9" lang="en"> <![endif]-->` +
		`<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->` +
		`<head>` +
			`<meta charset="utf-8">` +
			`<title>DCCP Inspector</title>` +
			`<style>` +
			`table, tr, td { font-family: 'Droid Sans Mono'; font-size: 12px; }` +
			`td { background: #fdfdfd; }` +
			`td { margin: 0; padding:0px; }` +
			`td.time { width: 100px; text-align: right }` +
			`td.state.client { text-align: right }` +
			`td.state.server { text-align: left }` +
			`td.detail { width: 200px }` +
			`td.left { text-align: right; padding-left: 3px; padding-right: 3px }` +
			`td.right { text-align: left; padding-left: 3px; padding-right: 3px }` +
			`td.file { width: 250px; text-align: right }` +
			`td.sep { width: 10px; text-align: center }` +
			`td.line { width: 30px; text-align: left }` +
			`td.client.nonempty { background: #fff0f0 }` +
			`td.server.nonempty { background: #f0f0ff }` +
			`td.pipe.nonempty { background: #f8e0f8 }` +
			`pre { padding: 0; margin: 0 }` +
			// Event coloring
			`.ev_warn { color: #c00 }` +
			`.client.ev_idle.nonempty { background: #f8e0e0 }` +
			`.server.ev_idle.nonempty { background: #e0e0f8 }` +
			`.ev_event, .ev_idle, .ev_rrtt_h, .ev_rrtt, .ev_wccval, .ev_info { color: #aaa }` +
			`.ev_write, .ev_read, .ev_drop { color: #000 }` +
			`.ev_end { background: #0c0 !important; color: #fff !important }` +
			`.ev_spacer { background: white !important; color: white !important }` +
			`</style>` +
			`<!-- script src="js/libs/modernizr-2.5.3.min.js"></script-->` +
		`</head>` +
		`<body><table cell-spacing="2px">`
	htmlFooter =
		`</table></body></html>`
	emitTmpl = template.Must(template.New("emit").Parse(
		`{{ define "emit" }}` +
			`<tr class="emit">` +
				`{{ $ev := .Event }}` +
				`<td class="time ev_{{ $ev }} "><pre>{{ .Time }}</pre></td>` +
				`{{ with .Client }}` +
					`{{ $n0 := or .State .Left .Right .Detail }}` +
					`{{ $ne := and $n0 "nonempty" }}` +
					`<td class="client state ev_{{ $ev }} {{ $ne }}"><pre>{{ .State }}</pre></td>` +
					`<td class="client left ev_{{ $ev }} {{ $ne }}"><pre>{{ .Left  }}</pre></td>` +
					`<td class="client detail ev_{{ $ev }} {{ $ne }}"><pre>{{ .Detail }}</pre></td>` +
					`<td class="client right ev_{{ $ev }} {{ $ne }}"><pre>{{ .Right }}</pre></td>` +
				`{{ end }}` +
				`{{ with .Pipe }}` +
					`{{ $n0 := or .State .Left .Right .Detail }}` +
					`{{ $ne := and $n0 "nonempty" }}` +
					`<td class="pipe left ev_{{ $ev }} {{ $ne }}"><pre>{{ .Left }}</pre></td>` +
					`<td class="pipe detail ev_{{ $ev }} {{ $ne }}"><pre>{{ .Detail }}</pre></td>` +
					`<td class="pipe right ev_{{ $ev }} {{ $ne }}"><pre>{{ .Right }}</pre></td>` +
				`{{ end }}` +
				`{{ with .Server }}` +
					`{{ $n0 := or .State .Left .Right .Detail }}` +
					`{{ $ne := and $n0 "nonempty" }}` +
					`<td class="server left ev_{{ $ev }} {{ $ne }}"><pre>{{ .Left }}</pre></td>` +
					`<td class="server detail ev_{{ $ev }} {{ $ne }}"><pre>{{ .Detail }}</pre></td>` +
					`<td class="server right ev_{{ $ev }} {{ $ne }}"><pre>{{ .Right }}</pre></td>` +
					`<td class="server state ev_{{ $ev }} {{ $ne }}"><pre>{{ .State }}</pre></td>` +
				`{{ end }}` +
				`<td class="file ev_{{ $ev }} "><pre>{{ .SourceFile }}</pre></td>` +
				`<td class="sep ev_{{ $ev }} "><pre>:</pre></td>` +
				`<td class="line ev_{{ $ev }} "><pre>{{ .SourceLine }}</pre></td>` +
			`</tr>` +
		`{{ end }}`,
	))
)

func htmlizePipe(e *emitPipe) string {
	var w bytes.Buffer
	if err := emitTmpl.Execute(&w, e); err != nil {
		fmt.Fprintf(os.Stderr, "template error (%s)\n", err)
		panic("error htmlizing emit")
	}
	return string(w.Bytes())
}

const htmlPacketWidth = 17

func pipeWrite(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Right:  "<——W",
			},
		}
	case "client":
		return &emitPipe{
			Client: emitSubPipe{
				State : r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Left:   "W——>",
			},
		}
	}
	return nil
}

func pipeRead(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "client":
		return &emitPipe{
			Client: emitSubPipe {
				State:  r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Left:   "R<——",
			},
		}
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: sprintPacketWidth(r, htmlPacketWidth),
				Right:  "——>R",
			},
		}
	}
	return nil
}

func pipeIdle(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "client":
		return &emitPipe{
			Client: emitSubPipe {
				State:  r.State,
				Detail: "",
			},
		}
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: "",
			},
		}
	}
	return nil
}

func pipeDrop(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "line":
		switch r.Module {
		case "server":
			return &emitPipe{
				Pipe: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Left:   "D<——",
				},
			}
		case "client":
			return &emitPipe{
				Pipe: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Right:  "——>D",
				},
			}
		}
	case "client":
		switch r.Comment {
		case "Slow app":
			return &emitPipe{
				Client: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Right:  "D<——",
				},
			}
		case "Slow strobe":
			return &emitPipe{
				Client: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Right:  "——>D",
				},
			}
		}
	case "server":
		switch r.Comment {
		case "Slow app":
			return &emitPipe{
				Server: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Left:  "——>D",
				},
			}
		case "Slow strobe":
			return &emitPipe{
				Server: emitSubPipe{
					Detail: sprintPacketWidth(r, htmlPacketWidth),
					Left:  "D<——",
				},
			}
		}
	}
	return nil
}

const htmlEventWidth = 30

func sprintPacketEventCommentHTML(r *dccp.LogRecord) string {
	if r.SeqNo == 0 {
		return fmt.Sprintf(" %s ", cut(r.Comment, htmlEventWidth-2))
	}
	return fmt.Sprintf(" %s %06x/%06x ", cut(r.Comment, htmlEventWidth-14-2), r.SeqNo, r.AckNo)
}

func pipeGeneric(r *dccp.LogRecord) *emitPipe {
	switch r.System {
	case "line":
		return &emitPipe{
			Pipe: emitSubPipe {
				Detail: sprintPacketEventCommentHTML(r),
			},
		}
	case "client":
		return &emitPipe{
			Client: emitSubPipe {
				State:  r.State,
				Detail: sprintPacketEventCommentHTML(r),
			},
		}
	case "server":
		return &emitPipe{
			Server: emitSubPipe{
				State:  r.State,
				Detail: sprintPacketEventCommentHTML(r),
			},
		}
	}
	return nil
}

// logPipeTimeSort sorts logPipe records by timestamp
type logPipeTimeSort []*logPipe

func (t logPipeTimeSort) Len() int {
	return len(t)
}

func (t logPipeTimeSort) Less(i, j int) bool {
	return t[i].Log.Time < t[j].Log.Time
}

func (t logPipeTimeSort) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fetchlogs downloads build failure logs from the Go dashboard so
// they can be accessed and searched from the local file system.
//
// It organizes these logs into two directories created in the
// directory specified by the -dir flag (which typically defaults to
// ~/.cache/fetchlogs). The log/ directory contains all log files
// named the same way they are named by the dashboard (which happens
// to be the SHA-1 of their contents). The rev/ directory contains
// symlinks back to these logs named
//
//    rev/<ISO 8601 commit date>-<git revision>/<builder>
//
// Fetchlogs will reuse existing log files and revision symlinks, so
// it only has to download logs that are new since the last time it
// was run.
//
// This makes failures easily searchable with standard tools. For
// example, to list the revisions and builders with a particular
// failure, use:
//
//    grep -lR <regexp> rev | sort
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"time"

	"golang.org/x/build/repos"
	"golang.org/x/build/types"
)

var defaultDir = filepath.Join(xdgCacheDir(), "fetchlogs")

var (
	flagN         = flag.Int("n", 300, "limit to most recent `N` commits")
	flagPar       = flag.Int("j", 5, "number of concurrent download `jobs`")
	flagDir       = flag.String("dir", defaultDir, "`directory` to save logs to")
	flagRepo      = flag.String("repo", "go", `repo to fetch logs for`)
	flagDashboard = flag.String("dashboard", "https://build.golang.org", `the dashboard root url`)
)

func main() {
	log.SetPrefix("fetchlogs: ")
	log.SetFlags(0)

	flag.Parse()
	if flag.NArg() != 0 {
		flag.Usage()
		os.Exit(2)
	}

	// If the top-level directory is the default XDG cache
	// directory, make sure it exists.
	if *flagDir == defaultDir {
		if err := xdgCreateDir(*flagDir); err != nil {
			log.Fatal(err)
		}
	}

	// Create directory structure.
	if err := os.Chdir(*flagDir); err != nil {
		log.Fatal(err)
	}
	ensureDir("log")
	ensureDir("rev")

	// Set up fetchers.
	fetcher := newFetcher(*flagPar)
	wg := sync.WaitGroup{}

	// Fetch dashboard pages.
	haveCommits := 0
	for page := 0; haveCommits < *flagN; page++ {
		dashURL := fmt.Sprintf("%s/?mode=json&page=%d", *flagDashboard, page)
		if *flagRepo != "go" {
			repo := repos.ByGerritProject[*flagRepo]
			if repo == nil {
				log.Fatalf("unknown repo %s", *flagRepo)
			}
			dashURL += "&repo=" + url.QueryEscape(repo.ImportPath)
		}
		index, err := fetcher.get(dashURL)
		if err != nil {
			log.Fatal(err)
		}

		var status types.BuildStatus
		if err = json.NewDecoder(index).Decode(&status); err != nil {
			log.Fatal("error unmarshalling result: ", err)
		}
		index.Close()

		for _, rev := range status.Revisions {
			haveCommits++
			if haveCommits > *flagN {
				break
			}
			if rev.Repo != *flagRepo {
				continue
			}

			// Create a revision directory. This way we
			// have a record of commits with no failures.
			date, err := parseRevDate(rev.Date)
			if err != nil {
				log.Fatal("malformed revision date: ", err)
			}
			revDir := revToDir(rev.Revision, date)
			ensureDir(revDir)

			// Save revision metadata.
			buf := bytes.Buffer{}
			enc := json.NewEncoder(&buf)
			if err = enc.Encode(rev); err != nil {
				log.Fatal(err)
			}
			if err = writeFileAtomic(filepath.Join(revDir, ".rev.json"), &buf); err != nil {
				log.Fatal("error saving revision metadata: ", err)
			}

			// Save builders list so Results list can be
			// interpreted.
			if err = enc.Encode(status.Builders); err != nil {
				log.Fatal(err)
			}
			if err = writeFileAtomic(filepath.Join(revDir, ".builders.json"), &buf); err != nil {
				log.Fatal("error saving builders metadata: ", err)
			}

			// Fetch revision logs.
			for i, res := range rev.Results {
				if res == "" || res == "ok" {
					continue
				}

				wg.Add(1)
				go func(rev, builder, logURL string) {
					defer wg.Done()
					logPath := filepath.Join("log", filepath.Base(logURL))
					err := fetcher.getFile(logURL, logPath)
					if err != nil {
						log.Fatal("error fetching log: ", err)
					}
					if err := linkLog(revDir, builder, logPath); err != nil {
						log.Fatal("error linking log: ", err)
					}
				}(revDir, status.Builders[i], res)
			}
		}
	}

	wg.Wait()
}

// A fetcher downloads files over HTTP concurrently. It allows
// limiting the number of concurrent downloads and correctly handles
// multiple (possibly concurrent) fetches from the same URL to the
// same file.
type fetcher struct {
	tokens chan struct{}

	pending struct {
		sync.Mutex
		m map[string]*pendingFetch
	}
}

type pendingFetch struct {
	wchan chan struct{} // closed when fetch completes

	// err is the error, if any, that occurred during this fetch.
	// It will be set before wchan is closed.
	err error
}

func newFetcher(jobs int) *fetcher {
	f := new(fetcher)

	f.tokens = make(chan struct{}, *flagPar)
	for i := 0; i < jobs; i++ {
		f.tokens <- struct{}{}
	}

	f.pending.m = make(map[string]*pendingFetch)

	return f
}

// get performs an HTTP GET for URL and returns the body, while
// obeying the job limit on fetcher.
func (f *fetcher) get(url string) (io.ReadCloser, error) {
	<-f.tokens
	fmt.Println("fetching", url)
	resp, err := http.Get(url)
	f.tokens <- struct{}{}
	if err != nil {
		return nil, err
	}

	return resp.Body, nil
}

// getFile performs an HTTP GET for URL and writes it to filename. If
// the destination file already exists, this returns immediately. If
// another goroutine is currently fetching filename, this blocks until
// the fetch is done and then returns.
func (f *fetcher) getFile(url string, filename string) error {
	// Do we already have it?
	if _, err := os.Stat(filename); err == nil {
		return nil
	} else if !os.IsNotExist(err) {
		return err
	}

	// Check if another fetcher is working on it.
	f.pending.Lock()
	if p, ok := f.pending.m[filename]; ok {
		f.pending.Unlock()
		<-p.wchan
		return p.err
	}

	p := &pendingFetch{wchan: make(chan struct{})}
	f.pending.m[filename] = p
	f.pending.Unlock()

	r, err := f.get(url)
	if err == nil {
		err = writeFileAtomic(filename, r)
		r.Close()
	}
	p.err = err

	close(p.wchan)
	return p.err
}

// ensureDir creates directory name if it does not exist.
func ensureDir(name string) {
	err := os.Mkdir(name, 0777)
	if err != nil && !os.IsExist(err) {
		log.Fatal("error creating directory ", name, ": ", err)
	}
}

// writeFileAtomic atomically creates a file called filename and
// copies the data from r to the file.
func writeFileAtomic(filename string, r io.Reader) error {
	tmpPath := filename + ".tmp"
	if f, err := os.Create(tmpPath); err != nil {
		return err
	} else {
		_, err := io.Copy(f, r)
		if err == nil {
			err = f.Sync()
		}
		err2 := f.Close()
		if err == nil {
			err = err2
		}
		if err != nil {
			os.Remove(tmpPath)
			return err
		}
	}
	if err := os.Rename(tmpPath, filename); err != nil {
		os.Remove(tmpPath)
		return err
	}
	return nil
}

// linkLog creates a symlink for finding logPath based on its git
// revision and builder.
func linkLog(revDir, builder, logPath string) error {
	// Create symlink.
	err := os.Symlink("../../"+logPath, filepath.Join(revDir, builder))
	if err != nil && !os.IsExist(err) {
		return err
	}

	return nil
}

// parseRevDate parses a revision date in RFC3339.
func parseRevDate(date string) (time.Time, error) {
	return time.Parse(time.RFC3339, date)
}

// revToDir returns the path of the revision directory for revision.
func revToDir(revision string, date time.Time) string {
	return filepath.Join("rev", date.Format("2006-01-02T15:04:05")+"-"+revision[:7])
}
<commit_msg>cmd/fetchlogs: link each subrepo Go revision to a unique subdirectory<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fetchlogs downloads build failure logs from the Go dashboard so
// they can be accessed and searched from the local file system.
//
// It organizes these logs into two directories created in the
// directory specified by the -dir flag (which typically defaults to
// ~/.cache/fetchlogs). The log/ directory contains all log files
// named the same way they are named by the dashboard (which happens
// to be the SHA-1 of their contents). The rev/ directory contains
// symlinks back to these logs named
//
//    rev/<ISO 8601 commit date>-<git revision>/<builder>
//
// Fetchlogs will reuse existing log files and revision symlinks, so
// it only has to download logs that are new since the last time it
// was run.
//
// This makes failures easily searchable with standard tools. For
// example, to list the revisions and builders with a particular
// failure, use:
//
//    grep -lR <regexp> rev | sort
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
	"golang.org/x/build/repos"
	"golang.org/x/build/types"
)

var defaultDir = filepath.Join(xdgCacheDir(), "fetchlogs")

var (
	flagN         = flag.Int("n", 300, "limit to most recent `N` commits")
	flagPar       = flag.Int("j", 5, "number of concurrent download `jobs`")
	flagDir       = flag.String("dir", defaultDir, "`directory` to save logs to")
	flagRepo      = flag.String("repo", "go", `repo to fetch logs for`)
	flagDashboard = flag.String("dashboard", "https://build.golang.org", `the dashboard root url`)
)

func main() {
	log.SetPrefix("fetchlogs: ")
	log.SetFlags(0)

	flag.Parse()
	if flag.NArg() != 0 {
		flag.Usage()
		os.Exit(2)
	}

	// If the top-level directory is the default XDG cache
	// directory, make sure it exists.
	if *flagDir == defaultDir {
		if err := xdgCreateDir(*flagDir); err != nil {
			log.Fatal(err)
		}
	}

	// Create directory structure.
	if err := os.Chdir(*flagDir); err != nil {
		log.Fatal(err)
	}
	ensureDir("log")
	ensureDir("rev")

	// Set up fetchers.
	fetcher := newFetcher(*flagPar)
	wg := sync.WaitGroup{}

	// Fetch dashboard pages.
	haveCommits := 0
	for page := 0; haveCommits < *flagN; page++ {
		dashURL := fmt.Sprintf("%s/?mode=json&page=%d", *flagDashboard, page)
		if *flagRepo != "go" {
			repo := repos.ByGerritProject[*flagRepo]
			if repo == nil {
				log.Fatalf("unknown repo %s", *flagRepo)
			}
			dashURL += "&repo=" + url.QueryEscape(repo.ImportPath)
		}
		index, err := fetcher.get(dashURL)
		if err != nil {
			log.Fatal(err)
		}

		var status types.BuildStatus
		if err = json.NewDecoder(index).Decode(&status); err != nil {
			log.Fatal("error unmarshalling result: ", err)
		}
		index.Close()

		for _, rev := range status.Revisions {
			haveCommits++
			if haveCommits > *flagN {
				break
			}
			if rev.Repo != *flagRepo {
				continue
			}

			// Create a revision directory. This way we
			// have a record of commits with no failures.
			date, err := parseRevDate(rev.Date)
			if err != nil {
				log.Fatal("malformed revision date: ", err)
			}
			var goDate time.Time
			if rev.GoRevision != "" {
				commit := goProject().GitCommit(rev.GoRevision)
				goDate = commit.CommitTime
			}
			revDir, revDirDepth := revToDir(rev.Revision, date, rev.GoRevision, goDate)
			ensureDir(revDir)

			if rev.GoRevision != "" {
				// In October 2021 we started creating a separate subdirectory for
				// each Go repo commit. (Previously, we overwrote the link for each
				// subrepo commit when downloading a new Go commit.) Remove the
				// previous links, if any, so that greplogs won't double-count them.
				prevRevDir, _ := revToDir(rev.Revision, date, "", time.Time{})
				if err := os.RemoveAll(prevRevDir); err != nil {
					log.Fatal(err)
				}
			}

			// Save revision metadata.
			buf := bytes.Buffer{}
			enc := json.NewEncoder(&buf)
			if err = enc.Encode(rev); err != nil {
				log.Fatal(err)
			}
			if err = writeFileAtomic(filepath.Join(revDir, ".rev.json"), &buf); err != nil {
				log.Fatal("error saving revision metadata: ", err)
			}

			// Save builders list so Results list can be
			// interpreted.
			if err = enc.Encode(status.Builders); err != nil {
				log.Fatal(err)
			}
			if err = writeFileAtomic(filepath.Join(revDir, ".builders.json"), &buf); err != nil {
				log.Fatal("error saving builders metadata: ", err)
			}

			// Fetch revision logs.
			for i, res := range rev.Results {
				if res == "" || res == "ok" {
					continue
				}

				wg.Add(1)
				go func(builder, logURL string) {
					defer wg.Done()
					logPath := filepath.Join("log", filepath.Base(logURL))
					err := fetcher.getFile(logURL, logPath)
					if err != nil {
						log.Fatal("error fetching log: ", err)
					}
					if err := linkLog(revDir, revDirDepth, builder, logPath); err != nil {
						log.Fatal("error linking log: ", err)
					}
				}(status.Builders[i], res)
			}
		}
	}

	wg.Wait()
}

// A fetcher downloads files over HTTP concurrently. It allows
// limiting the number of concurrent downloads and correctly handles
// multiple (possibly concurrent) fetches from the same URL to the
// same file.
type fetcher struct {
	tokens chan struct{}

	pending struct {
		sync.Mutex
		m map[string]*pendingFetch
	}
}

type pendingFetch struct {
	wchan chan struct{} // closed when fetch completes

	// err is the error, if any, that occurred during this fetch.
	// It will be set before wchan is closed.
	err error
}

func newFetcher(jobs int) *fetcher {
	f := new(fetcher)

	f.tokens = make(chan struct{}, *flagPar)
	for i := 0; i < jobs; i++ {
		f.tokens <- struct{}{}
	}

	f.pending.m = make(map[string]*pendingFetch)

	return f
}

// get performs an HTTP GET for URL and returns the body, while
// obeying the job limit on fetcher.
func (f *fetcher) get(url string) (io.ReadCloser, error) {
	<-f.tokens
	fmt.Println("fetching", url)
	resp, err := http.Get(url)
	f.tokens <- struct{}{}
	if err != nil {
		return nil, err
	}

	return resp.Body, nil
}

// getFile performs an HTTP GET for URL and writes it to filename. If
// the destination file already exists, this returns immediately. If
// another goroutine is currently fetching filename, this blocks until
// the fetch is done and then returns.
func (f *fetcher) getFile(url string, filename string) error {
	// Do we already have it?
	if _, err := os.Stat(filename); err == nil {
		return nil
	} else if !os.IsNotExist(err) {
		return err
	}

	// Check if another fetcher is working on it.
	f.pending.Lock()
	if p, ok := f.pending.m[filename]; ok {
		f.pending.Unlock()
		<-p.wchan
		return p.err
	}

	p := &pendingFetch{wchan: make(chan struct{})}
	f.pending.m[filename] = p
	f.pending.Unlock()

	r, err := f.get(url)
	if err == nil {
		err = writeFileAtomic(filename, r)
		r.Close()
	}
	p.err = err

	close(p.wchan)
	return p.err
}

var (
	goProjectOnce   sync.Once
	cachedGoProject *maintner.GerritProject
	goProjectErr    error
)

func getGoProject(ctx context.Context) (*maintner.GerritProject, error) {
	corpus, err := godata.Get(ctx)
	if err != nil {
		return nil, err
	}

	gp := corpus.Gerrit().Project("go.googlesource.com", "go")
	if gp == nil {
		return nil, fmt.Errorf("go.googlesource.com/go Gerrit project not found")
	}

	return gp, nil
}

func goProject() *maintner.GerritProject {
	goProjectOnce.Do(func() {
		cachedGoProject, goProjectErr = getGoProject(context.Background())
	})
	if goProjectErr != nil {
		log.Fatal(goProjectErr)
	}
	return cachedGoProject
}

// ensureDir creates directory name if it does not exist.
func ensureDir(name string) {
	err := os.MkdirAll(name, 0777)
	if err != nil {
		log.Fatal("error creating directory ", name, ": ", err)
	}
}

// writeFileAtomic atomically creates a file called filename and
// copies the data from r to the file.
func writeFileAtomic(filename string, r io.Reader) error {
	tmpPath := filename + ".tmp"
	if f, err := os.Create(tmpPath); err != nil {
		return err
	} else {
		_, err := io.Copy(f, r)
		if err == nil {
			err = f.Sync()
		}
		err2 := f.Close()
		if err == nil {
			err = err2
		}
		if err != nil {
			os.Remove(tmpPath)
			return err
		}
	}
	if err := os.Rename(tmpPath, filename); err != nil {
		os.Remove(tmpPath)
		return err
	}
	return nil
}

// linkLog creates a symlink for finding logPath based on its git
// revision and builder.
func linkLog(revDir string, revDirDepth int, builder, logPath string) error {
	// Create symlink.
	err := os.Symlink(strings.Repeat("../", revDirDepth)+logPath, filepath.Join(revDir, builder))
	if err != nil && !os.IsExist(err) {
		return err
	}

	return nil
}

// parseRevDate parses a revision date in RFC3339.
func parseRevDate(date string) (time.Time, error) {
	return time.Parse(time.RFC3339, date)
}

// revToDir returns the path of the revision directory for revision.
func revToDir(revision string, date time.Time, goRev string, goDate time.Time) (dir string, depth int) {
	if goDate.After(date) {
		date = goDate
	}
	dateStr := date.Format("2006-01-02T15:04:05")

	parts := []string{dateStr, revision[:7]}
	if goRev != "" {
		parts = append(parts, goRev[:7])
	}

	return filepath.Join("rev", strings.Join(parts, "-")), 2
}
<|endoftext|>


<commit_before>package serve

import (
	"flag"
	"fmt"
	"log"
	"net"
	"sync"

	"context"

	"google.golang.org/grpc"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/pb/tak/proto"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/symmetry"
	"github.com/nelhage/taktician/tak"
)

type Command struct {
	port int
}

func (*Command) Name() string     { return "serve" }
func (*Command) Synopsis() string { return "Seve Taktician RPCs via GRPC" }
func (*Command) Usage() string {
	return `serve
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.IntVar(&c.port, "port", 55430, "bind port")
}

type server struct {
	cache struct {
		sync.Mutex
		player  *ai.MinimaxAI
		cfg     ai.MinimaxConfig
		precise bool
	}
}

func (s *server) Analyze(ctx context.Context, req *pb.AnalyzeRequest) (*pb.AnalyzeResponse, error) {
	p, e := ptn.ParseTPS(req.Position)
	if e != nil {
		return nil, e
	}

	s.cache.Lock()
	defer s.cache.Unlock()

	if s.cache.cfg.Size != p.Size() || s.cache.cfg.Depth != int(req.Depth) || s.cache.precise != req.Precise {
		s.cache.cfg = ai.MinimaxConfig{
			Size:  p.Size(),
			Depth: int(req.Depth),
			Debug: 1,
		}
		if req.Precise {
			s.cache.cfg.MakePrecise()
		}
		s.cache.player = ai.NewMinimax(s.cache.cfg)
		s.cache.precise = req.Precise
	}

	var resp pb.AnalyzeResponse
	pv, value, _ := s.cache.player.Analyze(ctx, p)
	for _, m := range pv {
		resp.Pv = append(resp.Pv, ptn.FormatMove(m))
	}
	resp.Value = value

	return &resp, nil
}

func (s *server) Canonicalize(ctx context.Context, req *pb.CanonicalizeRequest) (*pb.CanonicalizeResponse, error) {
	var ms []tak.Move
	for _, mstr := range req.Moves {
		mv, e := ptn.ParseMove(mstr)
		if e != nil {
			return nil, e
		}
		ms = append(ms, mv)
	}

	ms, e := symmetry.Canonical(int(req.Size), ms)
	if e != nil {
		return nil, e
	}

	var outms []string
	for _, m := range ms {
		outms = append(outms, ptn.FormatMove(m))
	}
	return &pb.CanonicalizeResponse{
		Moves: outms,
	}, nil
}

func (s *server) IsPositionInTak(ctx context.Context, req *pb.IsPositionInTakRequest) (*pb.IsPositionInTakResponse, error) {
	p, e := ptn.ParseTPS(req.Position)
	if e != nil {
		return nil, e
	}

	var cfg = ai.MinimaxConfig{
		Size:  p.Size(),
		Depth: 1,
	}

	cfg.MakePrecise()

	player := ai.NewMinimax(cfg)

	pass, e := p.Move(tak.Move{Type: tak.Pass})
	pv, value, _ := player.Analyze(ctx, pass)

	var resp pb.IsPositionInTakResponse
	resp.InTak = value > ai.WinThreshold
	if resp.InTak {
		resp.TakMove = ptn.FormatMove(pv[0])
	}
	return &resp, nil

}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	log.Printf("Listening on port %d", c.port)
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", c.port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	pb.RegisterTakticianServer(grpcServer, &server{})

	grpcServer.Serve(lis)
	return subcommands.ExitSuccess
}
<commit_msg>isTak: cache<commit_after>package serve

import (
	"flag"
	"fmt"
	"log"
	"net"
	"sync"

	"context"

	"google.golang.org/grpc"

	"github.com/google/subcommands"
	"github.com/nelhage/taktician/ai"
	"github.com/nelhage/taktician/pb/tak/proto"
	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/symmetry"
	"github.com/nelhage/taktician/tak"
)

type Command struct {
	port int
}

func (*Command) Name() string     { return "serve" }
func (*Command) Synopsis() string { return "Seve Taktician RPCs via GRPC" }
func (*Command) Usage() string {
	return `serve
`
}

func (c *Command) SetFlags(flags *flag.FlagSet) {
	flags.IntVar(&c.port, "port", 55430, "bind port")
}

type cache struct {
	sync.Mutex
	player  *ai.MinimaxAI
	cfg     ai.MinimaxConfig
	precise bool
}

type server struct {
	analyzeCache cache
	istakCache   cache
}

func (c *cache) getPlayer(size int, depth int, precise bool) *ai.MinimaxAI {
	if c.cfg.Size != size || c.cfg.Depth != int(depth) || c.precise != precise {
		c.cfg = ai.MinimaxConfig{
			Size:  size,
			Depth: depth,
			Debug: 1,
		}
		if precise {
			c.cfg.MakePrecise()
		}
		c.player = ai.NewMinimax(c.cfg)
		c.precise = precise
	}
	return c.player
}

func (s *server) Analyze(ctx context.Context, req *pb.AnalyzeRequest) (*pb.AnalyzeResponse, error) {
	p, e := ptn.ParseTPS(req.Position)
	if e != nil {
		return nil, e
	}

	s.analyzeCache.Lock()
	defer s.analyzeCache.Unlock()
	player := s.analyzeCache.getPlayer(p.Size(), int(req.Depth), req.Precise)

	var resp pb.AnalyzeResponse
	pv, value, _ := player.Analyze(ctx, p)
	for _, m := range pv {
		resp.Pv = append(resp.Pv, ptn.FormatMove(m))
	}
	resp.Value = value

	return &resp, nil
}

func (s *server) Canonicalize(ctx context.Context, req *pb.CanonicalizeRequest) (*pb.CanonicalizeResponse, error) {
	var ms []tak.Move
	for _, mstr := range req.Moves {
		mv, e := ptn.ParseMove(mstr)
		if e != nil {
			return nil, e
		}
		ms = append(ms, mv)
	}

	ms, e := symmetry.Canonical(int(req.Size), ms)
	if e != nil {
		return nil, e
	}

	var outms []string
	for _, m := range ms {
		outms = append(outms, ptn.FormatMove(m))
	}
	return &pb.CanonicalizeResponse{
		Moves: outms,
	}, nil
}

func (s *server) IsPositionInTak(ctx context.Context, req *pb.IsPositionInTakRequest) (*pb.IsPositionInTakResponse, error) {
	p, e := ptn.ParseTPS(req.Position)
	if e != nil {
		return nil, e
	}

	s.istakCache.Lock()
	defer s.istakCache.Unlock()
	player := s.istakCache.getPlayer(p.Size(), 1, true)

	pass, e := p.Move(tak.Move{Type: tak.Pass})
	pv, value, _ := player.Analyze(ctx, pass)

	var resp pb.IsPositionInTakResponse
	resp.InTak = value > ai.WinThreshold
	if resp.InTak {
		resp.TakMove = ptn.FormatMove(pv[0])
	}
	return &resp, nil

}

func (c *Command) Execute(ctx context.Context, flag *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	log.Printf("Listening on port %d", c.port)
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", c.port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	pb.RegisterTakticianServer(grpcServer, &server{})

	grpcServer.Serve(lis)
	return subcommands.ExitSuccess
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"time"

	"github.com/mattn/go-mastodon"
	"github.com/urfave/cli"
)

func cmdFollowers(c *cli.Context) error {
	client := c.App.Metadata["client"].(*mastodon.Client)
	config := c.App.Metadata["config"].(*mastodon.Config)

	account, err := client.GetAccountCurrentUser(context.Background())
	if err != nil {
		return err
	}
	var maxID int64
	var followers []*mastodon.Account
	for {
		pg := mastodon.Pagination{MaxID: maxID}
		fs, err := client.GetAccountFollowers(context.Background(), account.ID, &pg)
		if err != nil {
			return err
		}
		followers = append(followers, fs...)
		if pg.MaxID == 0 {
			break
		}
		maxID = pg.MaxID
		time.Sleep(10 * time.Second)
	}
	s := newScreen(config)
	for _, follower := range followers {
		fmt.Fprintf(c.App.Writer, "%v,%v\n", follower.ID, s.acct(follower.Acct))
	}
	return nil
}
<commit_msg>Fix not to declare Pagination every time<commit_after>package main

import (
	"context"
	"fmt"
	"time"

	"github.com/mattn/go-mastodon"
	"github.com/urfave/cli"
)

func cmdFollowers(c *cli.Context) error {
	client := c.App.Metadata["client"].(*mastodon.Client)
	config := c.App.Metadata["config"].(*mastodon.Config)

	account, err := client.GetAccountCurrentUser(context.Background())
	if err != nil {
		return err
	}
	var followers []*mastodon.Account
	var pg mastodon.Pagination
	for {
		fs, err := client.GetAccountFollowers(context.Background(), account.ID, &pg)
		if err != nil {
			return err
		}
		followers = append(followers, fs...)
		if pg.MaxID == 0 {
			break
		}
		pg.SinceID = 0
		time.Sleep(10 * time.Second)
	}
	s := newScreen(config)
	for _, follower := range followers {
		fmt.Fprintf(c.App.Writer, "%v,%v\n", follower.ID, s.acct(follower.Acct))
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/johnweldon/nginx-log-parse/util"
)

func main() {
	p := util.NewParser(os.Stdin)
	for {
		select {
		case record := <-p.EntryCh:
			if record == nil {
				continue
			}
			fmt.Fprintf(os.Stdout, "%20s %16s %d %-40s %s\n", record.TimeLocal.Format("2006-01-02 15:04:05"), record.RemoteAddr, record.Status, record.Request, record.HttpReferrer)
		case log := <-p.Log:
			fmt.Fprintf(os.Stderr, "%s", log)
		case <-p.Dying():
			return
		}
	}
}
<commit_msg>missed a newline<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/johnweldon/nginx-log-parse/util"
)

func main() {
	p := util.NewParser(os.Stdin)
	for {
		select {
		case record := <-p.EntryCh:
			if record == nil {
				continue
			}
			fmt.Fprintf(os.Stdout, "%20s %16s %d %-40s %s\n", record.TimeLocal.Format("2006-01-02 15:04:05"), record.RemoteAddr, record.Status, record.Request, record.HttpReferrer)
		case log := <-p.Log:
			fmt.Fprintf(os.Stderr, "%s\n", log)
		case <-p.Dying():
			return
		}
	}
}
<|endoftext|>