<commit_before>package fetch

import (
	"bytes"
	"fmt"
	"github.com/disintegration/imaging"
	"github.com/rwcarlsen/goexif/exif"
	"image"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
)

func needsRotation(src io.Reader) (bool, int) {
	metadata, err := exif.Decode(src)
	if err != nil {
		fmt.Println("Error decoding EXIF data: " + err.Error())
		return false, 0
	}

	orientation, err := metadata.Get(exif.Orientation)
	if err != nil {
		fmt.Println("Error trying to find EXIF orientation: " + err.Error())
		return false, 0
	}

	angle := 0
	rotate := false

	switch orientation.String() {
	case "6":
		angle = 90
		rotate = true
	case "3":
		angle = 180
		rotate = true
	case "8":
		angle = 270
		rotate = true
	}

	return rotate, angle
}

func Resize(src io.Reader, c *CacheContext) (io.Reader, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	data := bytes.NewReader(raw)

	image, format, err := image.Decode(data)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	buf := new(bytes.Buffer)

	factor := float64(c.Width) / float64(image.Bounds().Size().X)
	height := int(float64(image.Bounds().Size().Y) * factor)

	image = imaging.Resize(image, c.Width, height, imaging.Linear)

	if _, err := data.Seek(0, 0); err != nil {
		fmt.Println(err.Error())
	}

	if rotate, angle := needsRotation(data); rotate {
		switch angle {
		case 90:
			image = imaging.Rotate90(image)
		case 180:
			image = imaging.Rotate180(image)
		case 270:
			image = imaging.Rotate270(image)
		}
	}

	switch format {
	case "jpeg":
		err = jpeg.Encode(buf, image, nil)
	case "png":
		err = png.Encode(buf, image)
	}

	return buf, err
}

func CenterCrop(src io.Reader, c *CacheContext) (io.Reader, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	data := bytes.NewReader(raw)

	image, format, err := image.Decode(data)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	data.Seek(0, 0)

	if rotate, angle := needsRotation(data); rotate {
		switch angle {
		case 90:
			image = imaging.Rotate90(image)
		case 180:
			image = imaging.Rotate180(image)
		case 270:
			image = imaging.Rotate270(image)
		}
	}

	buf := new(bytes.Buffer)

	height := image.Bounds().Size().Y
	width := image.Bounds().Size().X

	if width < height {
		image = imaging.CropCenter(image, width, width)
	} else if width > height {
		image = imaging.CropCenter(image, height, height)
	} else {
		image = imaging.CropCenter(image, width, height)
	}

	switch format {
	case "jpeg":
		err = jpeg.Encode(buf, image, nil)
	case "png":
		err = png.Encode(buf, image)
	}

	return buf, err
}
<commit_msg>Also here<commit_after>package fetch

import (
	"bytes"
	"fmt"
	"github.com/disintegration/imaging"
	"github.com/rwcarlsen/goexif/exif"
	"image"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"
)

func needsRotation(src io.Reader) (bool, int) {
	metadata, err := exif.Decode(src)
	if err != nil {
		fmt.Println("Error decoding EXIF data: " + err.Error())
		return false, 0
	}

	orientation, err := metadata.Get(exif.Orientation)
	if err != nil {
		fmt.Println("Error trying to find EXIF orientation: " + err.Error())
		return false, 0
	}

	angle := 0
	rotate := false

	switch orientation.String() {
	case "6":
		angle = 90
		rotate = true
	case "3":
		angle = 180
		rotate = true
	case "8":
		angle = 270
		rotate = true
	}

	return rotate, angle
}

func Resize(src io.Reader, c *CacheContext) (io.Reader, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	data := bytes.NewReader(raw)

	image, format, err := image.Decode(data)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	buf := new(bytes.Buffer)

	factor := float64(c.Width) / float64(image.Bounds().Size().X)
	height := int(float64(image.Bounds().Size().Y) * factor)

	image = imaging.Resize(image, c.Width, height, imaging.Linear)

	if _, err := data.Seek(0, 0); err != nil {
		fmt.Println(err.Error())
	}

	if rotate, angle := needsRotation(data); rotate {
		switch angle {
		case 90:
			image = imaging.Rotate90(image)
		case 180:
			image = imaging.Rotate180(image)
		case 270:
			image = imaging.Rotate270(image)
		}
	}

	switch format {
	case "jpeg":
		err = jpeg.Encode(buf, image, nil)
	case "png":
		err = png.Encode(buf, image)
	}

	return buf, err
}

func CenterCrop(src io.Reader, c *CacheContext) (io.Reader, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	data := bytes.NewReader(raw)

	image, format, err := image.Decode(data)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	if _, err := data.Seek(0, 0); err != nil {
		fmt.Println(err.Error())
	}

	if rotate, angle := needsRotation(data); rotate {
		switch angle {
		case 90:
			image = imaging.Rotate90(image)
		case 180:
			image = imaging.Rotate180(image)
		case 270:
			image = imaging.Rotate270(image)
		}
	}

	buf := new(bytes.Buffer)

	height := image.Bounds().Size().Y
	width := image.Bounds().Size().X

	if width < height {
		image = imaging.CropCenter(image, width, width)
	} else if width > height {
		image = imaging.CropCenter(image, height, height)
	} else {
		image = imaging.CropCenter(image, width, height)
	}

	switch format {
	case "jpeg":
		err = jpeg.Encode(buf, image, nil)
	case "png":
		err = png.Encode(buf, image)
	}

	return buf, err
}
<|endoftext|>


<commit_before>/**
mySql数据库组件类
创建人：邵炜
创建时间：2017年03月11日15:55:14
*/
package gutil

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/smtc/glog"
)

/**
数据库连接对象
创建人：邵炜
创建时间：2017年03月11日16:01:20
*/
type MySqlDBStruct struct {
	DbUser string //数据库用户名
	DbHost string //数据库地址
	DbPort int    //数据库端口
	DbPass string //数据库密码
	DbName string //数据库库名
}

/**
数据库连接
创建人：邵炜
创建时间：2017年03月11日15:56:06
输入参数：数据库对象
输出对象：数据库连接对象
*/
func MySqlSQlConntion(model MySqlDBStruct) *sql.DB {
	dbClause := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8&allowAllFiles=true", model.DbUser, model.DbPass, model.DbHost, model.DbPort, model.DbName)
	dbs, err := sql.Open("mysql", dbClause)
	if err != nil {
		glog.Error("mysql can't connection dbClause: %s err: %s \n", dbClause, err.Error())
		return nil
	}
	err = dbs.Ping()
	if err != nil {
		glog.Error("mysql can't ping dbClause: %s err: %s \n", dbClause, err.Error())
		return nil
	}
	return dbs
}

/**
数据库关闭
创建人：邵炜
创建时间：2017年03月11日16:14:50
输入参数：数据库连接对象
*/
func MySqlClose(dbs *sql.DB) {
	if dbs == nil {
		return
	}
	dbs.Close()
}

/**
查询方法
创建人:邵炜
创建时间:2015年12月29日17:26:41
修正时间：2017年03月11日16:21:45
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句 param执行SQL的语句参数化传递
输出参数: 查询返回条数  错误对象输出
*/
func MySqlSelect(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (*sql.Rows, error) {

	var (
		row *sql.Rows
		err error
	)
	err = dbs.Ping()
	if err != nil {
		glog.Error("mysql can't ping %s \n", err.Error())
		MySqlClose(dbs)
		dbs = MySqlSQlConntion(model)
	}
	if param == nil {
		row, err = dbs.Query(sqlStr)
	} else {
		row, err = dbs.Query(sqlStr, param...)
	}
	if err != nil {
		glog.Error("mysql query can't select sql: %s err: %s \n", sqlStr, err.Error())
		return nil, err
	}
	return row, nil
}

/**
数据库运行方法
创建人:邵炜
创建时间:2015年12月29日17:33:06
修正时间：2017年03月11日16:21:36
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句  param执行SQL的语句参数化传递
输出参数: 执行结果对象  错误对象输出
*/
func MySqlSqlExec(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (sql.Result, error) {
	var (
		exec sql.Result
		err  error
	)

	err = dbs.Ping()

	if err != nil {
		glog.Error("mysql can't ping %s \n", err.Error())
		MySqlClose(dbs)
		dbs = MySqlSQlConntion(model)
	}

	if param == nil {
		exec, err = dbs.Exec(sqlStr)
	} else {
		exec, err = dbs.Exec(sqlStr, param...)
	}

	if err != nil {
		glog.Error("mysql exec can't carried out sql: %s err: %s \n", sqlStr, err.Error())
		return nil, err
	}

	return exec, nil
}
<commit_msg>删除日志记录,交由上层程序进行查阅,修正数据库连接不上出现问题,无法查阅问题<commit_after>/**
mySql数据库组件类
创建人：邵炜
创建时间：2017年03月11日15:55:14
*/
package gutil

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/smtc/glog"
)

/**
数据库连接对象
创建人：邵炜
创建时间：2017年03月11日16:01:20
*/
type MySqlDBStruct struct {
	DbUser string //数据库用户名
	DbHost string //数据库地址
	DbPort int    //数据库端口
	DbPass string //数据库密码
	DbName string //数据库库名
}

/**
数据库连接
创建人：邵炜
创建时间：2017年03月11日15:56:06
输入参数：数据库对象
输出对象：数据库连接对象
*/
func MySqlSQlConntion(model MySqlDBStruct) (*sql.DB, error) {
	dbClause := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8&allowAllFiles=true", model.DbUser, model.DbPass, model.DbHost, model.DbPort, model.DbName)
	dbs, err := sql.Open("mysql", dbClause)
	if err != nil {
		return nil, err
	}
	err = dbs.Ping()
	if err != nil {
		return nil, err
	}
	return dbs, err
}

/**
数据库关闭
创建人：邵炜
创建时间：2017年03月11日16:14:50
输入参数：数据库连接对象
*/
func MySqlClose(dbs *sql.DB) {
	if dbs == nil {
		return
	}
	dbs.Close()
}

/**
查询方法
创建人:邵炜
创建时间:2015年12月29日17:26:41
修正时间：2017年03月11日16:21:45
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句 param执行SQL的语句参数化传递
输出参数: 查询返回条数  错误对象输出
*/
func MySqlSelect(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (*sql.Rows, error) {

	var (
		row *sql.Rows
		err error
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs = MySqlSQlConntion(model)
	}
	if param == nil {
		row, err = dbs.Query(sqlStr)
	} else {
		row, err = dbs.Query(sqlStr, param...)
	}
	if err != nil {
		return nil, err
	}
	return row, nil
}

/**
数据库运行方法
创建人:邵炜
创建时间:2015年12月29日17:33:06
修正时间：2017年03月11日16:21:36
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句  param执行SQL的语句参数化传递
输出参数: 执行结果对象  错误对象输出
*/
func MySqlSqlExec(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (sql.Result, error) {
	var (
		exec sql.Result
		err  error
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs = MySqlSQlConntion(model)
	}

	if param == nil {
		exec, err = dbs.Exec(sqlStr)
	} else {
		exec, err = dbs.Exec(sqlStr, param...)
	}

	if err != nil {
		return nil, err
	}

	return exec, nil
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "net"
  "bufio"
  "strings"
  "net/http"
  "time"
)

type IPSet struct {
  root BitNode
}

type BitNode struct {
  parent,zero,one *BitNode
  depth uint32
  full bool
  value uint32
}

func main () {

  ipsetCurrent := createIPSet()
  ipsetNew := createIPSet()

  blocklists := []string{ "https://www.confusticate.com/all.txt","https://www.confusticate.com/drop.txt","https://www.confusticate.com/edrop.txt","https://www.confusticate.com/test.txt"}

  for 1==1 {
    fmt.Println("# start")

    for _, url := range blocklists {
      fmt.Printf("# %s\n",url)

      data := downloadBlocklist(url)

      for _, ipnet := range data {
        ipsetNew.add(&ipnet)
      }
    }

    for _, newip := range ipsetNew.getAll() {
      if !ipsetCurrent.contains(&newip) {
        fmt.Printf("announce %s\n", newip.String())
      }
    }

    for _, existing := range ipsetCurrent.getAll() {
      if !ipsetNew.contains(&existing) {
        fmt.Printf("withdraw %s\n", existing.String())
      }
    }

    ipsetCurrent = ipsetNew
    ipsetNew = createIPSet()
    fmt.Println("# end")
    time.Sleep(10 * time.Second)
  }
}

func downloadBlocklist(url string) []net.IPNet {
  var nets = make([]net.IPNet,0)
  resp, err := http.Get(url)

  if err != nil {
    fmt.Println(err)
    return nil
  }

  scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
    line := scanner.Text()
    items := strings.Split(line,";")
    ipnet := StringToIPNet(items[0])

    if ipnet != nil {
        nets = append(nets,*ipnet)
    }
	}

  return nets
}

// ====================================================================================================================

func createIPSet() IPSet {
  return IPSet { root:  BitNode{parent: nil, zero :nil, one :nil, depth: 32, full: false, value: 0} }
}

func (ipset *IPSet) add(ipnet *net.IPNet) {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      addIP(&ipset.root,addr,uint32(bits))
  }
}

func (ipset *IPSet) getAll() []net.IPNet {
  return collectIPs(&ipset.root)
}

func (ipset *IPSet) contains(ipnet *net.IPNet) bool {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      return containsIP(&ipset.root,addr,uint32(bits))
  } else {
    return false
  }
}

func StringToIPNet(text string) *net.IPNet {
  var ip,ipnet,error = net.ParseCIDR(strings.TrimSpace(text))

  if error != nil {
    ip = net.ParseIP(text)
    if ip != nil {
      ipnet = &net.IPNet{IP: ip, Mask: net.CIDRMask(32,32)}
    }
  }

  return ipnet
}

func IPtoInt(addr net.IP) uint32 {
  return uint32(addr.To4()[0]) << 24 |
         uint32(addr.To4()[1]) << 16 |
         uint32(addr.To4()[2]) << 8 |
         uint32(addr.To4()[3])
}

func IntToIP(ip uint32) net.IP {
  result := make(net.IP, 4)
           result[3] = byte(ip)
           result[2] = byte(ip >>8)
           result[1] = byte(ip >>16)
           result[0] = byte(ip >> 24)
  return result
}

func CheckBit(num uint32, bit uint32) bool {
  return (num & (1 << bit)) != 0
}

func outputTree(node *BitNode) {
  if node.full {
    var ipnet = IPNetFromNode(node)
    fmt.Println(ipnet.String())
  } else {
    if node.zero != nil {
      outputTree(node.zero)
    }

    if node.one != nil {
      outputTree(node.one)
    }
  }
}

func collectIPs(node *BitNode) []net.IPNet {
  var nets = make([]net.IPNet,0)

  if node.full {
    var ipnet = IPNetFromNode(node)
    return []net.IPNet{ipnet}
  } else {
    if node.zero != nil {
      nets = append(nets,collectIPs(node.zero)...)
    }

    if node.one != nil {
      nets = append(nets,collectIPs(node.one)...)
    }
  }
  return nets
}

func IPFromNode(node *BitNode) net.IP {
  var cur = node
  var accumulate uint32 = 0

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return IntToIP(accumulate)
}

func IPNetFromNode(node *BitNode) net.IPNet {
  var cur = node
  var accumulate uint32 = 0
  var mask = int(32 - node.depth)

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return net.IPNet{IP: IntToIP(accumulate), Mask: net.CIDRMask(mask,32)}
}

func containsIP(node *BitNode, addr uint32, mask uint32) bool {
  if node.full {
    return true
  }

  if 32-node.depth > mask {
    return false
  }

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      return false
    } else {
      return containsIP(node.one,addr,mask)
    }
  } else {
    if node.zero == nil {
      return false
    } else {
      return containsIP(node.zero,addr,mask)
    }
  }
}

func addIP(node *BitNode, addr uint32, mask uint32) bool {

  if node.depth == 0 || 32 - node.depth == mask || node.full {
    node.full = true
    return node.full
  }

  var child *BitNode

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 1}
      node.one = child
    } else {
      child = node.one
    }
  } else {
    if node.zero == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 0 }
      node.zero = child
    } else {
      child = node.zero
    }
  }

  addIP(child, addr, mask)

  node.full = (node.one != nil && node.one.full) && (node.zero != nil && node.zero.full)
  return node.full
}
<commit_msg>Fix up when overlapping ranges change<commit_after>package main

import (
  "fmt"
  "net"
  "bufio"
  "strings"
  "net/http"
  "time"
)

type IPSet struct {
  root BitNode
}

type BitNode struct {
  parent,zero,one *BitNode
  depth uint32
  full bool
  value uint32
}

func main () {

  ipsetCurrent := createIPSet()
  ipsetNew := createIPSet()

  blocklists := []string{ "https://www.confusticate.com/all.txt","https://www.confusticate.com/drop.txt","https://www.confusticate.com/edrop.txt","https://www.confusticate.com/test.txt"}

  for 1==1 {
    fmt.Println("# start")

    for _, url := range blocklists {
      fmt.Printf("# %s\n",url)

      data := downloadBlocklist(url)

      for _, ipnet := range data {
        ipsetNew.add(&ipnet)
      }
    }

    // If IPs in the new set aren't in the current set, announce them
    for _, newip := range ipsetNew.getAll() {
      current := ipsetCurrent.contains(&newip)
      if current == nil {
        fmt.Printf("announce %s\n", newip.String())
      } else {
        // However, if the thing that matched wasn't identical to the existing one
        // I.e. more specific IP, withdraw the existing one *and* annouce the new one
        if current.String() != newip.String() {
          fmt.Printf("withdraw %s\n", current.String())
          fmt.Printf("announce %s\n", newip.String())
        }
      }
    }

    // If IPs from the current set aren't in the new set
    for _, existing := range ipsetCurrent.getAll() {
      newip := ipsetNew.contains(&existing)

      if newip == nil {
        fmt.Printf("withdraw %s\n", existing.String())
      } else {
        // However, if the thing that matched wasn't identical to the existing one
        // I.e. more specific IP, withdraw the existing one *and* annouce the new one
        if newip != nil && newip.String() != existing.String() {
          fmt.Printf("withdraw %s\n", existing.String())
          fmt.Printf("announce %s\n", newip.String())
        }
      }
    }

    ipsetCurrent = ipsetNew
    ipsetNew = createIPSet()
    fmt.Println("# end")
    time.Sleep(10 * time.Second)
  }
}

func downloadBlocklist(url string) []net.IPNet {
  var nets = make([]net.IPNet,0)
  resp, err := http.Get(url)

  if err != nil {
    fmt.Println(err)
    return nil
  }

  scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
    line := scanner.Text()
    items := strings.Split(line,";")
    ipnet := StringToIPNet(items[0])

    if ipnet != nil {
        nets = append(nets,*ipnet)
    }
	}

  return nets
}

// ====================================================================================================================

func createIPSet() IPSet {
  return IPSet { root:  BitNode{parent: nil, zero :nil, one :nil, depth: 32, full: false, value: 0} }
}

func (ipset *IPSet) add(ipnet *net.IPNet) {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      addIP(&ipset.root,addr,uint32(bits))
  }
}

func (ipset *IPSet) getAll() []net.IPNet {
  return collectIPs(&ipset.root)
}

func (ipset *IPSet) contains(ipnet *net.IPNet) *net.IPNet {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      return containsIP(&ipset.root,addr,uint32(bits))
  } else {
    return nil
  }
}

func StringToIPNet(text string) *net.IPNet {
  var ip,ipnet,error = net.ParseCIDR(strings.TrimSpace(text))

  if error != nil {
    ip = net.ParseIP(text)
    if ip != nil {
      ipnet = &net.IPNet{IP: ip, Mask: net.CIDRMask(32,32)}
    }
  }

  return ipnet
}

func IPtoInt(addr net.IP) uint32 {
  return uint32(addr.To4()[0]) << 24 |
         uint32(addr.To4()[1]) << 16 |
         uint32(addr.To4()[2]) << 8 |
         uint32(addr.To4()[3])
}

func IntToIP(ip uint32) net.IP {
  result := make(net.IP, 4)
           result[3] = byte(ip)
           result[2] = byte(ip >>8)
           result[1] = byte(ip >>16)
           result[0] = byte(ip >> 24)
  return result
}

func CheckBit(num uint32, bit uint32) bool {
  return (num & (1 << bit)) != 0
}

func outputTree(node *BitNode) {
  if node.full {
    var ipnet = IPNetFromNode(node)
    fmt.Println(ipnet.String())
  } else {
    if node.zero != nil {
      outputTree(node.zero)
    }

    if node.one != nil {
      outputTree(node.one)
    }
  }
}

func collectIPs(node *BitNode) []net.IPNet {
  var nets = make([]net.IPNet,0)

  if node.full {
    var ipnet = IPNetFromNode(node)
    return []net.IPNet{ipnet}
  } else {
    if node.zero != nil {
      nets = append(nets,collectIPs(node.zero)...)
    }

    if node.one != nil {
      nets = append(nets,collectIPs(node.one)...)
    }
  }
  return nets
}

func IPFromNode(node *BitNode) net.IP {
  var cur = node
  var accumulate uint32 = 0

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return IntToIP(accumulate)
}

func IPNetFromNode(node *BitNode) net.IPNet {
  var cur = node
  var accumulate uint32 = 0
  var mask = int(32 - node.depth)

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return net.IPNet{IP: IntToIP(accumulate), Mask: net.CIDRMask(mask,32)}
}

func containsIP(node *BitNode, addr uint32, mask uint32) *net.IPNet {
  if node.full {
    ipnet := IPNetFromNode(node)

    return &ipnet
  }

  if 32-node.depth > mask {
    return nil
  }

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      return nil
    } else {
      return containsIP(node.one,addr,mask)
    }
  } else {
    if node.zero == nil {
      return nil
    } else {
      return containsIP(node.zero,addr,mask)
    }
  }
}

func addIP(node *BitNode, addr uint32, mask uint32) bool {

  if node.depth == 0 || 32 - node.depth == mask || node.full {
    node.full = true
    return node.full
  }

  var child *BitNode

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 1}
      node.one = child
    } else {
      child = node.one
    }
  } else {
    if node.zero == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 0 }
      node.zero = child
    } else {
      child = node.zero
    }
  }

  addIP(child, addr, mask)

  node.full = (node.one != nil && node.one.full) && (node.zero != nil && node.zero.full)
  return node.full
}
<|endoftext|>


<commit_before>package fnet

type FrameConn interface {
	FrameSize() int

	// SendFrame sends a bounded-size frame over the connection.
	// PRE: b :->[] bs, len(bs) = FrameSize
	// RET: b :->[] xs, len(xs) = FrameSize
	// EFF: if err = nil then SendFrame(bs) else (SendFrame(xs) OR NoEffects)
	SendFrame(b []byte) error // only frames of valid size or less

	// RecvFrame receives a bounded-size fram over the connection
	// PRE: b :->[] mm, len(mm) = FrameSize.
	// RET: b :->[] bs, len(bs) = FrameSize
	// EFF: if err = nil then RecvFrame(bs) else (RecvFrame(bs) OR NoEffects)
	RecvFrame(b []byte) (n int, err error)
}
<commit_msg>Added back FrameConn Stop method<commit_after>package fnet

type FrameConn interface {
	FrameSize() int

	//Gracefully stop the FrameConn
	Stop()

	// SendFrame sends a bounded-size frame over the connection.
	// PRE: b :->[] bs, len(bs) = FrameSize
	// RET: b :->[] xs, len(xs) = FrameSize
	// EFF: if err = nil then SendFrame(bs) else (SendFrame(xs) OR NoEffects)
	SendFrame(b []byte) error // only frames of valid size or less

	// RecvFrame receives a bounded-size fram over the connection
	// PRE: b :->[] mm, len(mm) = FrameSize.
	// RET: b :->[] bs, len(bs) = FrameSize
	// EFF: if err = nil then RecvFrame(bs) else (RecvFrame(bs) OR NoEffects)
	RecvFrame(b []byte) (n int, err error)
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/
package csp

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/x509"
	"encoding/asn1"
	"encoding/pem"
	"io"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
)

// LoadPrivateKey loads a private key from a file in keystorePath.  It looks
// for a file ending in "_sk" and expects a PEM-encoded PKCS8 EC private key.
func LoadPrivateKey(keystorePath string) (*ecdsa.PrivateKey, error) {
	var priv *ecdsa.PrivateKey

	walkFunc := func(path string, info os.FileInfo, pathErr error) error {
		if !strings.HasSuffix(path, "_sk") {
			return nil
		}

		rawKey, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		priv, err = parsePrivateKeyPEM(rawKey)
		if err != nil {
			return errors.WithMessage(err, path)
		}

		return nil
	}

	err := filepath.Walk(keystorePath, walkFunc)
	if err != nil {
		return nil, err
	}

	return priv, err
}

func parsePrivateKeyPEM(rawKey []byte) (*ecdsa.PrivateKey, error) {
	block, _ := pem.Decode(rawKey)
	if block == nil {
		return nil, errors.New("bytes are not PEM encoded")
	}

	key, err := x509.ParsePKCS8PrivateKey(block.Bytes)
	if err != nil {
		return nil, errors.WithMessage(err, "pem bytes are not PKCS8 encoded ")
	}

	priv, ok := key.(*ecdsa.PrivateKey)
	if !ok {
		return nil, errors.New("pem bytes do not contain an EC private key")
	}
	return priv, nil
}

// GeneratePrivateKey creates an EC private key using a P-256 curve and stores
// it in keystorePath.
func GeneratePrivateKey(keystorePath string) (*ecdsa.PrivateKey, error) {
	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		return nil, errors.WithMessage(err, "failed to generate private key")
	}

	pkcs8Encoded, err := x509.MarshalPKCS8PrivateKey(priv)
	if err != nil {
		return nil, errors.WithMessage(err, "failed to marshal private key")
	}

	pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8Encoded})

	keyFile := filepath.Join(keystorePath, "priv_sk")
	err = ioutil.WriteFile(keyFile, pemEncoded, 0o600)
	if err != nil {
		return nil, errors.WithMessagef(err, "failed to save private key to file %s", keyFile)
	}

	return priv, err
}

/**
ECDSA signer implements the crypto.Signer interface for ECDSA keys.  The
Sign method ensures signatures are created with Low S values since Fabric
normalizes all signatures to Low S.
See https://github.com/bitcoin/bips/blob/master/bip-0146.mediawiki#low_s
for more detail.
*/
type ECDSASigner struct {
	PrivateKey *ecdsa.PrivateKey
}

// Public returns the ecdsa.PublicKey associated with PrivateKey.
func (e *ECDSASigner) Public() crypto.PublicKey {
	return &e.PrivateKey.PublicKey
}

// Sign signs the digest and ensures that signatures use the Low S value.
func (e *ECDSASigner) Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) {
	r, s, err := ecdsa.Sign(rand, e.PrivateKey, digest)
	if err != nil {
		return nil, err
	}

	// ensure Low S signatures
	sig := toLowS(
		e.PrivateKey.PublicKey,
		ECDSASignature{
			R: r,
			S: s,
		},
	)

	// return marshaled aignature
	return asn1.Marshal(sig)
}

/**
When using ECDSA, both (r,s) and (r, -s mod n) are valid signatures.  In order
to protect against signature malleability attacks, Fabric normalizes all
signatures to a canonical form where s is at most half the order of the curve.
In order to make signatures compliant with what Fabric expects, toLowS creates
signatures in this canonical form.
*/
func toLowS(key ecdsa.PublicKey, sig ECDSASignature) ECDSASignature {
	// calculate half order of the curve
	halfOrder := new(big.Int).Div(key.Curve.Params().N, big.NewInt(2))
	// check if s is greater than half order of curve
	if sig.S.Cmp(halfOrder) == 1 {
		// Set s to N - s so that s will be less than or equal to half order
		sig.S.Sub(key.Params().N, sig.S)
	}
	return sig
}

type ECDSASignature struct {
	R, S *big.Int
}
<commit_msg>Update csp.go<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/
package csp

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/x509"
	"encoding/asn1"
	"encoding/pem"
	"io"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
)

// LoadPrivateKey loads a private key from a file in keystorePath.  It looks
// for a file ending in "_sk" and expects a PEM-encoded PKCS8 EC private key.
func LoadPrivateKey(keystorePath string) (*ecdsa.PrivateKey, error) {
	var priv *ecdsa.PrivateKey

	walkFunc := func(path string, info os.FileInfo, pathErr error) error {
		if !strings.HasSuffix(path, "_sk") {
			return nil
		}

		rawKey, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		priv, err = parsePrivateKeyPEM(rawKey)
		if err != nil {
			return errors.WithMessage(err, path)
		}

		return nil
	}

	err := filepath.Walk(keystorePath, walkFunc)
	if err != nil {
		return nil, err
	}

	return priv, err
}

func parsePrivateKeyPEM(rawKey []byte) (*ecdsa.PrivateKey, error) {
	block, _ := pem.Decode(rawKey)
	if block == nil {
		return nil, errors.New("bytes are not PEM encoded")
	}

	key, err := x509.ParsePKCS8PrivateKey(block.Bytes)
	if err != nil {
		return nil, errors.WithMessage(err, "pem bytes are not PKCS8 encoded ")
	}

	priv, ok := key.(*ecdsa.PrivateKey)
	if !ok {
		return nil, errors.New("pem bytes do not contain an EC private key")
	}
	return priv, nil
}

// GeneratePrivateKey creates an EC private key using a P-256 curve and stores
// it in keystorePath.
func GeneratePrivateKey(keystorePath string) (*ecdsa.PrivateKey, error) {
	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		return nil, errors.WithMessage(err, "failed to generate private key")
	}

	pkcs8Encoded, err := x509.MarshalPKCS8PrivateKey(priv)
	if err != nil {
		return nil, errors.WithMessage(err, "failed to marshal private key")
	}

	pemEncoded := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8Encoded})

	keyFile := filepath.Join(keystorePath, "priv_sk")
	err = ioutil.WriteFile(keyFile, pemEncoded, 0o600)
	if err != nil {
		return nil, errors.WithMessagef(err, "failed to save private key to file %s", keyFile)
	}

	return priv, err
}

/**
ECDSA signer implements the crypto.Signer interface for ECDSA keys.  The
Sign method ensures signatures are created with Low S values since Fabric
normalizes all signatures to Low S.
See https://github.com/bitcoin/bips/blob/master/bip-0146.mediawiki#low_s
for more detail.
*/
type ECDSASigner struct {
	PrivateKey *ecdsa.PrivateKey
}

// Public returns the ecdsa.PublicKey associated with PrivateKey.
func (e *ECDSASigner) Public() crypto.PublicKey {
	return &e.PrivateKey.PublicKey
}

// Sign signs the digest and ensures that signatures use the Low S value.
func (e *ECDSASigner) Sign(rand io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) {
	r, s, err := ecdsa.Sign(rand, e.PrivateKey, digest)
	if err != nil {
		return nil, err
	}

	// ensure Low S signatures
	sig := toLowS(
		e.PrivateKey.PublicKey,
		ECDSASignature{
			R: r,
			S: s,
		},
	)

	// return marshaled signature
	return asn1.Marshal(sig)
}

/**
When using ECDSA, both (r,s) and (r, -s mod n) are valid signatures.  In order
to protect against signature malleability attacks, Fabric normalizes all
signatures to a canonical form where s is at most half the order of the curve.
In order to make signatures compliant with what Fabric expects, toLowS creates
signatures in this canonical form.
*/
func toLowS(key ecdsa.PublicKey, sig ECDSASignature) ECDSASignature {
	// calculate half order of the curve
	halfOrder := new(big.Int).Div(key.Curve.Params().N, big.NewInt(2))
	// check if s is greater than half order of curve
	if sig.S.Cmp(halfOrder) == 1 {
		// Set s to N - s so that s will be less than or equal to half order
		sig.S.Sub(key.Params().N, sig.S)
	}
	return sig
}

type ECDSASignature struct {
	R, S *big.Int
}
<|endoftext|>


<commit_before>package agollo

import (
	. "github.com/tevid/gohamcrest"
	"testing"
	"time"
)

func TestInit(t *testing.T) {
	config := GetAppConfig(nil)

	Assert(t, config, NotNilVal())
	Assert(t, "test", Equal(config.AppId))
	Assert(t, "dev", Equal(config.Cluster))
	Assert(t, "application,abc1", Equal(config.NamespaceName))
	Assert(t, "localhost:8888", Equal(config.Ip))

	apolloConfig := GetCurrentApolloConfig()[defaultNamespace]
	Assert(t, "test", Equal(apolloConfig.AppId))
	Assert(t, "dev", Equal(apolloConfig.Cluster))
	Assert(t, "application", Equal(apolloConfig.NamespaceName))

}

func TestStructInit(t *testing.T) {

	readyConfig := &AppConfig{
		AppId:         "test1",
		Cluster:       "dev1",
		NamespaceName: "application1",
		Ip:            "localhost:8889",
	}

	InitCustomConfig(func() (*AppConfig, error) {
		return readyConfig, nil
	})

	time.Sleep(1 * time.Second)

	config := GetAppConfig(nil)
	Assert(t, config, NotNilVal())
	Assert(t, "test1", Equal(config.AppId))
	Assert(t, "dev1", Equal(config.Cluster))
	Assert(t, "application1", Equal(config.NamespaceName))
	Assert(t, "localhost:8889", Equal(config.Ip))

	apolloConfig := GetCurrentApolloConfig()[config.NamespaceName]
	Assert(t, "test1", Equal(apolloConfig.AppId))
	Assert(t, "dev1", Equal(apolloConfig.Cluster))
	Assert(t, "application1", Equal(apolloConfig.NamespaceName))

	//revert file config
	initFileConfig()
}

func TestGetConfigUrl(t *testing.T) {
	appConfig := getTestAppConfig()
	url := getConfigUrl(appConfig)
	Assert(t, url, StartWith("http://localhost:8888/configs/test/dev/application?releaseKey=&ip="))
}

func TestGetConfigUrlByHost(t *testing.T) {
	appConfig := getTestAppConfig()
	url := getConfigUrlByHost(appConfig, "http://baidu.com/")
	Assert(t, url, StartWith("http://baidu.com/configs/test/dev/application?releaseKey=&ip="))
}

func TestGetServicesConfigUrl(t *testing.T) {
	appConfig := getTestAppConfig()
	url := getServicesConfigUrl(appConfig)
	ip := getInternal()
	Assert(t, "http://localhost:8888/services/config?appId=test&ip="+ip, Equal(url))
}

func getTestAppConfig() *AppConfig {
	jsonStr := `{
    "appId": "test",
    "cluster": "dev",
    "namespaceName": "application",
    "ip": "localhost:8888",
    "releaseKey": "1"
	}`
	config, _ := createAppConfigWithJson(jsonStr)

	return config
}

func TestSyncServerIpList(t *testing.T) {
	trySyncServerIpList(t)
}

func trySyncServerIpList(t *testing.T) {
	server := runMockServicesConfigServer()
	defer server.Close()

	newAppConfig := getTestAppConfig()
	newAppConfig.Ip = server.URL
	err := syncServerIpList(newAppConfig)

	Assert(t, err, NilVal())

	serverLen := getServersLen()

	Assert(t, 10, Equal(serverLen))

}

func TestSelectHost(t *testing.T) {
	//mock ip data
	trySyncServerIpList(t)

	t.Log("appconfig host:" + appConfig.getHost())
	t.Log("appconfig select host:" + appConfig.selectHost())

	host := "http://localhost:8888/"
	Assert(t, host, Equal(appConfig.getHost()))
	Assert(t, host, Equal(appConfig.selectHost()))

	//check select next time
	appConfig.setNextTryConnTime(5)
	Assert(t, host, NotEqual(appConfig.selectHost()))
	time.Sleep(6 * time.Second)
	Assert(t, host, Equal(appConfig.selectHost()))

	//check servers
	appConfig.setNextTryConnTime(5)
	firstHost := appConfig.selectHost()
	Assert(t, host, NotEqual(firstHost))
	setDownNode(firstHost)

	secondHost := appConfig.selectHost()
	Assert(t, host, NotEqual(secondHost))
	Assert(t, firstHost, NotEqual(secondHost))
	setDownNode(secondHost)

	thirdHost := appConfig.selectHost()
	Assert(t, host, NotEqual(thirdHost))
	Assert(t, firstHost, NotEqual(thirdHost))
	Assert(t, secondHost, NotEqual(thirdHost))

	servers.Range(func(k, v interface{}) bool {
		setDownNode(k.(string))
		return true
	})

	Assert(t, "", Equal(appConfig.selectHost()))

	//no servers
	//servers = make(map[string]*serverInfo, 0)
	deleteServers()
	Assert(t, "", Equal(appConfig.selectHost()))
}

func deleteServers() {
	servers.Range(func(k, v interface{}) bool {
		servers.Delete(k)
		return true
	})
}

func getServersLen() int {
	len := 0
	servers.Range(func(k, v interface{}) bool {
		len++
		return true
	})
	return len
}
<commit_msg>fix test<commit_after>package agollo

import (
	. "github.com/tevid/gohamcrest"
	"testing"
	"time"
)

func TestInit(t *testing.T) {
	config := GetAppConfig(nil)
	time.Sleep(1 * time.Second)

	Assert(t, config, NotNilVal())
	Assert(t, "test", Equal(config.AppId))
	Assert(t, "dev", Equal(config.Cluster))
	Assert(t, "application,abc1", Equal(config.NamespaceName))
	Assert(t, "localhost:8888", Equal(config.Ip))

	apolloConfig := GetCurrentApolloConfig()[defaultNamespace]
	Assert(t, "test", Equal(apolloConfig.AppId))
	Assert(t, "dev", Equal(apolloConfig.Cluster))
	Assert(t, "application", Equal(apolloConfig.NamespaceName))

}

func TestStructInit(t *testing.T) {

	readyConfig := &AppConfig{
		AppId:         "test1",
		Cluster:       "dev1",
		NamespaceName: "application1",
		Ip:            "localhost:8889",
	}

	InitCustomConfig(func() (*AppConfig, error) {
		return readyConfig, nil
	})

	time.Sleep(1 * time.Second)

	config := GetAppConfig(nil)
	Assert(t, config, NotNilVal())
	Assert(t, "test1", Equal(config.AppId))
	Assert(t, "dev1", Equal(config.Cluster))
	Assert(t, "application1", Equal(config.NamespaceName))
	Assert(t, "localhost:8889", Equal(config.Ip))

	apolloConfig := GetCurrentApolloConfig()[config.NamespaceName]
	Assert(t, "test1", Equal(apolloConfig.AppId))
	Assert(t, "dev1", Equal(apolloConfig.Cluster))
	Assert(t, "application1", Equal(apolloConfig.NamespaceName))

	//revert file config
	initFileConfig()
}

func TestGetConfigUrl(t *testing.T) {
	appConfig := getTestAppConfig()
	url := getConfigUrl(appConfig)
	Assert(t, url, StartWith("http://localhost:8888/configs/test/dev/application?releaseKey=&ip="))
}

func TestGetConfigUrlByHost(t *testing.T) {
	appConfig := getTestAppConfig()
	url := getConfigUrlByHost(appConfig, "http://baidu.com/")
	Assert(t, url, StartWith("http://baidu.com/configs/test/dev/application?releaseKey=&ip="))
}

func TestGetServicesConfigUrl(t *testing.T) {
	appConfig := getTestAppConfig()
	url := getServicesConfigUrl(appConfig)
	ip := getInternal()
	Assert(t, "http://localhost:8888/services/config?appId=test&ip="+ip, Equal(url))
}

func getTestAppConfig() *AppConfig {
	jsonStr := `{
    "appId": "test",
    "cluster": "dev",
    "namespaceName": "application",
    "ip": "localhost:8888",
    "releaseKey": "1"
	}`
	config, _ := createAppConfigWithJson(jsonStr)

	return config
}

func TestSyncServerIpList(t *testing.T) {
	trySyncServerIpList(t)
}

func trySyncServerIpList(t *testing.T) {
	server := runMockServicesConfigServer()
	defer server.Close()

	newAppConfig := getTestAppConfig()
	newAppConfig.Ip = server.URL
	err := syncServerIpList(newAppConfig)

	Assert(t, err, NilVal())

	serverLen := getServersLen()

	Assert(t, 10, Equal(serverLen))

}

func TestSelectHost(t *testing.T) {
	//mock ip data
	trySyncServerIpList(t)

	t.Log("appconfig host:" + appConfig.getHost())
	t.Log("appconfig select host:" + appConfig.selectHost())

	host := "http://localhost:8888/"
	Assert(t, host, Equal(appConfig.getHost()))
	Assert(t, host, Equal(appConfig.selectHost()))

	//check select next time
	appConfig.setNextTryConnTime(5)
	Assert(t, host, NotEqual(appConfig.selectHost()))
	time.Sleep(6 * time.Second)
	Assert(t, host, Equal(appConfig.selectHost()))

	//check servers
	appConfig.setNextTryConnTime(5)
	firstHost := appConfig.selectHost()
	Assert(t, host, NotEqual(firstHost))
	setDownNode(firstHost)

	secondHost := appConfig.selectHost()
	Assert(t, host, NotEqual(secondHost))
	Assert(t, firstHost, NotEqual(secondHost))
	setDownNode(secondHost)

	thirdHost := appConfig.selectHost()
	Assert(t, host, NotEqual(thirdHost))
	Assert(t, firstHost, NotEqual(thirdHost))
	Assert(t, secondHost, NotEqual(thirdHost))

	servers.Range(func(k, v interface{}) bool {
		setDownNode(k.(string))
		return true
	})

	Assert(t, "", Equal(appConfig.selectHost()))

	//no servers
	//servers = make(map[string]*serverInfo, 0)
	deleteServers()
	Assert(t, "", Equal(appConfig.selectHost()))
}

func deleteServers() {
	servers.Range(func(k, v interface{}) bool {
		servers.Delete(k)
		return true
	})
}

func getServersLen() int {
	len := 0
	servers.Range(func(k, v interface{}) bool {
		len++
		return true
	})
	return len
}
<|endoftext|>


<commit_before>package internal

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/onsi/ginkgo/v2/formatter"
	"github.com/onsi/ginkgo/v2/types"
)

type ProgressSignalRegistrar func(func()) context.CancelFunc

func RegisterForProgressSignal(handler func()) context.CancelFunc {
	signalChannel := make(chan os.Signal, 1)
	if len(PROGRESS_SIGNALS) > 0 {
		signal.Notify(signalChannel, PROGRESS_SIGNALS...)
	}
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		for {
			select {
			case <-signalChannel:
				handler()
			case <-ctx.Done():
				signal.Stop(signalChannel)
				return
			}
		}
	}()

	return cancel
}

type Goroutines []Goroutine

type Goroutine struct {
	ID    uint64
	State string
	Stack FunctionCalls
}

func (g Goroutine) Report(color string) string {
	if !g.Stack.HasHighlights() {
		color = "{{gray}}"
	}
	out := &strings.Builder{}
	fmt.Fprintf(out, "%sgoroutine %d [%s]{{/}}\n", color, g.ID, g.State)

	for _, functionCall := range g.Stack {
		fmt.Fprintf(out, "%s\n", functionCall.Report(color))
	}

	return out.String()
}

type FunctionCalls []FunctionCall

func (fcs FunctionCalls) HasHighlights() bool {
	for _, fc := range fcs {
		if fc.Highlight {
			return true
		}
	}
	return false
}

type FunctionCall struct {
	Function  string
	Filename  string
	Line      int64
	Highlight bool
}

func (fc FunctionCall) Report(color string) string {
	if fc.Highlight {
		out := fmt.Sprintf("{{bold}}%s> %s\n    %s:%d{{/}}", color, fc.Function, fc.Filename, fc.Line)
		src := sourceAt(fc.Filename, int(fc.Line), 2, color)
		if src != "" {
			out += "\n" + src
		}
		return out
	} else {
		return fmt.Sprintf("{{gray}}  %s\n    %s:%d{{/}}", fc.Function, fc.Filename, fc.Line)
	}
}

type ProgressStepCursor struct {
	Name         string
	CodeLocation types.CodeLocation
	StartTime    time.Time
}

type ProgressReport struct {
	CurrentSpecReport    types.SpecReport
	CurrentNode          Node
	CurrentNodeStartTime time.Time
	CurrentStep          ProgressStepCursor

	SpecGoroutine         Goroutine
	HighlightedGoroutines Goroutines
	OtherGoroutines       Goroutines
}

var cycleJoiner = formatter.New(formatter.ColorModePassthrough)

func (pc ProgressReport) Report(color string, includeAllGoroutines bool) string {
	now := time.Now()
	out := &strings.Builder{}
	indent := ""
	if pc.CurrentSpecReport.LeafNodeText != "" {
		if len(pc.CurrentSpecReport.ContainerHierarchyTexts) > 0 {
			out.WriteString(cycleJoiner.CycleJoin(pc.CurrentSpecReport.ContainerHierarchyTexts, " ", []string{"{{/}}", "{{gray}}"}))
			out.WriteString(" ")
		}

		fmt.Fprintf(out, "{{bold}}%s%s{{/}} (Spec Runtime: %s)\n", color, pc.CurrentSpecReport.LeafNodeText, now.Sub(pc.CurrentSpecReport.StartTime).Round(time.Millisecond))
		fmt.Fprintf(out, "  {{gray}}%s{{/}}\n", pc.CurrentSpecReport.LeafNodeLocation)
		indent += "  "
	}
	if !pc.CurrentNode.IsZero() {
		fmt.Fprintf(out, "%sIn {{bold}}%s[%s]{{/}}", indent, color, pc.CurrentNode.NodeType)
		if pc.CurrentNode.Text != "" && !pc.CurrentNode.NodeType.Is(types.NodeTypeIt) {
			fmt.Fprintf(out, " {{bold}}%s%s{{/}}", color, pc.CurrentNode.Text)
		}
		fmt.Fprintf(out, " (Node Runtime: %s)\n", now.Sub(pc.CurrentNodeStartTime).Round(time.Millisecond))
		fmt.Fprintf(out, "%s  {{gray}}%s{{/}}\n", indent, pc.CurrentNode.CodeLocation)
		indent += "  "
	}
	if pc.CurrentStep.Name != "" {
		fmt.Fprintf(out, "%sAt {{bold}}%s[By Step] %s{{/}} (Step Runtime: %s)\n", indent, color, pc.CurrentStep.Name, now.Sub(pc.CurrentStep.StartTime).Round(time.Millisecond))
		fmt.Fprintf(out, "%s  {{gray}}%s{{/}}\n", indent, pc.CurrentStep.CodeLocation)
	}

	out.WriteString("\n{{bold}}{{underline}}Spec Goroutine{{/}}\n")
	out.WriteString(pc.SpecGoroutine.Report(color))

	if len(pc.HighlightedGoroutines) > 0 {
		out.WriteString("\n{{bold}}{{underline}}Goroutines of Interest{{/}}\n")
		for _, goroutine := range pc.HighlightedGoroutines {
			out.WriteString(goroutine.Report(color))
			out.WriteString("\n")
		}
	}

	if includeAllGoroutines && len(pc.OtherGoroutines) > 0 {
		out.WriteString("\n{{gray}}{{bold}}{{underline}}Other Goroutines{{/}}\n")
		for _, goroutine := range pc.OtherGoroutines {
			out.WriteString(goroutine.Report(color))
			out.WriteString("\n")
		}
	}

	return out.String()
}

func NewProgressReport(report types.SpecReport, currentNode Node, currentNodeStartTime time.Time, currentStep ProgressStepCursor) (ProgressReport, error) {
	pc := ProgressReport{
		CurrentSpecReport:    report,
		CurrentNode:          currentNode,
		CurrentNodeStartTime: currentNodeStartTime,
		CurrentStep:          currentStep,
	}

	goroutines, err := extractRunningGoroutines()
	if err != nil {
		return pc, err
	}

	// now we want to try to find goroutines of interest.  these will be goroutines that have any function calls with code in packagesOfInterest:
	packagesOfInterest := map[string]bool{}
	addPackageFor := func(filename string) {
		if filename != "" {
			packagesOfInterest[packageFromFilename(filename)] = true
		}
	}
	for _, location := range report.ContainerHierarchyLocations {
		addPackageFor(location.FileName)
	}
	addPackageFor(report.LeafNodeLocation.FileName)
	addPackageFor(currentNode.CodeLocation.FileName)
	addPackageFor(currentStep.CodeLocation.FileName)

	//First, we find the SpecGoroutine - this will be the goroutine that includes `runNode`
	specGoRoutineIdx := -1
	runNodeFunctionCallIdx := -1
OUTER:
	for goroutineIdx, goroutine := range goroutines {
		for functionCallIdx, functionCall := range goroutine.Stack {
			if strings.Contains(functionCall.Function, "ginkgo/v2/internal.(*Suite).runNode.func") {
				specGoRoutineIdx = goroutineIdx
				runNodeFunctionCallIdx = functionCallIdx
				break OUTER
			}
		}
	}

	//Now, we find the first non-Ginkgo function call
	if specGoRoutineIdx > -1 {
		for runNodeFunctionCallIdx >= 0 {
			if strings.Contains(goroutines[specGoRoutineIdx].Stack[runNodeFunctionCallIdx].Function, "ginkgo/v2/internal") {
				runNodeFunctionCallIdx--
				continue
			}
			//found it!  lets add its package of interest
			addPackageFor(goroutines[specGoRoutineIdx].Stack[runNodeFunctionCallIdx].Filename)
			break
		}
	}

	// Now we go through all goroutines and highlight any lines with packages in `packagesOfInterest`
	// Any goroutines with highlighted lines end up in the HighlightGoRoutines
	for goroutineIdx, goroutine := range goroutines {
		hasHighlights := false
		isGinkgoEntryPoint := false
		for functionCallIdx, functionCall := range goroutine.Stack {
			if strings.Contains(functionCall.Function, "ginkgo/v2.RunSpecs") {
				isGinkgoEntryPoint = true
				break
			}
			if packagesOfInterest[packageFromFilename(functionCall.Filename)] {
				goroutine.Stack[functionCallIdx].Highlight = true
				hasHighlights = true
			}
		}
		if goroutineIdx == specGoRoutineIdx {
			pc.SpecGoroutine = goroutine
		} else if hasHighlights && !isGinkgoEntryPoint {
			pc.HighlightedGoroutines = append(pc.HighlightedGoroutines, goroutine)
		} else {
			pc.OtherGoroutines = append(pc.OtherGoroutines, goroutine)
		}
	}

	return pc, nil
}

func extractRunningGoroutines() (Goroutines, error) {
	var stack []byte
	for size := 64 * 1024; ; size *= 2 {
		stack = make([]byte, size)
		if n := runtime.Stack(stack, true); n < size {
			stack = stack[:n]
			break
		}
	}

	r := bufio.NewReader(bytes.NewReader(stack))
	out := Goroutines{}
	idx := -1
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}

		line = strings.TrimSuffix(line, "\n")

		//skip blank lines
		if line == "" {
			continue
		}

		//parse headers for new goroutine frames
		if strings.HasPrefix(line, "goroutine") {
			out = append(out, Goroutine{})
			idx = len(out) - 1

			line = strings.TrimPrefix(line, "goroutine ")
			line = strings.TrimSuffix(line, ":")
			fields := strings.SplitN(line, " ", 2)
			if len(fields) != 2 {
				return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid goroutine frame header: %s", line))
			}
			out[idx].ID, err = strconv.ParseUint(fields[0], 10, 64)
			if err != nil {
				return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid goroutine ID: %s", fields[1]))
			}

			out[idx].State = strings.TrimSuffix(strings.TrimPrefix(fields[1], "["), "]")
			continue
		}

		//if we are here we must be at a function call entry in the stack
		functionCall := FunctionCall{
			Function: strings.TrimPrefix(line, "created by "), // no need to track 'created by'
		}

		line, err = r.ReadString('\n')
		line = strings.TrimSuffix(line, "\n")
		if err == io.EOF {
			return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid function call: %s -- missing file name and line number", functionCall.Function))
		}
		line = strings.TrimLeft(line, " \t")
		fields := strings.SplitN(line, ":", 2)
		if len(fields) != 2 {
			return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid filename nad line number: %s", line))
		}
		functionCall.Filename = fields[0]
		line = strings.Split(fields[1], " ")[0]
		functionCall.Line, err = strconv.ParseInt(line, 10, 64)
		if err != nil {
			return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid function call line number: %s\n%s", line, err.Error()))
		}
		out[idx].Stack = append(out[idx].Stack, functionCall)
	}

	return out, nil
}

var _SOURCE_CACHE = map[string][]string{}

func sourceAt(filename string, lineNumber int, span int, color string) string {
	if filename == "" {
		return ""
	}
	var lines []string
	var ok bool
	if lines, ok = _SOURCE_CACHE[filename]; !ok {
		data, err := os.ReadFile(filename)
		if err != nil {
			return ""
		}
		lines = strings.Split(string(data), "\n")
		_SOURCE_CACHE[filename] = lines
	}
	idx := lineNumber - span - 1
	out := []string{}
	for idx < lineNumber+span {
		if idx >= 0 && idx <= len(lines)-1 {
			if idx == lineNumber-1 {
				out = append(out, "      {{bold}}"+color+"> "+lines[idx]+"{{/}}")
			} else {
				out = append(out, "      | "+lines[idx])
			}
		}
		idx++
	}
	return strings.Join(out, "\n")
}

func packageFromFilename(fname string) string {
	return filepath.Dir(fname)
}
<commit_msg>allow subpackages in heuristic for determining which packages to highlight in a progres report<commit_after>package internal

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/onsi/ginkgo/v2/formatter"
	"github.com/onsi/ginkgo/v2/types"
)

type ProgressSignalRegistrar func(func()) context.CancelFunc

func RegisterForProgressSignal(handler func()) context.CancelFunc {
	signalChannel := make(chan os.Signal, 1)
	if len(PROGRESS_SIGNALS) > 0 {
		signal.Notify(signalChannel, PROGRESS_SIGNALS...)
	}
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		for {
			select {
			case <-signalChannel:
				handler()
			case <-ctx.Done():
				signal.Stop(signalChannel)
				return
			}
		}
	}()

	return cancel
}

type Goroutines []Goroutine

type Goroutine struct {
	ID    uint64
	State string
	Stack FunctionCalls
}

func (g Goroutine) Report(color string) string {
	if !g.Stack.HasHighlights() {
		color = "{{gray}}"
	}
	out := &strings.Builder{}
	fmt.Fprintf(out, "%sgoroutine %d [%s]{{/}}\n", color, g.ID, g.State)

	for _, functionCall := range g.Stack {
		fmt.Fprintf(out, "%s\n", functionCall.Report(color))
	}

	return out.String()
}

type FunctionCalls []FunctionCall

func (fcs FunctionCalls) HasHighlights() bool {
	for _, fc := range fcs {
		if fc.Highlight {
			return true
		}
	}
	return false
}

type FunctionCall struct {
	Function  string
	Filename  string
	Line      int64
	Highlight bool
}

func (fc FunctionCall) Report(color string) string {
	if fc.Highlight {
		out := fmt.Sprintf("{{bold}}%s> %s\n    %s:%d{{/}}", color, fc.Function, fc.Filename, fc.Line)
		src := sourceAt(fc.Filename, int(fc.Line), 2, color)
		if src != "" {
			out += "\n" + src
		}
		return out
	} else {
		return fmt.Sprintf("{{gray}}  %s\n    %s:%d{{/}}", fc.Function, fc.Filename, fc.Line)
	}
}

type ProgressStepCursor struct {
	Name         string
	CodeLocation types.CodeLocation
	StartTime    time.Time
}

type ProgressReport struct {
	CurrentSpecReport    types.SpecReport
	CurrentNode          Node
	CurrentNodeStartTime time.Time
	CurrentStep          ProgressStepCursor

	SpecGoroutine         Goroutine
	HighlightedGoroutines Goroutines
	OtherGoroutines       Goroutines
}

var cycleJoiner = formatter.New(formatter.ColorModePassthrough)

func (pc ProgressReport) Report(color string, includeAllGoroutines bool) string {
	now := time.Now()
	out := &strings.Builder{}
	indent := ""
	if pc.CurrentSpecReport.LeafNodeText != "" {
		if len(pc.CurrentSpecReport.ContainerHierarchyTexts) > 0 {
			out.WriteString(cycleJoiner.CycleJoin(pc.CurrentSpecReport.ContainerHierarchyTexts, " ", []string{"{{/}}", "{{gray}}"}))
			out.WriteString(" ")
		}

		fmt.Fprintf(out, "{{bold}}%s%s{{/}} (Spec Runtime: %s)\n", color, pc.CurrentSpecReport.LeafNodeText, now.Sub(pc.CurrentSpecReport.StartTime).Round(time.Millisecond))
		fmt.Fprintf(out, "  {{gray}}%s{{/}}\n", pc.CurrentSpecReport.LeafNodeLocation)
		indent += "  "
	}
	if !pc.CurrentNode.IsZero() {
		fmt.Fprintf(out, "%sIn {{bold}}%s[%s]{{/}}", indent, color, pc.CurrentNode.NodeType)
		if pc.CurrentNode.Text != "" && !pc.CurrentNode.NodeType.Is(types.NodeTypeIt) {
			fmt.Fprintf(out, " {{bold}}%s%s{{/}}", color, pc.CurrentNode.Text)
		}
		fmt.Fprintf(out, " (Node Runtime: %s)\n", now.Sub(pc.CurrentNodeStartTime).Round(time.Millisecond))
		fmt.Fprintf(out, "%s  {{gray}}%s{{/}}\n", indent, pc.CurrentNode.CodeLocation)
		indent += "  "
	}
	if pc.CurrentStep.Name != "" {
		fmt.Fprintf(out, "%sAt {{bold}}%s[By Step] %s{{/}} (Step Runtime: %s)\n", indent, color, pc.CurrentStep.Name, now.Sub(pc.CurrentStep.StartTime).Round(time.Millisecond))
		fmt.Fprintf(out, "%s  {{gray}}%s{{/}}\n", indent, pc.CurrentStep.CodeLocation)
	}

	out.WriteString("\n{{bold}}{{underline}}Spec Goroutine{{/}}\n")
	out.WriteString(pc.SpecGoroutine.Report(color))

	if len(pc.HighlightedGoroutines) > 0 {
		out.WriteString("\n{{bold}}{{underline}}Goroutines of Interest{{/}}\n")
		for _, goroutine := range pc.HighlightedGoroutines {
			out.WriteString(goroutine.Report(color))
			out.WriteString("\n")
		}
	}

	if includeAllGoroutines && len(pc.OtherGoroutines) > 0 {
		out.WriteString("\n{{gray}}{{bold}}{{underline}}Other Goroutines{{/}}\n")
		for _, goroutine := range pc.OtherGoroutines {
			out.WriteString(goroutine.Report(color))
			out.WriteString("\n")
		}
	}

	return out.String()
}

func NewProgressReport(report types.SpecReport, currentNode Node, currentNodeStartTime time.Time, currentStep ProgressStepCursor) (ProgressReport, error) {
	pc := ProgressReport{
		CurrentSpecReport:    report,
		CurrentNode:          currentNode,
		CurrentNodeStartTime: currentNodeStartTime,
		CurrentStep:          currentStep,
	}

	goroutines, err := extractRunningGoroutines()
	if err != nil {
		return pc, err
	}

	// now we want to try to find goroutines of interest.  these will be goroutines that have any function calls with code in packagesOfInterest:
	packagesOfInterest := map[string]bool{}
	addPackageFor := func(filename string) {
		if filename != "" {
			packagesOfInterest[packageFromFilename(filename)] = true
		}
	}
	isPackageOfInterest := func(filename string) bool {
		stackPackage := packageFromFilename(filename)
		for packageOfInterest := range packagesOfInterest {
			if strings.HasPrefix(stackPackage, packageOfInterest) {
				return true
			}
		}
		return false
	}
	for _, location := range report.ContainerHierarchyLocations {
		addPackageFor(location.FileName)
	}
	addPackageFor(report.LeafNodeLocation.FileName)
	addPackageFor(currentNode.CodeLocation.FileName)
	addPackageFor(currentStep.CodeLocation.FileName)

	//First, we find the SpecGoroutine - this will be the goroutine that includes `runNode`
	specGoRoutineIdx := -1
	runNodeFunctionCallIdx := -1
OUTER:
	for goroutineIdx, goroutine := range goroutines {
		for functionCallIdx, functionCall := range goroutine.Stack {
			if strings.Contains(functionCall.Function, "ginkgo/v2/internal.(*Suite).runNode.func") {
				specGoRoutineIdx = goroutineIdx
				runNodeFunctionCallIdx = functionCallIdx
				break OUTER
			}
		}
	}

	//Now, we find the first non-Ginkgo function call
	if specGoRoutineIdx > -1 {
		for runNodeFunctionCallIdx >= 0 {
			if strings.Contains(goroutines[specGoRoutineIdx].Stack[runNodeFunctionCallIdx].Function, "ginkgo/v2/internal") {
				runNodeFunctionCallIdx--
				continue
			}
			//found it!  lets add its package of interest
			addPackageFor(goroutines[specGoRoutineIdx].Stack[runNodeFunctionCallIdx].Filename)
			break
		}
	}

	// Now we go through all goroutines and highlight any lines with packages in `packagesOfInterest`
	// Any goroutines with highlighted lines end up in the HighlightGoRoutines
	for goroutineIdx, goroutine := range goroutines {
		hasHighlights := false
		isGinkgoEntryPoint := false
		for functionCallIdx, functionCall := range goroutine.Stack {
			if strings.Contains(functionCall.Function, "ginkgo/v2.RunSpecs") {
				isGinkgoEntryPoint = true
				break
			}
			if isPackageOfInterest(functionCall.Filename) {
				goroutine.Stack[functionCallIdx].Highlight = true
				hasHighlights = true
			}
		}
		if goroutineIdx == specGoRoutineIdx {
			pc.SpecGoroutine = goroutine
		} else if hasHighlights && !isGinkgoEntryPoint {
			pc.HighlightedGoroutines = append(pc.HighlightedGoroutines, goroutine)
		} else {
			pc.OtherGoroutines = append(pc.OtherGoroutines, goroutine)
		}
	}

	return pc, nil
}

func extractRunningGoroutines() (Goroutines, error) {
	var stack []byte
	for size := 64 * 1024; ; size *= 2 {
		stack = make([]byte, size)
		if n := runtime.Stack(stack, true); n < size {
			stack = stack[:n]
			break
		}
	}

	r := bufio.NewReader(bytes.NewReader(stack))
	out := Goroutines{}
	idx := -1
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}

		line = strings.TrimSuffix(line, "\n")

		//skip blank lines
		if line == "" {
			continue
		}

		//parse headers for new goroutine frames
		if strings.HasPrefix(line, "goroutine") {
			out = append(out, Goroutine{})
			idx = len(out) - 1

			line = strings.TrimPrefix(line, "goroutine ")
			line = strings.TrimSuffix(line, ":")
			fields := strings.SplitN(line, " ", 2)
			if len(fields) != 2 {
				return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid goroutine frame header: %s", line))
			}
			out[idx].ID, err = strconv.ParseUint(fields[0], 10, 64)
			if err != nil {
				return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid goroutine ID: %s", fields[1]))
			}

			out[idx].State = strings.TrimSuffix(strings.TrimPrefix(fields[1], "["), "]")
			continue
		}

		//if we are here we must be at a function call entry in the stack
		functionCall := FunctionCall{
			Function: strings.TrimPrefix(line, "created by "), // no need to track 'created by'
		}

		line, err = r.ReadString('\n')
		line = strings.TrimSuffix(line, "\n")
		if err == io.EOF {
			return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid function call: %s -- missing file name and line number", functionCall.Function))
		}
		line = strings.TrimLeft(line, " \t")
		fields := strings.SplitN(line, ":", 2)
		if len(fields) != 2 {
			return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid filename nad line number: %s", line))
		}
		functionCall.Filename = fields[0]
		line = strings.Split(fields[1], " ")[0]
		functionCall.Line, err = strconv.ParseInt(line, 10, 64)
		if err != nil {
			return nil, types.GinkgoErrors.FailedToParseStackTrace(fmt.Sprintf("Invalid function call line number: %s\n%s", line, err.Error()))
		}
		out[idx].Stack = append(out[idx].Stack, functionCall)
	}

	return out, nil
}

var _SOURCE_CACHE = map[string][]string{}

func sourceAt(filename string, lineNumber int, span int, color string) string {
	if filename == "" {
		return ""
	}
	var lines []string
	var ok bool
	if lines, ok = _SOURCE_CACHE[filename]; !ok {
		data, err := os.ReadFile(filename)
		if err != nil {
			return ""
		}
		lines = strings.Split(string(data), "\n")
		_SOURCE_CACHE[filename] = lines
	}
	idx := lineNumber - span - 1
	out := []string{}
	for idx < lineNumber+span {
		if idx >= 0 && idx <= len(lines)-1 {
			if idx == lineNumber-1 {
				out = append(out, "      {{bold}}"+color+"> "+lines[idx]+"{{/}}")
			} else {
				out = append(out, "      | "+lines[idx])
			}
		}
		idx++
	}
	return strings.Join(out, "\n")
}

func packageFromFilename(fname string) string {
	return filepath.Dir(fname)
}
<|endoftext|>


<commit_before>package nes

import (
	"image"
	"log"
)

type NES struct {
	CPU         *CPU
	PPU         *PPU
	CPUMemory   Memory
	PPUMemory   Memory
	Mapper      Mapper
	RAM         []byte
	Cartridge   *Cartridge
	Controller1 *Controller
	Controller2 *Controller
}

func NewNES(path string) (*NES, error) {
	cartridge, err := LoadNESFile(path)
	if err != nil {
		return nil, err
	}
	ram := make([]byte, 2048)
	controller1 := NewController()
	controller2 := NewController()
	mapper := NewMapper(cartridge)
	nes := NES{
		nil, nil, nil, nil,
		mapper, ram, cartridge, controller1, controller2}
	nes.CPUMemory = NewCPUMemory(&nes)
	nes.PPUMemory = NewPPUMemory(&nes)
	nes.CPU = NewCPU(&nes)
	nes.PPU = NewPPU(&nes)
	return &nes, nil
}

func (nes *NES) Step() int {
	cpuCycles := nes.CPU.Step()
	ppuCycles := cpuCycles * 3
	for i := 0; i < ppuCycles; i++ {
		nes.PPU.Step()
	}
	return cpuCycles
}

func (nes *NES) StepFrame() {
	frame := nes.PPU.Frame
	for frame == nes.PPU.Frame {
		nes.Step()
	}
}

func (nes *NES) Buffer() *image.RGBA {
	return nes.PPU.buffer
}

func (nes *NES) SetPressed(controller, button int, pressed bool) {
	switch controller {
	case 1:
		nes.Controller1.SetPressed(button, pressed)
	case 2:
		nes.Controller2.SetPressed(button, pressed)
	default:
		log.Fatalf("unhandled controller press: %d", controller)
	}
}
<commit_msg>NES.BackgroundColor<commit_after>package nes

import (
	"image"
	"image/color"
	"log"
)

type NES struct {
	CPU         *CPU
	PPU         *PPU
	CPUMemory   Memory
	PPUMemory   Memory
	Mapper      Mapper
	RAM         []byte
	Cartridge   *Cartridge
	Controller1 *Controller
	Controller2 *Controller
}

func NewNES(path string) (*NES, error) {
	cartridge, err := LoadNESFile(path)
	if err != nil {
		return nil, err
	}
	ram := make([]byte, 2048)
	controller1 := NewController()
	controller2 := NewController()
	mapper := NewMapper(cartridge)
	nes := NES{
		nil, nil, nil, nil,
		mapper, ram, cartridge, controller1, controller2}
	nes.CPUMemory = NewCPUMemory(&nes)
	nes.PPUMemory = NewPPUMemory(&nes)
	nes.CPU = NewCPU(&nes)
	nes.PPU = NewPPU(&nes)
	return &nes, nil
}

func (nes *NES) Step() int {
	cpuCycles := nes.CPU.Step()
	ppuCycles := cpuCycles * 3
	for i := 0; i < ppuCycles; i++ {
		nes.PPU.Step()
	}
	return cpuCycles
}

func (nes *NES) StepFrame() {
	frame := nes.PPU.Frame
	for frame == nes.PPU.Frame {
		nes.Step()
	}
}

func (nes *NES) Buffer() *image.RGBA {
	return nes.PPU.buffer
}

func (nes *NES) BackgroundColor() color.RGBA {
	return palette[nes.PPU.readPalette(0)%64]
}

func (nes *NES) SetPressed(controller, button int, pressed bool) {
	switch controller {
	case 1:
		nes.Controller1.SetPressed(button, pressed)
	case 2:
		nes.Controller2.SetPressed(button, pressed)
	default:
		log.Fatalf("unhandled controller press: %d", controller)
	}
}
<|endoftext|>


<commit_before>package gitstats

import (
	"fmt"
	"time"

	"github.com/google/go-github/github"
	. "github.com/intelsdi-x/snap-plugin-utilities/logger"
	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
	"golang.org/x/oauth2"
)

const (
	// Name of plugin
	Name = "rt-gitstats"
	// Version of plugin
	Version = 1
	// Type of plugin
	Type = plugin.CollectorPluginType
)

// make sure that we actually satisify requierd interface
var _ plugin.CollectorPlugin = (*Gitstats)(nil)

var (
	repoMetricNames = []string{
		"forks",
		"issues",
		"network",
		"stars",
		"subscribers",
		"watches",
		"size",
	}
	userMetricNames = []string{
		"public_repos",
		"public_gists",
		"followers",
		"following",
		"private_repos",
		"private_gists",
		"plan_private_repos",
		"plan_seats",
		"plan_filled_seats",
	}
)

type Gitstats struct {
}

// CollectMetrics collects metrics for testing
func (f *Gitstats) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var err error

	conf := mts[0].Config().Table()
	accessToken, ok := conf["access_token"]
	if !ok || accessToken.(ctypes.ConfigValueStr).Value == "" {
		return nil, fmt.Errorf("access token missing from config, %v", conf)
	}

	metrics, err := gitStats(accessToken.(ctypes.ConfigValueStr).Value, mts)
	if err != nil {
		return nil, err
	}

	return metrics, nil
}

type repoName struct {
	Repo  string
	Owner string
}

func gitStats(accessToken string, mts []plugin.MetricType) ([]plugin.MetricType, error) {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accessToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)
	collectionTime := time.Now()
	repos := make(map[string]map[string]map[string]int)
	users := make(map[string]map[string]int)

	userRepos := make(map[string]struct{})
	authUser := ""
	useRepo := ""
	conf := mts[0].Config().Table()
	confUser, ok := conf["user"]
	if ok && confUser.(ctypes.ConfigValueStr).Value != "" {
		authUser = confUser.(ctypes.ConfigValueStr).Value
	}
	confRepo, ok := conf["repo"]
	if ok && confRepo.(ctypes.ConfigValueStr).Value != "" {
		useRepo = confRepo.(ctypes.ConfigValueStr).Value
	}

	metrics := make([]plugin.MetricType, 0)

	for _, m := range mts {
		ns := m.Namespace().Strings()
		switch ns[3] {
		case "repo":
			user := ns[4]
			repo := ns[5]
			stat := ns[6]

			if user == "*" {
				//need to get user
				if authUser == "" {
					gitUser, _, err := client.Users.Get("")
					if err != nil {
						LogError("failed to get authenticated user.", err)
						return nil, err
					}
					stats, err := userStats(gitUser, client)
					if err != nil {
						LogError("failed to get stats from user object.", err)
						return nil, err
					}
					users[*gitUser.Login] = stats
					authUser = *gitUser.Login
				}
				user = authUser
			}
			if repo == "*" && useRepo == "" {
				// get list of all repos owned by the user.
				if _, ok := userRepos[user]; !ok {
					opt := &github.RepositoryListOptions{Type: "owner"}
					repoList, _, err := client.Repositories.List(user, opt)
					if err != nil {
						LogError("failed to get repos owned by user.", err)
						return nil, err
					}
					userRepos[user] = struct{}{}
					if _, ok := repos[user]; !ok {
						repos[user] = make(map[string]map[string]int)
					}
					for _, r := range repoList {
						stats, err := repoStats(r)
						if err != nil {
							LogError("failed to get stats from repo object.", err)
							return nil, err
						}
						repos[user][*r.Name] = stats
					}
				}
				for repo, stats := range repos[user] {
					mt := plugin.MetricType{
						Data_:      stats[stat],
						Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", user, repo, stat),
						Timestamp_: collectionTime,
						Version_:   m.Version(),
					}
					metrics = append(metrics, mt)
				}
			} else {
				if repo == "*" {
					repo = useRepo
				}
				if _, ok := repos[user]; !ok {
					repos[user] = make(map[string]map[string]int)
				}
				if _, ok := repos[user][repo]; !ok {
					r, _, err := client.Repositories.Get(user, repo)
					if err != nil {
						LogError("failed to user repos.", err)
						return nil, err
					}
					stats, err := repoStats(r)
					if err != nil {
						LogError("failed to get stats from repo object.", err)
						return nil, err
					}
					repos[user][repo] = stats
				}
				mt := plugin.MetricType{
					Data_:      repos[user][repo][stat],
					Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", user, repo, stat),
					Timestamp_: collectionTime,
					Version_:   m.Version(),
				}
				metrics = append(metrics, mt)
			}

		case "user":
			user := ns[4]
			stat := ns[5]
			if user == "*" {
				//need to get user
				if authUser == "" {
					gitUser, _, err := client.Users.Get(user)
					if err != nil {
						LogError("failed to get authenticated user.", err)
						return nil, err
					}
					authUser = *gitUser.Login
					stats, err := userStats(gitUser, client)
					if err != nil {
						LogError("failed to get stats from user object", err)
						return nil, err
					}
					users[*gitUser.Login] = stats
				}
			} else {
				if _, ok := users[user]; !ok {
					u, _, err := client.Users.Get(user)
					if err != nil {
						LogError("failed to lookup user.", err)
						return nil, err
					}
					stats, err := userStats(u, client)
					if err != nil {
						LogError("failed to get stats from user object.", err)
						return nil, err
					}
					users[user] = stats
				}
			}
			mt := plugin.MetricType{
				Data_:      users[user][stat],
				Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "user", user, stat),
				Timestamp_: collectionTime,
				Version_:   m.Version(),
			}
			metrics = append(metrics, mt)
		}
	}

	return metrics, nil
}

func userStats(user *github.User, client *github.Client) (map[string]int, error) {
	stats := make(map[string]int)
	if user.PublicRepos != nil {
		stats["public_repos"] = *user.PublicRepos
	}
	if user.PublicGists != nil {
		stats["public_gists"] = *user.PublicGists
	}
	if user.Followers != nil {
		stats["followers"] = *user.Followers
	}
	if user.Following != nil {
		stats["following"] = *user.Following
	}

	if *user.Type == "Organization" {
		org, _, err := client.Organizations.Get(*user.Login)
		if err != nil {
			LogError("failed to lookup org data.", err)
			return nil, err
		}
		if org.PrivateGists != nil {
			stats["private_gists"] = *org.PrivateGists
		}
		if org.TotalPrivateRepos != nil {
			stats["private_repos"] = *org.TotalPrivateRepos
		}
		if org.DiskUsage != nil {
			stats["disk_usage"] = *org.DiskUsage
		}
	}

	return stats, nil
}

func repoStats(resp *github.Repository) (map[string]int, error) {
	stats := make(map[string]int)

	if resp.ForksCount != nil {
		stats["forks"] = *resp.ForksCount
	}
	if resp.OpenIssuesCount != nil {
		stats["issues"] = *resp.OpenIssuesCount
	}
	if resp.NetworkCount != nil {
		stats["network"] = *resp.NetworkCount
	}
	if resp.StargazersCount != nil {
		stats["stars"] = *resp.StargazersCount
	}
	if resp.SubscribersCount != nil {
		stats["subcribers"] = *resp.SubscribersCount
	}
	if resp.WatchersCount != nil {
		stats["watchers"] = *resp.WatchersCount
	}
	if resp.Size != nil {
		stats["size"] = *resp.Size
	}
	return stats, nil
}

//GetMetricTypes returns metric types for testing
func (f *Gitstats) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType, error) {
	mts := make([]plugin.MetricType, 0)
	for _, metricName := range repoMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo").
				AddDynamicElement("owner", "repository owner").
				AddDynamicElement("repo", "repository name").
				AddStaticElement(metricName),
			Config_: cfg.ConfigDataNode,
		})
	}
	for _, metricName := range userMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "user").
				AddDynamicElement("user", "user or orginisation name").
				AddStaticElement(metricName),
			Config_: cfg.ConfigDataNode,
		})
	}
	return mts, nil
}

//GetConfigPolicy returns a ConfigPolicyTree for testing
func (f *Gitstats) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("access_token", true)
	rule1, _ := cpolicy.NewStringRule("user", false, "")
	rule2, _ := cpolicy.NewStringRule("repo", false, "")

	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	p.Add(rule1)
	p.Add(rule2)
	c.Add([]string{"raintank", "apps", "gitstats"}, p)
	return c, nil
}

//Meta returns meta data for testing
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		Name,
		Version,
		Type,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Unsecure(true),
		plugin.ConcurrencyCount(1000),
	)
}
<commit_msg>set user correctly in metric namespace<commit_after>package gitstats

import (
	"fmt"
	"time"

	"github.com/google/go-github/github"
	. "github.com/intelsdi-x/snap-plugin-utilities/logger"
	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
	"golang.org/x/oauth2"
)

const (
	// Name of plugin
	Name = "rt-gitstats"
	// Version of plugin
	Version = 1
	// Type of plugin
	Type = plugin.CollectorPluginType
)

// make sure that we actually satisify requierd interface
var _ plugin.CollectorPlugin = (*Gitstats)(nil)

var (
	repoMetricNames = []string{
		"forks",
		"issues",
		"network",
		"stars",
		"subscribers",
		"watches",
		"size",
	}
	userMetricNames = []string{
		"public_repos",
		"public_gists",
		"followers",
		"following",
		"private_repos",
		"private_gists",
		"plan_private_repos",
		"plan_seats",
		"plan_filled_seats",
	}
)

type Gitstats struct {
}

// CollectMetrics collects metrics for testing
func (f *Gitstats) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var err error

	conf := mts[0].Config().Table()
	accessToken, ok := conf["access_token"]
	if !ok || accessToken.(ctypes.ConfigValueStr).Value == "" {
		return nil, fmt.Errorf("access token missing from config, %v", conf)
	}

	metrics, err := gitStats(accessToken.(ctypes.ConfigValueStr).Value, mts)
	if err != nil {
		return nil, err
	}

	return metrics, nil
}

type repoName struct {
	Repo  string
	Owner string
}

func gitStats(accessToken string, mts []plugin.MetricType) ([]plugin.MetricType, error) {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accessToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)
	collectionTime := time.Now()
	repos := make(map[string]map[string]map[string]int)
	users := make(map[string]map[string]int)

	userRepos := make(map[string]struct{})
	authUser := ""
	useRepo := ""
	conf := mts[0].Config().Table()
	confUser, ok := conf["user"]
	if ok && confUser.(ctypes.ConfigValueStr).Value != "" {
		authUser = confUser.(ctypes.ConfigValueStr).Value
	}
	confRepo, ok := conf["repo"]
	if ok && confRepo.(ctypes.ConfigValueStr).Value != "" {
		useRepo = confRepo.(ctypes.ConfigValueStr).Value
	}

	metrics := make([]plugin.MetricType, 0)

	for _, m := range mts {
		ns := m.Namespace().Strings()
		switch ns[3] {
		case "repo":
			user := ns[4]
			repo := ns[5]
			stat := ns[6]

			if user == "*" {
				//need to get user
				if authUser == "" {
					gitUser, _, err := client.Users.Get("")
					if err != nil {
						LogError("failed to get authenticated user.", err)
						return nil, err
					}
					stats, err := userStats(gitUser, client)
					if err != nil {
						LogError("failed to get stats from user object.", err)
						return nil, err
					}
					users[*gitUser.Login] = stats
					authUser = *gitUser.Login
				}
				user = authUser
			}
			if repo == "*" && useRepo == "" {
				// get list of all repos owned by the user.
				if _, ok := userRepos[user]; !ok {
					opt := &github.RepositoryListOptions{Type: "owner"}
					repoList, _, err := client.Repositories.List(user, opt)
					if err != nil {
						LogError("failed to get repos owned by user.", err)
						return nil, err
					}
					userRepos[user] = struct{}{}
					if _, ok := repos[user]; !ok {
						repos[user] = make(map[string]map[string]int)
					}
					for _, r := range repoList {
						stats, err := repoStats(r)
						if err != nil {
							LogError("failed to get stats from repo object.", err)
							return nil, err
						}
						repos[user][*r.Name] = stats
					}
				}
				for repo, stats := range repos[user] {
					mt := plugin.MetricType{
						Data_:      stats[stat],
						Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", user, repo, stat),
						Timestamp_: collectionTime,
						Version_:   m.Version(),
					}
					metrics = append(metrics, mt)
				}
			} else {
				if repo == "*" {
					repo = useRepo
				}
				if _, ok := repos[user]; !ok {
					repos[user] = make(map[string]map[string]int)
				}
				if _, ok := repos[user][repo]; !ok {
					r, _, err := client.Repositories.Get(user, repo)
					if err != nil {
						LogError("failed to user repos.", err)
						return nil, err
					}
					stats, err := repoStats(r)
					if err != nil {
						LogError("failed to get stats from repo object.", err)
						return nil, err
					}
					repos[user][repo] = stats
				}
				mt := plugin.MetricType{
					Data_:      repos[user][repo][stat],
					Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", user, repo, stat),
					Timestamp_: collectionTime,
					Version_:   m.Version(),
				}
				metrics = append(metrics, mt)
			}

		case "user":
			user := ns[4]
			stat := ns[5]
			if user == "*" {
				//need to get user
				if authUser == "" {
					gitUser, _, err := client.Users.Get(user)
					if err != nil {
						LogError("failed to get authenticated user.", err)
						return nil, err
					}
					authUser = *gitUser.Login
					user = authUser
					stats, err := userStats(gitUser, client)
					if err != nil {
						LogError("failed to get stats from user object", err)
						return nil, err
					}
					users[user] = stats
				}
			} else {
				if _, ok := users[user]; !ok {
					u, _, err := client.Users.Get(user)
					if err != nil {
						LogError("failed to lookup user.", err)
						return nil, err
					}
					stats, err := userStats(u, client)
					if err != nil {
						LogError("failed to get stats from user object.", err)
						return nil, err
					}
					users[user] = stats
				}
			}
			mt := plugin.MetricType{
				Data_:      users[user][stat],
				Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "user", user, stat),
				Timestamp_: collectionTime,
				Version_:   m.Version(),
			}
			metrics = append(metrics, mt)
		}
	}

	return metrics, nil
}

func userStats(user *github.User, client *github.Client) (map[string]int, error) {
	stats := make(map[string]int)
	if user.PublicRepos != nil {
		stats["public_repos"] = *user.PublicRepos
	}
	if user.PublicGists != nil {
		stats["public_gists"] = *user.PublicGists
	}
	if user.Followers != nil {
		stats["followers"] = *user.Followers
	}
	if user.Following != nil {
		stats["following"] = *user.Following
	}

	if *user.Type == "Organization" {
		org, _, err := client.Organizations.Get(*user.Login)
		if err != nil {
			LogError("failed to lookup org data.", err)
			return nil, err
		}
		if org.PrivateGists != nil {
			stats["private_gists"] = *org.PrivateGists
		}
		if org.TotalPrivateRepos != nil {
			stats["private_repos"] = *org.TotalPrivateRepos
		}
		if org.DiskUsage != nil {
			stats["disk_usage"] = *org.DiskUsage
		}
	}

	return stats, nil
}

func repoStats(resp *github.Repository) (map[string]int, error) {
	stats := make(map[string]int)

	if resp.ForksCount != nil {
		stats["forks"] = *resp.ForksCount
	}
	if resp.OpenIssuesCount != nil {
		stats["issues"] = *resp.OpenIssuesCount
	}
	if resp.NetworkCount != nil {
		stats["network"] = *resp.NetworkCount
	}
	if resp.StargazersCount != nil {
		stats["stars"] = *resp.StargazersCount
	}
	if resp.SubscribersCount != nil {
		stats["subcribers"] = *resp.SubscribersCount
	}
	if resp.WatchersCount != nil {
		stats["watchers"] = *resp.WatchersCount
	}
	if resp.Size != nil {
		stats["size"] = *resp.Size
	}
	return stats, nil
}

//GetMetricTypes returns metric types for testing
func (f *Gitstats) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType, error) {
	mts := make([]plugin.MetricType, 0)
	for _, metricName := range repoMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo").
				AddDynamicElement("owner", "repository owner").
				AddDynamicElement("repo", "repository name").
				AddStaticElement(metricName),
			Config_: cfg.ConfigDataNode,
		})
	}
	for _, metricName := range userMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "user").
				AddDynamicElement("user", "user or orginisation name").
				AddStaticElement(metricName),
			Config_: cfg.ConfigDataNode,
		})
	}
	return mts, nil
}

//GetConfigPolicy returns a ConfigPolicyTree for testing
func (f *Gitstats) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("access_token", true)
	rule1, _ := cpolicy.NewStringRule("user", false, "")
	rule2, _ := cpolicy.NewStringRule("repo", false, "")

	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	p.Add(rule1)
	p.Add(rule2)
	c.Add([]string{"raintank", "apps", "gitstats"}, p)
	return c, nil
}

//Meta returns meta data for testing
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		Name,
		Version,
		Type,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Unsecure(true),
		plugin.ConcurrencyCount(1000),
	)
}
<|endoftext|>


<commit_before>// Reddit audiences crawler
// Rémy Mathieu © 2016

package app

import (
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const (
	REDDIT_SUBREDDIT_URL       = "https://reddit.com/r/"
	SECONDS_BETWEEN_EACH_CRAWL = 1
)

var subredditsToCrawl chan string

func StartCrawlingJob(a *App) {
	subredditsToCrawl = make(chan string)

	// starts the worker
	go Worker(a)

	// starts the main loop
	// regularly feeding the worker.
	if a.Config.Crawl {
		log.Println("info: starts crawling job.")
		ticker := time.NewTicker(time.Second * 30)
		for range ticker.C {
			log.Println("info: crawling job is running.")
			Feeder(a)
		}
		ticker.Stop()
	}
}

// Worker is the routine dealing with the HTTP
// call + reading hte DOM.
func Worker(a *App) {
	for subreddit := range subredditsToCrawl {
		if audience, subscribers, articles, err := readDOMData(subreddit); err == nil {
			// store the audience+subscribers and update the last crawl time
			// ----------------------
			if err := a.DB().InsertAudienceValue(subreddit, audience, subscribers); err != nil {
				log.Println("err:", err.Error())
			} else {
				log.Printf("info: subreddit %s has %d active users (%d subscribers)\n", subreddit, audience, subscribers)
			}

			// store each articles if not already present at this rank
			// ----------------------
			if err := storeArticles(a, articles); err != nil {
				log.Printf("err: %s", err.Error())
			}
		} else if err != nil {
			log.Println("err:", err.Error())
		}

		time.Sleep(SECONDS_BETWEEN_EACH_CRAWL * time.Second) // wait a bit before the next crawl
	}
}

// Feeder retrieves the audience of subreddits for which
// the last crawl time is more than some minutes.
func Feeder(a *App) {
	// crawl each subreddit each 5 minutes
	five := time.Minute * 5
	t := time.Now().Add(-five)
	subreddits, err := a.DB().FindSubredditsToCrawl(t)

	if err != nil {
		log.Printf("err: can't retrieve subreddits to crawl: %s\n", err.Error())
	}

	for _, subreddit := range subreddits {
		log.Println("info: crawling", subreddit)
		subredditsToCrawl <- subreddit
	}
}

// readDOMData gets the subreddit page on reddit
// and gets the current audience, the subscribers and
// the article infos from the DOM.
// NOTE(remy): we stop as soon as we have a DOM error because
// it has great chances that the full DOM is corrupted/not retrieved.
func readDOMData(subreddit string) (int64, int64, []Article, error) {
	var audience int64
	var subscribers int64
	var err error

	doc, err := getSubredditPage(REDDIT_SUBREDDIT_URL + subreddit)
	if err != nil {
		return 0, 0, nil, fmt.Errorf("while crawling %s: %s", subreddit, err.Error())
	}

	// audience
	// ----------------------

	s := doc.Find("p.users-online span.number").First()

	value := s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s audience: no text value in the dom node.", subreddit)
	}

	if audience, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// subscribers
	// ----------------------

	s = doc.Find("span.subscribers span.number").First()

	value = s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s subscribers: no text value in the dom node.", subreddit)
	}

	if subscribers, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// articles
	// ----------------------

	now := time.Now()
	articles := make([]Article, 0)
	s = doc.Find(".link").Each(func(i int, selec *goquery.Selection) {
		l := selec.Find("p.title a.title")
		title := l.First()
		link, _ := l.Attr("href")
		strPos := selec.ChildrenFiltered(".rank").First()
		articleId, _ := selec.Attr("data-fullname")
		author, _ := selec.Attr("data-author")

		// remove the t[1-3]_ from the article id
		for i := 0; i < 4; i++ {
			articleId = strings.TrimPrefix(articleId, fmt.Sprintf("t%d_", i))
		}

		rank, err := strconv.Atoi(strPos.Text())
		if err != nil {
			rank = 0 // it's probably a promoted or stickied article
		}

		promoted := false
		if selec.HasClass("promoted") {
			promoted = true
		}

		sticky := false
		if selec.HasClass("stickied") {
			sticky = true
		}

		articles = append(articles, Article{
			Subreddit:    subreddit,
			ArticleId:    articleId,
			ArticleTitle: title.Text(),
			ArticleLink:  link,
			Author:       author,
			Rank:         rank,
			CrawlTime:    now,
			Promoted:     promoted,
			Sticky:       sticky,
		})
	})

	return audience, subscribers, articles, err
}

// storeArticles checks for each article if the
// info isn't already present in database, if not,
// it stores it. If changed, it also stores it.
func storeArticles(a *App, articles []Article) error {
	if len(articles) == 0 {
		return nil
	}

	for _, article := range articles {
		id, rank, err := a.DB().FindArticleLastState(article.Subreddit, article.ArticleId)
		if err != nil {
			return fmt.Errorf("while retrieving article last state: %s", err.Error())
		}

		// already stored at this rank
		if article.ArticleId == id && rank == article.Rank {
			continue
		}

		// not already store, do it now
		if _, err := a.DB().InsertArticle(article); err != nil {
			return err
		}
	}

	return nil
}

func getSubredditPage(url string) (*goquery.Document, error) {
	r, err := NewRequest(url)
	if err != nil {
		return nil, err
	}

	client := http.Client{}
	resp, err := client.Do(r)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	return goquery.NewDocumentFromResponse(resp)
}

func cleanInt(str string) (int64, error) {
	// sometimes it starts with ~
	if strings.HasPrefix(str, "~") {
		str = str[1:]
	}
	// , for thousands etc.
	str = strings.Replace(str, ",", "", -1)
	// finally trim
	str = strings.Trim(str, " ")

	return strconv.ParseInt(str, 10, 64)
}
<commit_msg>app: sleep a random time between each crawl.<commit_after>// Reddit audiences crawler
// Rémy Mathieu © 2016

package app

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const (
	REDDIT_SUBREDDIT_URL           = "https://reddit.com/r/"
	MIN_SECONDS_BETWEEN_EACH_CRAWL = 1
)

var subredditsToCrawl chan string

func StartCrawlingJob(a *App) {
	subredditsToCrawl = make(chan string)

	// starts the worker
	go Worker(a)

	// starts the main loop
	// regularly feeding the worker.
	if a.Config.Crawl {
		log.Println("info: starts crawling job.")
		ticker := time.NewTicker(time.Second * 30)
		for range ticker.C {
			log.Println("info: crawling job is running.")
			Feeder(a)
		}
		ticker.Stop()
	}
}

// Worker is the routine dealing with the HTTP
// call + reading hte DOM.
func Worker(a *App) {
	for subreddit := range subredditsToCrawl {
		if audience, subscribers, articles, err := readDOMData(subreddit); err == nil {
			// store the audience+subscribers and update the last crawl time
			// ----------------------
			if err := a.DB().InsertAudienceValue(subreddit, audience, subscribers); err != nil {
				log.Println("err:", err.Error())
			} else {
				log.Printf("info: subreddit %s has %d active users (%d subscribers)\n", subreddit, audience, subscribers)
			}

			// store each articles if not already present at this rank
			// ----------------------
			if err := storeArticles(a, articles); err != nil {
				log.Printf("err: %s", err.Error())
			}
		} else if err != nil {
			log.Println("err:", err.Error())
		}

		r := time.Duration(((rand.Int() % 2) + MIN_SECONDS_BETWEEN_EACH_CRAWL)) * time.Second

		time.Sleep(r) // wait a bit before the next crawl
	}
}

// Feeder retrieves the audience of subreddits for which
// the last crawl time is more than some minutes.
func Feeder(a *App) {
	// crawl each subreddit each 5 minutes
	five := time.Minute * 5
	t := time.Now().Add(-five)
	subreddits, err := a.DB().FindSubredditsToCrawl(t)

	if err != nil {
		log.Printf("err: can't retrieve subreddits to crawl: %s\n", err.Error())
	}

	for _, subreddit := range subreddits {
		log.Println("info: crawling", subreddit)
		subredditsToCrawl <- subreddit
	}
}

// readDOMData gets the subreddit page on reddit
// and gets the current audience, the subscribers and
// the article infos from the DOM.
// NOTE(remy): we stop as soon as we have a DOM error because
// it has great chances that the full DOM is corrupted/not retrieved.
func readDOMData(subreddit string) (int64, int64, []Article, error) {
	var audience int64
	var subscribers int64
	var err error

	doc, err := getSubredditPage(REDDIT_SUBREDDIT_URL + subreddit)
	if err != nil {
		return 0, 0, nil, fmt.Errorf("while crawling %s: %s", subreddit, err.Error())
	}

	// audience
	// ----------------------

	s := doc.Find("p.users-online span.number").First()

	value := s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s audience: no text value in the dom node.", subreddit)
	}

	if audience, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// subscribers
	// ----------------------

	s = doc.Find("span.subscribers span.number").First()

	value = s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s subscribers: no text value in the dom node.", subreddit)
	}

	if subscribers, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// articles
	// ----------------------

	now := time.Now()
	articles := make([]Article, 0)
	s = doc.Find(".link").Each(func(i int, selec *goquery.Selection) {
		l := selec.Find("p.title a.title")
		title := l.First()
		link, _ := l.Attr("href")
		strPos := selec.ChildrenFiltered(".rank").First()
		articleId, _ := selec.Attr("data-fullname")
		author, _ := selec.Attr("data-author")

		// remove the t[1-3]_ from the article id
		for i := 0; i < 4; i++ {
			articleId = strings.TrimPrefix(articleId, fmt.Sprintf("t%d_", i))
		}

		rank, err := strconv.Atoi(strPos.Text())
		if err != nil {
			rank = 0 // it's probably a promoted or stickied article
		}

		promoted := false
		if selec.HasClass("promoted") {
			promoted = true
		}

		sticky := false
		if selec.HasClass("stickied") {
			sticky = true
		}

		articles = append(articles, Article{
			Subreddit:    subreddit,
			ArticleId:    articleId,
			ArticleTitle: title.Text(),
			ArticleLink:  link,
			Author:       author,
			Rank:         rank,
			CrawlTime:    now,
			Promoted:     promoted,
			Sticky:       sticky,
		})
	})

	return audience, subscribers, articles, err
}

// storeArticles checks for each article if the
// info isn't already present in database, if not,
// it stores it. If changed, it also stores it.
func storeArticles(a *App, articles []Article) error {
	if len(articles) == 0 {
		return nil
	}

	for _, article := range articles {
		id, rank, err := a.DB().FindArticleLastState(article.Subreddit, article.ArticleId)
		if err != nil {
			return fmt.Errorf("while retrieving article last state: %s", err.Error())
		}

		// already stored at this rank
		if article.ArticleId == id && rank == article.Rank {
			continue
		}

		// not already store, do it now
		if _, err := a.DB().InsertArticle(article); err != nil {
			return err
		}
	}

	return nil
}

func getSubredditPage(url string) (*goquery.Document, error) {
	r, err := NewRequest(url)
	if err != nil {
		return nil, err
	}

	client := http.Client{}
	resp, err := client.Do(r)

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	return goquery.NewDocumentFromResponse(resp)
}

func cleanInt(str string) (int64, error) {
	// sometimes it starts with ~
	if strings.HasPrefix(str, "~") {
		str = str[1:]
	}
	// , for thousands etc.
	str = strings.Replace(str, ",", "", -1)
	// finally trim
	str = strings.Trim(str, " ")

	return strconv.ParseInt(str, 10, 64)
}
<|endoftext|>