<commit_before>/* Go (cgo) interface to libgeoip */
package geoip

/*
#cgo CFLAGS: -I/opt/local/include -I/usr/local/include -I/usr/include
#cgo LDFLAGS: -lGeoIP -L/opt/local/lib -L/usr/local/lib -L/usr/lib
#include <stdio.h>
#include <errno.h>
#include <GeoIP.h>
#include <GeoIPCity.h>

//typedef GeoIP* GeoIP_pnt
*/
import "C"

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"sync"
	"unsafe"
)

type GeoIP struct {
	db *C.GeoIP

	// We don't use GeoIP's thread-safe API calls, which means there is a
	// single global netmask variable that gets clobbered in the main
	// lookup routine.  Any calls which have _GeoIP_seek_record_gl need to
	// be wrapped in this mutex.

	mu sync.Mutex
}

func (gi *GeoIP) free() {
	if gi == nil {
		return
	}
	if gi.db == nil {
		gi = nil
		return
	}
	C.GeoIP_delete(gi.db)
	gi = nil
	return
}

// Default convenience wrapper around OpenDb
func Open(files ...string) (*GeoIP, error) {
	return OpenDb(files, GEOIP_MEMORY_CACHE)
}

// Opens a GeoIP database by filename with specified GeoIPOptions flag.
// All formats supported by libgeoip are supported though there are only
// functions to access some of the databases in this API.
// If you don't pass a filename, it will try opening the database from
// a list of common paths.
func OpenDb(files []string, flag int) (*GeoIP, error) {
	if len(files) == 0 {
		files = []string{
			"/usr/share/GeoIP/GeoIP.dat",       // Linux default
			"/usr/share/local/GeoIP/GeoIP.dat", // source install?
			"/usr/local/share/GeoIP/GeoIP.dat", // FreeBSD
			"/opt/local/share/GeoIP/GeoIP.dat", // MacPorts
			"/usr/share/GeoIP/GeoIP.dat",       // ArchLinux
		}
	}

	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	for _, file := range files {

		// libgeoip prints errors if it can't open the file, so check first
		if _, err := os.Stat(file); err != nil {
			if os.IsExist(err) {
				log.Println(err)
			}
			continue
		}

		cbase := C.CString(file)
		defer C.free(unsafe.Pointer(cbase))

		g.db, err = C.GeoIP_open(cbase, C.int(flag))
		if g.db != nil && err != nil {
			break
		}
	}
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%s): %s", files, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%s)", files)
	}

	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)
	return g, nil
}

// SetCustomDirectory sets the default location for the GeoIP .dat files used when
// calling OpenType()
func SetCustomDirectory(dir string) {
	cdir := C.CString(dir)
	// GeoIP doesn't copy the string, so don't free it when we're done here.
	// defer C.free(unsafe.Pointer(cdir))
	C.GeoIP_setup_custom_directory(cdir)
}

// OpenType opens a specified GeoIP database type in the default location with the
// specified GeoIPOptions flag. Constants are defined for each database type
// (for example GEOIP_COUNTRY_EDITION).
func OpenType(dbType int, flag int) (*GeoIP, error) {
	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	g.db, err = C.GeoIP_open_type(C.int(dbType), C.int(flag))
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%d): %s", dbType, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%d)", dbType)
	}

	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)

	return g, nil
}

// Takes an IPv4 address string and returns the organization name for that IP.
// Requires the GeoIP organization database.
func (gi *GeoIP) GetOrg(ip string) string {
	name, _ := gi.GetName(ip)
	return name
}

// Works on the ASN, Netspeed, Organization and probably other
// databases, takes and IP string and returns a "name" and the
// netmask.
func (gi *GeoIP) GetName(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

type GeoIPRecord struct {
	CountryCode  string
	CountryCode3 string
	CountryName  string
	Region       string
	City         string
	PostalCode   string
	Latitude     float32
	Longitude    float32
	// DMACode       int
	AreaCode      int
	CharSet       int
	ContinentCode string
}

// Returns the "City Record" for an IP address. Requires the GeoCity(Lite)
// database - http://www.maxmind.com/en/city
func (gi *GeoIP) GetRecord(ip string) *GeoIPRecord {
	if gi.db == nil {
		return nil
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))

	gi.mu.Lock()
	record := C.GeoIP_record_by_addr(gi.db, cip)
	gi.mu.Unlock()

	if record == nil {
		return nil
	}
	// defer C.free(unsafe.Pointer(record))
	defer C.GeoIPRecord_delete(record)
	rec := new(GeoIPRecord)
	rec.CountryCode = C.GoString(record.country_code)
	rec.CountryCode3 = C.GoString(record.country_code3)
	rec.CountryName = C.GoString(record.country_name)
	rec.Region = C.GoString(record.region)
	rec.City = C.GoString(record.city)
	rec.PostalCode = C.GoString(record.postal_code)
	rec.Latitude = float32(record.latitude)
	rec.Longitude = float32(record.longitude)
	rec.AreaCode = int(record.area_code)
	rec.CharSet = int(record.charset)
	rec.ContinentCode = C.GoString(record.continent_code)

	return rec
}

// Returns the Time Zone name from the tz database
// e.g. "America/New_York"
func GetTimeZone(country_code, region string) string {
	return C.GoString(C.GeoIP_time_zone_by_country_and_region(C.CString(country_code), C.CString(region)))
}

// Returns the country code and region code for an IP address. Requires
// the GeoIP Region database.
func (gi *GeoIP) GetRegion(ip string) (string, string) {
	if gi.db == nil {
		return "", ""
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))

	gi.mu.Lock()
	region := C.GeoIP_region_by_addr(gi.db, cip)
	gi.mu.Unlock()

	if region == nil {
		return "", ""
	}

	countryCode := C.GoString(&region.country_code[0])
	regionCode := C.GoString(&region.region[0])
	defer C.free(unsafe.Pointer(region))

	return countryCode, regionCode
}

// Returns the region name given a country code and region code
func GetRegionName(countryCode, regionCode string) string {

	cc := C.CString(countryCode)
	defer C.free(unsafe.Pointer(cc))

	rc := C.CString(regionCode)
	defer C.free(unsafe.Pointer(rc))

	region := C.GeoIP_region_name_by_code(cc, rc)
	if region == nil {
		return ""
	}

	// it's a static string constant, don't free this
	regionName := C.GoString(region)

	return regionName
}

// Same as GetName() but for IPv6 addresses.
func (gi *GeoIP) GetNameV6(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr_v6(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// Takes an IPv4 address string and returns the country code for that IP
// and the netmask for that IP range.
func (gi *GeoIP) GetCountry(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr(gi.db, cip)

	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// GetCountry_v6 works the same as GetCountry except for IPv6 addresses, be sure to
// load a database with IPv6 data to get any results.
func (gi *GeoIP) GetCountry_v6(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr_v6(gi.db, cip)
	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}
<commit_msg>Add DMACode and MetroCode fields to GeoIPRecord and only get AreaCode if db version > 0<commit_after>/* Go (cgo) interface to libgeoip */
package geoip

/*
#cgo CFLAGS: -I/opt/local/include -I/usr/local/include -I/usr/include
#cgo LDFLAGS: -lGeoIP -L/opt/local/lib -L/usr/local/lib -L/usr/lib
#include <stdio.h>
#include <errno.h>
#include <GeoIP.h>
#include <GeoIPCity.h>

//typedef GeoIP* GeoIP_pnt
*/
import "C"

import (
	"fmt"
	"log"
	"os"
	"runtime"
	"sync"
	"unsafe"
)

type GeoIP struct {
	db *C.GeoIP

	// We don't use GeoIP's thread-safe API calls, which means there is a
	// single global netmask variable that gets clobbered in the main
	// lookup routine.  Any calls which have _GeoIP_seek_record_gl need to
	// be wrapped in this mutex.

	mu sync.Mutex
}

func (gi *GeoIP) free() {
	if gi == nil {
		return
	}
	if gi.db == nil {
		gi = nil
		return
	}
	C.GeoIP_delete(gi.db)
	gi = nil
	return
}

// Default convenience wrapper around OpenDb
func Open(files ...string) (*GeoIP, error) {
	return OpenDb(files, GEOIP_MEMORY_CACHE)
}

// Opens a GeoIP database by filename with specified GeoIPOptions flag.
// All formats supported by libgeoip are supported though there are only
// functions to access some of the databases in this API.
// If you don't pass a filename, it will try opening the database from
// a list of common paths.
func OpenDb(files []string, flag int) (*GeoIP, error) {
	if len(files) == 0 {
		files = []string{
			"/usr/share/GeoIP/GeoIP.dat",       // Linux default
			"/usr/share/local/GeoIP/GeoIP.dat", // source install?
			"/usr/local/share/GeoIP/GeoIP.dat", // FreeBSD
			"/opt/local/share/GeoIP/GeoIP.dat", // MacPorts
			"/usr/share/GeoIP/GeoIP.dat",       // ArchLinux
		}
	}

	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	for _, file := range files {

		// libgeoip prints errors if it can't open the file, so check first
		if _, err := os.Stat(file); err != nil {
			if os.IsExist(err) {
				log.Println(err)
			}
			continue
		}

		cbase := C.CString(file)
		defer C.free(unsafe.Pointer(cbase))

		g.db, err = C.GeoIP_open(cbase, C.int(flag))
		if g.db != nil && err != nil {
			break
		}
	}
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%s): %s", files, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%s)", files)
	}

	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)
	return g, nil
}

// SetCustomDirectory sets the default location for the GeoIP .dat files used when
// calling OpenType()
func SetCustomDirectory(dir string) {
	cdir := C.CString(dir)
	// GeoIP doesn't copy the string, so don't free it when we're done here.
	// defer C.free(unsafe.Pointer(cdir))
	C.GeoIP_setup_custom_directory(cdir)
}

// OpenType opens a specified GeoIP database type in the default location with the
// specified GeoIPOptions flag. Constants are defined for each database type
// (for example GEOIP_COUNTRY_EDITION).
func OpenType(dbType int, flag int) (*GeoIP, error) {
	g := &GeoIP{}
	runtime.SetFinalizer(g, (*GeoIP).free)

	var err error

	g.db, err = C.GeoIP_open_type(C.int(dbType), C.int(flag))
	if err != nil {
		return nil, fmt.Errorf("Error opening GeoIP database (%d): %s", dbType, err)
	}

	if g.db == nil {
		return nil, fmt.Errorf("Didn't open GeoIP database (%d)", dbType)
	}

	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)

	return g, nil
}

// Takes an IPv4 address string and returns the organization name for that IP.
// Requires the GeoIP organization database.
func (gi *GeoIP) GetOrg(ip string) string {
	name, _ := gi.GetName(ip)
	return name
}

// Works on the ASN, Netspeed, Organization and probably other
// databases, takes and IP string and returns a "name" and the
// netmask.
func (gi *GeoIP) GetName(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

type GeoIPRecord struct {
	CountryCode   string
	CountryCode3  string
	CountryName   string
	Region        string
	City          string
	PostalCode    string
	Latitude      float32
	Longitude     float32
	MetroCode     int
	AreaCode      int
	CharSet       int
	ContinentCode string
}

// Returns the "City Record" for an IP address. Requires the GeoCity(Lite)
// database - http://www.maxmind.com/en/city
func (gi *GeoIP) GetRecord(ip string) *GeoIPRecord {
	if gi.db == nil {
		return nil
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))

	gi.mu.Lock()
	record := C.GeoIP_record_by_addr(gi.db, cip)
	gi.mu.Unlock()

	if record == nil {
		return nil
	}
	// defer C.free(unsafe.Pointer(record))
	defer C.GeoIPRecord_delete(record)
	rec := new(GeoIPRecord)
	rec.CountryCode = C.GoString(record.country_code)
	rec.CountryCode3 = C.GoString(record.country_code3)
	rec.CountryName = C.GoString(record.country_name)
	rec.Region = C.GoString(record.region)
	rec.City = C.GoString(record.city)
	rec.PostalCode = C.GoString(record.postal_code)
	rec.Latitude = float32(record.latitude)
	rec.Longitude = float32(record.longitude)
	rec.CharSet = int(record.charset)
	rec.ContinentCode = C.GoString(record.continent_code)

	if gi.db.databaseType != C.GEOIP_CITY_EDITION_REV0 {
		/* DIRTY HACK BELOW:
		   The GeoIPRecord struct in GeoIPCity.h contains an int32 union of metro_code and dma_code.
		   The union is unnamed, so cgo names it anon0 and assumes it's a 4-byte array.
		*/
		union_int := (*int32)(unsafe.Pointer(&record.anon0))
		rec.MetroCode = int(*union_int)
		rec.AreaCode = int(record.area_code)
	}

	return rec
}

// Returns the Time Zone name from the tz database
// e.g. "America/New_York"
func GetTimeZone(country_code, region string) string {
	return C.GoString(C.GeoIP_time_zone_by_country_and_region(C.CString(country_code), C.CString(region)))
}

// Returns the country code and region code for an IP address. Requires
// the GeoIP Region database.
func (gi *GeoIP) GetRegion(ip string) (string, string) {
	if gi.db == nil {
		return "", ""
	}

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))

	gi.mu.Lock()
	region := C.GeoIP_region_by_addr(gi.db, cip)
	gi.mu.Unlock()

	if region == nil {
		return "", ""
	}

	countryCode := C.GoString(&region.country_code[0])
	regionCode := C.GoString(&region.region[0])
	defer C.free(unsafe.Pointer(region))

	return countryCode, regionCode
}

// Returns the region name given a country code and region code
func GetRegionName(countryCode, regionCode string) string {

	cc := C.CString(countryCode)
	defer C.free(unsafe.Pointer(cc))

	rc := C.CString(regionCode)
	defer C.free(unsafe.Pointer(rc))

	region := C.GeoIP_region_name_by_code(cc, rc)
	if region == nil {
		return ""
	}

	// it's a static string constant, don't free this
	regionName := C.GoString(region)

	return regionName
}

// Same as GetName() but for IPv6 addresses.
func (gi *GeoIP) GetNameV6(ip string) (name string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	cname := C.GeoIP_name_by_addr_v6(gi.db, cip)

	if cname != nil {
		name = C.GoString(cname)
		defer C.free(unsafe.Pointer(cname))
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// Takes an IPv4 address string and returns the country code for that IP
// and the netmask for that IP range.
func (gi *GeoIP) GetCountry(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr(gi.db, cip)

	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}

// GetCountry_v6 works the same as GetCountry except for IPv6 addresses, be sure to
// load a database with IPv6 data to get any results.
func (gi *GeoIP) GetCountry_v6(ip string) (cc string, netmask int) {
	if gi.db == nil {
		return
	}

	gi.mu.Lock()
	defer gi.mu.Unlock()

	cip := C.CString(ip)
	defer C.free(unsafe.Pointer(cip))
	ccountry := C.GeoIP_country_code_by_addr_v6(gi.db, cip)
	if ccountry != nil {
		cc = C.GoString(ccountry)
		netmask = int(C.GeoIP_last_netmask(gi.db))
		return
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "flag"
  "fmt"
  "io"
  "log"
  "net/http"
  "net/http/cookiejar"
  "strings"
  "path"
  "code.google.com/p/go.net/html"
)

func fetch(url string) {
  res, err := http.Get(url)
  if err != nil {
    log.Fatal(err)
  }
  if err != nil {
    log.Fatal(err)
  }
  // http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
  fo, err := os.Create(path.Base(url))
  if err != nil {
    panic(err)
  }
  defer func() {
    if err := fo.Close(); err != nil {
      panic(err)
    }
  }()
  buf := make([]byte, 65536)
  for {
    n, err := res.Body.Read(buf)
    if err != nil && err != io.EOF {
      panic(err)
    }
    if n == 0 {
      break
    }

    if _, err := fo.Write(buf[:n]); err != nil {
      panic(err)
    }
  }
  res.Body.Close()
}

func main() {
  //var start string
  //flag.StringVar(&start, "start", "", "starting url")
  //flag.Parse()

  flag.Parse()
  start := flag.Args()[0]

  // http://stackoverflow.com/questions/18414212/golang-how-to-follow-location-with-cookie
  options := cookiejar.Options{}
  cookie_jar, err := cookiejar.New(&options)
  client := &http.Client{
    Jar: cookie_jar,
  }
  res, err := client.Get(start)
  if err != nil {
    log.Fatal(err)
  }
  //fmt.Printf("%s\n", start)

  z := html.NewTokenizer(res.Body)
  for {
    tt := z.Next()
    if tt == html.ErrorToken {
      break
    }
    if tt == html.StartTagToken {
      tn, _ := z.TagName()
      if len(tn) == 1 && tn[0] == 'a' {
        for {
          key, value, more := z.TagAttr()
          // http://stackoverflow.com/questions/14230145/what-is-the-best-way-to-convert-byte-array-to-string
          if string(key) == "href" {
            v := string(value)
            if strings.HasPrefix(v, "schedules/") {
              fuckedurl := path.Join(path.Dir(start), v)
              // yep, hack it
              // thx go for making me rename the variable
              url := strings.Replace(fuckedurl, ":/", "://", 1)
              fmt.Printf("%s\n", url)
              fetch(url)
            }
          }
          if !more {
            break
          }
        }
      }
      // ...
      //return ...
    }
    // Process the current token.
  }
  res.Body.Close()
  if err != nil {
    log.Fatal(err)
  }
}
<commit_msg>Logging in<commit_after>package main

import (
  "os"
  "flag"
  "fmt"
  "io"
  "log"
  "net/http"
  "net/http/cookiejar"
  "net/url"
  "strings"
  "path"
  "code.google.com/p/go.net/html"
)

func Split2(str, sep string) (string, string) {
  s := strings.Split(str, sep)
  return s[0], s[1]
}

func fetch(url string) {
  res, err := http.Get(url)
  if err != nil {
    log.Fatal(err)
  }
  if err != nil {
    log.Fatal(err)
  }
  // http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
  fo, err := os.Create(path.Base(url))
  if err != nil {
    panic(err)
  }
  defer func() {
    if err := fo.Close(); err != nil {
      panic(err)
    }
  }()
  buf := make([]byte, 65536)
  for {
    n, err := res.Body.Read(buf)
    if err != nil && err != io.EOF {
      panic(err)
    }
    if n == 0 {
      break
    }

    if _, err := fo.Write(buf[:n]); err != nil {
      panic(err)
    }
  }
  res.Body.Close()
}

func Fuckoff(pres *http.Response) {
}

func main() {
  //var start string
  //flag.StringVar(&start, "start", "", "starting url")
  //flag.Parse()

  flag.Parse()
  start := flag.Args()[0]
  username, password := Split2(flag.Args()[1], ":")

  // http://stackoverflow.com/questions/18414212/golang-how-to-follow-location-with-cookie
  options := cookiejar.Options{}
  cookie_jar, err := cookiejar.New(&options)
  client := &http.Client{
    Jar: cookie_jar,
  }
  pres, err := client.PostForm(start + "index.php?action=login2",
    url.Values{"user": {username}, "passwrd": {password}})
  if err != nil {
    log.Fatal(err)
  }
  Fuckoff(pres)

  res, err := client.Get(start)
  if err != nil {
    log.Fatal(err)
  }
  //fmt.Printf("%s\n", start)

  z := html.NewTokenizer(res.Body)
  for {
    tt := z.Next()
    if tt == html.ErrorToken {
      break
    }
    if tt == html.StartTagToken {
      tn, _ := z.TagName()
      if len(tn) == 1 && tn[0] == 'a' {
        for {
          key, value, more := z.TagAttr()
          // http://stackoverflow.com/questions/14230145/what-is-the-best-way-to-convert-byte-array-to-string
          if string(key) == "href" {
            v := string(value)
            if strings.HasPrefix(v, "schedules/") {
              fuckedurl := path.Join(path.Dir(start), v)
              // yep, hack it
              // thx go for making me rename the variable
              url := strings.Replace(fuckedurl, ":/", "://", 1)
              fmt.Printf("%s\n", url)
              fetch(url)
            }
          }
          if !more {
            break
          }
        }
      }
      // ...
      //return ...
    }
    // Process the current token.
  }
  res.Body.Close()
  if err != nil {
    log.Fatal(err)
  }
}
<|endoftext|>


<commit_before>package main

import "fmt"

type A struct {
	BB []*B
}

type B struct {
}

func main(){
	c := &A{}
	fmt.Printf("%+v,%+v,%d,%d\n",c,c.BB,len(c.BB),cap(c.BB))
	//can not use c.BB[0] not exist
	d := make([]*B,1,1)
	fmt.Printf("%+v",d[0])
	d[0] = nil
}
<commit_msg>make arr<commit_after>package main

import "fmt"

type A struct {
	BB []*B
}

type B struct {
}

func main(){
	c := &A{}
	fmt.Printf("%+v,%+v,%d,%d\n",c,c.BB,len(c.BB),cap(c.BB))
	//can not use c.BB[0] not exist
	d := make([]*B,1,1)
	fmt.Printf("%+v",d[0])
	d[0] = nil

	fmt.Println("=================")
	arr := make([]*B,1)
	fmt.Println("len,cap: ",len(arr),cap(arr))
	b := &B{}
	arr = append(arr,b)
	fmt.Println("=================")
	fmt.Println("len,cap: ",len(arr),cap(arr))
	//会增加一个nil在里面
	fmt.Printf("%+v\n",arr)
	fmt.Println("+++++++++++++++")
	arr1 := make([]*B,1)
	arr1[0] = b
	fmt.Println("len,cap: ",len(arr1),cap(arr1))
	fmt.Printf("%+v\n",arr1)
	fmt.Println("+++++++++++++++")
	arr2 := make([]*B,0)
	fmt.Println("len,cap: ",len(arr2),cap(arr2))
	arr2 = append(arr2,b)
	fmt.Println("len,cap: ",len(arr2),cap(arr2))
	fmt.Printf("%+v\n",arr2)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package plugin

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/cli-runtime/pkg/genericclioptions"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
)

var (
	pluginLong = templates.LongDesc(i18n.T(`
		Provides utilities for interacting with plugins.

		Plugins provide extended functionality that is not part of the major command-line distribution.
		Please refer to the documentation and examples for more information about how write your own plugins.

		The easiest way to discover and install plugins is via the kubernetes sub-project krew.
		To install krew, visit [krew.sigs.k8s.io](https://krew.sigs.k8s.io/docs/user-guide/setup/install/)`))

	pluginListLong = templates.LongDesc(i18n.T(`
		List all available plugin files on a user's PATH.

		Available plugin files are those that are:
		- executable
		- anywhere on the user's PATH
		- begin with "kubectl-"
`))

	ValidPluginFilenamePrefixes = []string{"kubectl"}
)

func NewCmdPlugin(streams genericclioptions.IOStreams) *cobra.Command {
	cmd := &cobra.Command{
		Use:                   "plugin [flags]",
		DisableFlagsInUseLine: true,
		Short:                 i18n.T("Provides utilities for interacting with plugins"),
		Long:                  pluginLong,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args)
		},
	}

	cmd.AddCommand(NewCmdPluginList(streams))
	return cmd
}

type PluginListOptions struct {
	Verifier PathVerifier
	NameOnly bool

	PluginPaths []string

	genericclioptions.IOStreams
}

// NewCmdPluginList provides a way to list all plugin executables visible to kubectl
func NewCmdPluginList(streams genericclioptions.IOStreams) *cobra.Command {
	o := &PluginListOptions{
		IOStreams: streams,
	}

	cmd := &cobra.Command{
		Use:   "list",
		Short: i18n.T("List all visible plugin executables on a user's PATH"),
		Long:  pluginListLong,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(cmd))
			cmdutil.CheckErr(o.Run())
		},
	}

	cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path")
	return cmd
}

func (o *PluginListOptions) Complete(cmd *cobra.Command) error {
	o.Verifier = &CommandOverrideVerifier{
		root:        cmd.Root(),
		seenPlugins: make(map[string]string),
	}

	o.PluginPaths = filepath.SplitList(os.Getenv("PATH"))
	return nil
}

func (o *PluginListOptions) Run() error {
	pluginsFound := false
	isFirstFile := true
	pluginErrors := []error{}
	pluginWarnings := 0

	for _, dir := range uniquePathsList(o.PluginPaths) {
		if len(strings.TrimSpace(dir)) == 0 {
			continue
		}

		files, err := ioutil.ReadDir(dir)
		if err != nil {
			if _, ok := err.(*os.PathError); ok {
				fmt.Fprintf(o.ErrOut, "Unable to read directory %q from your PATH: %v. Skipping...\n", dir, err)
				continue
			}

			pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to read directory %q in your PATH: %v", dir, err))
			continue
		}

		for _, f := range files {
			if f.IsDir() {
				continue
			}
			if !hasValidPrefix(f.Name(), ValidPluginFilenamePrefixes) {
				continue
			}

			if isFirstFile {
				fmt.Fprintf(o.Out, "The following compatible plugins are available:\n\n")
				pluginsFound = true
				isFirstFile = false
			}

			pluginPath := f.Name()
			if !o.NameOnly {
				pluginPath = filepath.Join(dir, pluginPath)
			}

			fmt.Fprintf(o.Out, "%s\n", pluginPath)
			if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 {
				for _, err := range errs {
					fmt.Fprintf(o.ErrOut, "  - %s\n", err)
					pluginWarnings++
				}
			}
		}
	}

	if !pluginsFound {
		pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to find any kubectl plugins in your PATH"))
	}

	if pluginWarnings > 0 {
		if pluginWarnings == 1 {
			pluginErrors = append(pluginErrors, fmt.Errorf("error: one plugin warning was found"))
		} else {
			pluginErrors = append(pluginErrors, fmt.Errorf("error: %v plugin warnings were found", pluginWarnings))
		}
	}
	if len(pluginErrors) > 0 {
		errs := bytes.NewBuffer(nil)
		for _, e := range pluginErrors {
			fmt.Fprintln(errs, e)
		}
		return fmt.Errorf("%s", errs.String())
	}

	return nil
}

// pathVerifier receives a path and determines if it is valid or not
type PathVerifier interface {
	// Verify determines if a given path is valid
	Verify(path string) []error
}

type CommandOverrideVerifier struct {
	root        *cobra.Command
	seenPlugins map[string]string
}

// Verify implements PathVerifier and determines if a given path
// is valid depending on whether or not it overwrites an existing
// kubectl command path, or a previously seen plugin.
func (v *CommandOverrideVerifier) Verify(path string) []error {
	if v.root == nil {
		return []error{fmt.Errorf("unable to verify path with nil root")}
	}

	// extract the plugin binary name
	segs := strings.Split(path, "/")
	binName := segs[len(segs)-1]

	cmdPath := strings.Split(binName, "-")
	if len(cmdPath) > 1 {
		// the first argument is always "kubectl" for a plugin binary
		cmdPath = cmdPath[1:]
	}

	errors := []error{}

	if isExec, err := isExecutable(path); err == nil && !isExec {
		errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path))
	} else if err != nil {
		errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err))
	}

	if existingPath, ok := v.seenPlugins[binName]; ok {
		errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath))
	} else {
		v.seenPlugins[binName] = path
	}

	if cmd, _, err := v.root.Find(cmdPath); err == nil {
		errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath()))
	}

	return errors
}

func isExecutable(fullPath string) (bool, error) {
	info, err := os.Stat(fullPath)
	if err != nil {
		return false, err
	}

	if runtime.GOOS == "windows" {
		fileExt := strings.ToLower(filepath.Ext(fullPath))

		switch fileExt {
		case ".bat", ".cmd", ".com", ".exe", ".ps1":
			return true, nil
		}
		return false, nil
	}

	if m := info.Mode(); !m.IsDir() && m&0111 != 0 {
		return true, nil
	}

	return false, nil
}

// uniquePathsList deduplicates a given slice of strings without
// sorting or otherwise altering its order in any way.
func uniquePathsList(paths []string) []string {
	seen := map[string]bool{}
	newPaths := []string{}
	for _, p := range paths {
		if seen[p] {
			continue
		}
		seen[p] = true
		newPaths = append(newPaths, p)
	}
	return newPaths
}

func hasValidPrefix(filepath string, validPrefixes []string) bool {
	for _, prefix := range validPrefixes {
		if !strings.HasPrefix(filepath, prefix+"-") {
			continue
		}
		return true
	}
	return false
}
<commit_msg>Adding an example for kubectl plugin list<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package plugin

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/cli-runtime/pkg/genericclioptions"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
)

var (
	pluginLong = templates.LongDesc(i18n.T(`
		Provides utilities for interacting with plugins.

		Plugins provide extended functionality that is not part of the major command-line distribution.
		Please refer to the documentation and examples for more information about how write your own plugins.

		The easiest way to discover and install plugins is via the kubernetes sub-project krew.
		To install krew, visit [krew.sigs.k8s.io](https://krew.sigs.k8s.io/docs/user-guide/setup/install/)`))

	pluginExample = templates.Examples(i18n.T(`
		# List all available plugins
		kubectl plugin list`))

	pluginListLong = templates.LongDesc(i18n.T(`
		List all available plugin files on a user's PATH.

		Available plugin files are those that are:
		- executable
		- anywhere on the user's PATH
		- begin with "kubectl-"
`))

	ValidPluginFilenamePrefixes = []string{"kubectl"}
)

func NewCmdPlugin(streams genericclioptions.IOStreams) *cobra.Command {
	cmd := &cobra.Command{
		Use:                   "plugin [flags]",
		DisableFlagsInUseLine: true,
		Short:                 i18n.T("Provides utilities for interacting with plugins"),
		Long:                  pluginLong,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args)
		},
	}

	cmd.AddCommand(NewCmdPluginList(streams))
	return cmd
}

type PluginListOptions struct {
	Verifier PathVerifier
	NameOnly bool

	PluginPaths []string

	genericclioptions.IOStreams
}

// NewCmdPluginList provides a way to list all plugin executables visible to kubectl
func NewCmdPluginList(streams genericclioptions.IOStreams) *cobra.Command {
	o := &PluginListOptions{
		IOStreams: streams,
	}

	cmd := &cobra.Command{
		Use:     "list",
		Short:   i18n.T("List all visible plugin executables on a user's PATH"),
		Example: pluginExample,
		Long:    pluginListLong,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(cmd))
			cmdutil.CheckErr(o.Run())
		},
	}

	cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path")
	return cmd
}

func (o *PluginListOptions) Complete(cmd *cobra.Command) error {
	o.Verifier = &CommandOverrideVerifier{
		root:        cmd.Root(),
		seenPlugins: make(map[string]string),
	}

	o.PluginPaths = filepath.SplitList(os.Getenv("PATH"))
	return nil
}

func (o *PluginListOptions) Run() error {
	pluginsFound := false
	isFirstFile := true
	pluginErrors := []error{}
	pluginWarnings := 0

	for _, dir := range uniquePathsList(o.PluginPaths) {
		if len(strings.TrimSpace(dir)) == 0 {
			continue
		}

		files, err := ioutil.ReadDir(dir)
		if err != nil {
			if _, ok := err.(*os.PathError); ok {
				fmt.Fprintf(o.ErrOut, "Unable to read directory %q from your PATH: %v. Skipping...\n", dir, err)
				continue
			}

			pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to read directory %q in your PATH: %v", dir, err))
			continue
		}

		for _, f := range files {
			if f.IsDir() {
				continue
			}
			if !hasValidPrefix(f.Name(), ValidPluginFilenamePrefixes) {
				continue
			}

			if isFirstFile {
				fmt.Fprintf(o.Out, "The following compatible plugins are available:\n\n")
				pluginsFound = true
				isFirstFile = false
			}

			pluginPath := f.Name()
			if !o.NameOnly {
				pluginPath = filepath.Join(dir, pluginPath)
			}

			fmt.Fprintf(o.Out, "%s\n", pluginPath)
			if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 {
				for _, err := range errs {
					fmt.Fprintf(o.ErrOut, "  - %s\n", err)
					pluginWarnings++
				}
			}
		}
	}

	if !pluginsFound {
		pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to find any kubectl plugins in your PATH"))
	}

	if pluginWarnings > 0 {
		if pluginWarnings == 1 {
			pluginErrors = append(pluginErrors, fmt.Errorf("error: one plugin warning was found"))
		} else {
			pluginErrors = append(pluginErrors, fmt.Errorf("error: %v plugin warnings were found", pluginWarnings))
		}
	}
	if len(pluginErrors) > 0 {
		errs := bytes.NewBuffer(nil)
		for _, e := range pluginErrors {
			fmt.Fprintln(errs, e)
		}
		return fmt.Errorf("%s", errs.String())
	}

	return nil
}

// pathVerifier receives a path and determines if it is valid or not
type PathVerifier interface {
	// Verify determines if a given path is valid
	Verify(path string) []error
}

type CommandOverrideVerifier struct {
	root        *cobra.Command
	seenPlugins map[string]string
}

// Verify implements PathVerifier and determines if a given path
// is valid depending on whether or not it overwrites an existing
// kubectl command path, or a previously seen plugin.
func (v *CommandOverrideVerifier) Verify(path string) []error {
	if v.root == nil {
		return []error{fmt.Errorf("unable to verify path with nil root")}
	}

	// extract the plugin binary name
	segs := strings.Split(path, "/")
	binName := segs[len(segs)-1]

	cmdPath := strings.Split(binName, "-")
	if len(cmdPath) > 1 {
		// the first argument is always "kubectl" for a plugin binary
		cmdPath = cmdPath[1:]
	}

	errors := []error{}

	if isExec, err := isExecutable(path); err == nil && !isExec {
		errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path))
	} else if err != nil {
		errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err))
	}

	if existingPath, ok := v.seenPlugins[binName]; ok {
		errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath))
	} else {
		v.seenPlugins[binName] = path
	}

	if cmd, _, err := v.root.Find(cmdPath); err == nil {
		errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath()))
	}

	return errors
}

func isExecutable(fullPath string) (bool, error) {
	info, err := os.Stat(fullPath)
	if err != nil {
		return false, err
	}

	if runtime.GOOS == "windows" {
		fileExt := strings.ToLower(filepath.Ext(fullPath))

		switch fileExt {
		case ".bat", ".cmd", ".com", ".exe", ".ps1":
			return true, nil
		}
		return false, nil
	}

	if m := info.Mode(); !m.IsDir() && m&0111 != 0 {
		return true, nil
	}

	return false, nil
}

// uniquePathsList deduplicates a given slice of strings without
// sorting or otherwise altering its order in any way.
func uniquePathsList(paths []string) []string {
	seen := map[string]bool{}
	newPaths := []string{}
	for _, p := range paths {
		if seen[p] {
			continue
		}
		seen[p] = true
		newPaths = append(newPaths, p)
	}
	return newPaths
}

func hasValidPrefix(filepath string, validPrefixes []string) bool {
	for _, prefix := range validPrefixes {
		if !strings.HasPrefix(filepath, prefix+"-") {
			continue
		}
		return true
	}
	return false
}
<|endoftext|>


<commit_before>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

import (
	"fmt"
	"github.com/uniqush/uniqush-conn/rpc"
	//"labix.org/v2/mgo/bson"
	"testing"
)

func marshalUnmarshal(cmd *Command) error {
	data, err := cmd.Marshal()
	if err != nil {
		return err
	}

	c, err := UnmarshalCommand(data)
	if !c.eq(cmd) {
		return fmt.Errorf("Not same")
	}
	return nil
}

func TestCommandMarshalNoParamsNoMessage(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	marshalUnmarshal(cmd)
}

func TestCommandMarshalNoParams(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = ""
	marshalUnmarshal(cmd)
}

func TestCommandMarshalNoBody(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = ""
	cmd.Message = new(rpc.Message)
	cmd.Message.Header = make(map[string]string, 3)
	cmd.Message.Header["a"] = "h"
	cmd.Message.Header["b"] = "i"
	cmd.Message.Header["b"] = "j"
	marshalUnmarshal(cmd)
}

func TestCommandMarshal(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = "new"
	cmd.Message = new(rpc.Message)
	cmd.Message.Header = make(map[string]string, 3)
	cmd.Message.Header["a"] = "h"
	cmd.Message.Header["b"] = "i"
	cmd.Message.Header["b"] = "j"
	cmd.Message.Body = []byte{1, 2, 3, 3}
	marshalUnmarshal(cmd)
}

func TestRandomize(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = "new"
	cmd.Message = new(rpc.Message)
	cmd.Message.Header = make(map[string]string, 3)
	cmd.Message.Header["a"] = "h"
	cmd.Message.Header["b"] = "i"
	cmd.Message.Header["b"] = "j"
	cmd.Message.Body = []byte{1, 2, 3, 3}
	marshalUnmarshal(cmd)
	cmd.Randomize()
}

func BenchmarkCommandMarshalUnmarshal(b *testing.B) {
	b.StopTimer()
	cmds := make([]*Command, b.N)
	for i, _ := range cmds {
		cmd := randomCommand()
		cmds[i] = cmd
	}
	b.StartTimer()
	for _, cmd := range cmds {
		data, _ := cmd.Marshal()
		UnmarshalCommand(data)
	}
}

// We don't need this benchmark to run for all ci test.
// func BenchmarkCommandMarshalUnmarshalBson(b *testing.B) {
// 	b.StopTimer()
// 	cmds := make([]*Command, b.N)
// 	for i, _ := range cmds {
// 		cmd := randomCommand()
// 		cmds[i] = cmd
// 	}
// 	b.StartTimer()
// 	for _, cmd := range cmds {
// 		data, _ := bson.Marshal(cmd)
// 		c := new(Command)
// 		bson.Unmarshal(data, c)
// 	}
// }
<commit_msg>depends on bson pkg again. #24.<commit_after>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

import (
	"fmt"
	"github.com/uniqush/uniqush-conn/rpc"
	"labix.org/v2/mgo/bson"
	"testing"
)

func marshalUnmarshal(cmd *Command) error {
	data, err := cmd.Marshal()
	if err != nil {
		return err
	}

	c, err := UnmarshalCommand(data)
	if !c.eq(cmd) {
		return fmt.Errorf("Not same")
	}
	return nil
}

func TestCommandMarshalNoParamsNoMessage(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	marshalUnmarshal(cmd)
}

func TestCommandMarshalNoParams(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = ""
	marshalUnmarshal(cmd)
}

func TestCommandMarshalNoBody(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = ""
	cmd.Message = new(rpc.Message)
	cmd.Message.Header = make(map[string]string, 3)
	cmd.Message.Header["a"] = "h"
	cmd.Message.Header["b"] = "i"
	cmd.Message.Header["b"] = "j"
	marshalUnmarshal(cmd)
}

func TestCommandMarshal(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = "new"
	cmd.Message = new(rpc.Message)
	cmd.Message.Header = make(map[string]string, 3)
	cmd.Message.Header["a"] = "h"
	cmd.Message.Header["b"] = "i"
	cmd.Message.Header["b"] = "j"
	cmd.Message.Body = []byte{1, 2, 3, 3}
	marshalUnmarshal(cmd)
}

func TestRandomize(t *testing.T) {
	cmd := new(Command)
	cmd.Type = 1
	cmd.Params = make([]string, 2)
	cmd.Params[0] = "hello"
	cmd.Params[1] = "new"
	cmd.Message = new(rpc.Message)
	cmd.Message.Header = make(map[string]string, 3)
	cmd.Message.Header["a"] = "h"
	cmd.Message.Header["b"] = "i"
	cmd.Message.Header["b"] = "j"
	cmd.Message.Body = []byte{1, 2, 3, 3}
	marshalUnmarshal(cmd)
	cmd.Randomize()
}

func BenchmarkCommandMarshalUnmarshal(b *testing.B) {
	b.StopTimer()
	cmds := make([]*Command, b.N)
	for i, _ := range cmds {
		cmd := randomCommand()
		cmds[i] = cmd
	}
	b.StartTimer()
	for _, cmd := range cmds {
		data, _ := cmd.Marshal()
		UnmarshalCommand(data)
	}
}

func BenchmarkCommandMarshalUnmarshalBson(b *testing.B) {
	b.StopTimer()
	cmds := make([]*Command, b.N)
	for i, _ := range cmds {
		cmd := randomCommand()
		cmds[i] = cmd
	}
	b.StartTimer()
	for _, cmd := range cmds {
		data, _ := bson.Marshal(cmd)
		c := new(Command)
		bson.Unmarshal(data, c)
	}
}
<|endoftext|>


<commit_before>package protocol

import (
	log "github.com/sirupsen/logrus"

	"github.com/bytom/errors"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
	"github.com/bytom/protocol/state"
)

var (
	// ErrBadBlock is returned when a block is invalid.
	ErrBadBlock = errors.New("invalid block")

	// ErrBadStateRoot is returned when the computed assets merkle root
	// disagrees with the one declared in a block header.
	ErrBadStateRoot = errors.New("invalid state merkle root")
)

// BlockExist check is a block in chain or orphan
func (c *Chain) BlockExist(hash *bc.Hash) bool {
	return c.orphanManage.BlockExist(hash) || c.index.BlockExist(hash)
}

// GetBlockByHash return a block by given hash
func (c *Chain) GetBlockByHash(hash *bc.Hash) (*types.Block, error) {
	return c.store.GetBlock(hash)
}

// GetBlockByHeight return a block by given height
func (c *Chain) GetBlockByHeight(height uint64) (*types.Block, error) {
	node := c.index.NodeByHeight(height)
	if node == nil {
		return nil, errors.New("can't find block in given hight")
	}
	return c.store.GetBlock(&node.Hash)
}

// ConnectBlock append block to end of chain
func (c *Chain) ConnectBlock(block *types.Block) error {
	c.state.cond.L.Lock()
	defer c.state.cond.L.Unlock()
	return c.connectBlock(block)
}

func (c *Chain) connectBlock(block *types.Block) (err error) {
	bcBlock := types.MapBlock(block)
	utxoView := state.NewUtxoViewpoint()
	bcBlock.TransactionStatus, err = c.store.GetTransactionStatus(&bcBlock.ID)
	if err != nil {
		return err
	}

	if err := c.store.GetTransactionsUtxo(utxoView, bcBlock.Transactions); err != nil {
		return err
	}
	if err := utxoView.ApplyBlock(bcBlock, bcBlock.TransactionStatus); err != nil {
		return err
	}

	if err := c.setState(block, utxoView); err != nil {
		return err
	}

	for _, tx := range block.Transactions {
		c.txPool.RemoveTransaction(&tx.Tx.ID)
	}
	return nil
}

func (c *Chain) getReorganizeBlocks(block *types.Block) ([]*types.Block, []*types.Block) {
	attachBlocks := []*types.Block{}
	detachBlocks := []*types.Block{}
	ancestor := block

	for !c.index.InMainchain(block.Hash()) {
		attachBlocks = append([]*types.Block{ancestor}, attachBlocks...)
		ancestor, _ = c.GetBlockByHash(&ancestor.PreviousBlockHash)
	}

	for d, _ := c.store.GetBlock(c.state.hash); d.Hash() != ancestor.Hash(); d, _ = c.store.GetBlock(&d.PreviousBlockHash) {
		detachBlocks = append(detachBlocks, d)
	}

	return attachBlocks, detachBlocks
}

func (c *Chain) reorganizeChain(block *types.Block) error {
	attachBlocks, detachBlocks := c.getReorganizeBlocks(block)
	utxoView := state.NewUtxoViewpoint()

	for _, d := range detachBlocks {
		detachBlock := types.MapBlock(d)
		if err := c.store.GetTransactionsUtxo(utxoView, detachBlock.Transactions); err != nil {
			return err
		}
		txStatus, err := c.GetTransactionStatus(&detachBlock.ID)
		if err != nil {
			return err
		}
		if err := utxoView.DetachBlock(detachBlock, txStatus); err != nil {
			return err
		}
		log.WithFields(log.Fields{"height": detachBlock.Height, "hash": detachBlock.ID.String()}).Debug("Detach from mainchain")
	}

	for _, a := range attachBlocks {
		attachBlock := types.MapBlock(a)
		if err := c.store.GetTransactionsUtxo(utxoView, attachBlock.Transactions); err != nil {
			return err
		}
		txStatus, err := c.GetTransactionStatus(&attachBlock.ID)
		if err != nil {
			return err
		}

		if err := utxoView.ApplyBlock(attachBlock, txStatus); err != nil {
			return err
		}
		log.WithFields(log.Fields{"height": attachBlock.Height, "hash": attachBlock.ID.String()}).Debug("Attach from mainchain")
	}

	return c.setState(block, utxoView)
}

// SaveBlock will validate and save block into storage
func (c *Chain) SaveBlock(block *types.Block) error {
	blockEnts := types.MapBlock(block)
	if err := c.validateBlock(blockEnts); err != nil {
		return errors.Sub(ErrBadBlock, err)
	}

	if err := c.store.SaveBlock(block, blockEnts.TransactionStatus); err != nil {
		return err
	}
	log.WithFields(log.Fields{"height": block.Height, "hash": blockEnts.ID.String()}).Info("Block saved on disk")

	c.orphanManage.Delete(&blockEnts.ID)
	parent := c.index.GetNode(&block.PreviousBlockHash)
	node, err := NewBlockNode(&block.BlockHeader, parent)
	if err != nil {
		return err
	}

	c.index.AddNode(node)
	return nil
}

func (c *Chain) findBestChainTail(block *types.Block) (bestBlock *types.Block) {
	bestBlock = block
	blockHash := block.Hash()
	preorphans, ok := c.orphanManage.preOrphans[blockHash]
	if !ok {
		return
	}

	for _, preorphan := range preorphans {
		orphanBlock, ok := c.orphanManage.Get(preorphan)
		if !ok {
			continue
		}

		if err := c.SaveBlock(orphanBlock); err != nil {
			log.WithFields(log.Fields{
				"height": block.Height,
				"hash":   blockHash.String(),
			}).Errorf("findBestChainTail fail on save block %v", err)
			continue
		}

		if subResult := c.findBestChainTail(orphanBlock); subResult.Height > bestBlock.Height {
			bestBlock = subResult
		}
	}

	return
}

type processBlockResponse struct {
	isOrphan bool
	err      error
}

type processBlockMsg struct {
	block *types.Block
	reply chan processBlockResponse
}

func (c *Chain) ProcessBlock(block *types.Block) (bool, error) {
	reply := make(chan processBlockResponse, 1)
	c.processBlockCh <- &processBlockMsg{block: block, reply: reply}
	response := <-reply
	return response.isOrphan, response.err
}

func (c *Chain) blockProcesser() {
	for msg := range c.processBlockCh {
		isOrphan, err := c.processBlock(msg.block)
		msg.reply <- processBlockResponse{isOrphan: isOrphan, err: err}
	}
}

// ProcessBlock is the entry for handle block insert
func (c *Chain) processBlock(block *types.Block) (bool, error) {
	blockHash := block.Hash()
	if c.BlockExist(&blockHash) {
		log.WithField("hash", blockHash.String()).Debug("Skip process due to block already been handled")
		return c.orphanManage.BlockExist(&blockHash), nil
	}
	if !c.store.BlockExist(&block.PreviousBlockHash) {
		log.WithField("hash", blockHash.String()).Debug("Add block to orphan manage")
		c.orphanManage.Add(block)
		return true, nil
	}
	if err := c.SaveBlock(block); err != nil {
		return false, err
	}

	bestBlock := c.findBestChainTail(block)
	bestMainChain := c.index.BestNode()
	bestBlockHash := bestBlock.Hash()
	bestNode := c.index.GetNode(&bestBlockHash)

	if bestNode.parent == bestMainChain {
		log.WithField("hash", blockHash.String()).Debug("Start to append block to the tail of mainchain")
		return false, c.connectBlock(bestBlock)
	}

	if bestNode.height > bestMainChain.height && bestNode.workSum.Cmp(bestMainChain.workSum) >= 0 {
		log.WithField("hash", blockHash.String()).Debug("Start to reorganize mainchain")
		return false, c.reorganizeChain(bestBlock)
	}

	return false, nil
}
<commit_msg>fix get reorganize blocks stuck (#549)<commit_after>package protocol

import (
	log "github.com/sirupsen/logrus"

	"github.com/bytom/errors"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
	"github.com/bytom/protocol/state"
)

var (
	// ErrBadBlock is returned when a block is invalid.
	ErrBadBlock = errors.New("invalid block")

	// ErrBadStateRoot is returned when the computed assets merkle root
	// disagrees with the one declared in a block header.
	ErrBadStateRoot = errors.New("invalid state merkle root")
)

// BlockExist check is a block in chain or orphan
func (c *Chain) BlockExist(hash *bc.Hash) bool {
	return c.orphanManage.BlockExist(hash) || c.index.BlockExist(hash)
}

// GetBlockByHash return a block by given hash
func (c *Chain) GetBlockByHash(hash *bc.Hash) (*types.Block, error) {
	return c.store.GetBlock(hash)
}

// GetBlockByHeight return a block by given height
func (c *Chain) GetBlockByHeight(height uint64) (*types.Block, error) {
	node := c.index.NodeByHeight(height)
	if node == nil {
		return nil, errors.New("can't find block in given hight")
	}
	return c.store.GetBlock(&node.Hash)
}

// ConnectBlock append block to end of chain
func (c *Chain) ConnectBlock(block *types.Block) error {
	c.state.cond.L.Lock()
	defer c.state.cond.L.Unlock()
	return c.connectBlock(block)
}

func (c *Chain) connectBlock(block *types.Block) (err error) {
	bcBlock := types.MapBlock(block)
	utxoView := state.NewUtxoViewpoint()
	bcBlock.TransactionStatus, err = c.store.GetTransactionStatus(&bcBlock.ID)
	if err != nil {
		return err
	}

	if err := c.store.GetTransactionsUtxo(utxoView, bcBlock.Transactions); err != nil {
		return err
	}
	if err := utxoView.ApplyBlock(bcBlock, bcBlock.TransactionStatus); err != nil {
		return err
	}

	if err := c.setState(block, utxoView); err != nil {
		return err
	}

	for _, tx := range block.Transactions {
		c.txPool.RemoveTransaction(&tx.Tx.ID)
	}
	return nil
}

func (c *Chain) getReorganizeBlocks(block *types.Block) ([]*types.Block, []*types.Block) {
	attachBlocks := []*types.Block{}
	detachBlocks := []*types.Block{}
	ancestor := block

	for !c.index.InMainchain(ancestor.Hash()) {
		attachBlocks = append([]*types.Block{ancestor}, attachBlocks...)
		ancestor, _ = c.GetBlockByHash(&ancestor.PreviousBlockHash)
	}

	for d, _ := c.store.GetBlock(c.state.hash); d.Hash() != ancestor.Hash(); d, _ = c.store.GetBlock(&d.PreviousBlockHash) {
		detachBlocks = append(detachBlocks, d)
	}

	return attachBlocks, detachBlocks
}

func (c *Chain) reorganizeChain(block *types.Block) error {
	attachBlocks, detachBlocks := c.getReorganizeBlocks(block)
	utxoView := state.NewUtxoViewpoint()

	for _, d := range detachBlocks {
		detachBlock := types.MapBlock(d)
		if err := c.store.GetTransactionsUtxo(utxoView, detachBlock.Transactions); err != nil {
			return err
		}
		txStatus, err := c.GetTransactionStatus(&detachBlock.ID)
		if err != nil {
			return err
		}
		if err := utxoView.DetachBlock(detachBlock, txStatus); err != nil {
			return err
		}
		log.WithFields(log.Fields{"height": detachBlock.Height, "hash": detachBlock.ID.String()}).Debug("Detach from mainchain")
	}

	for _, a := range attachBlocks {
		attachBlock := types.MapBlock(a)
		if err := c.store.GetTransactionsUtxo(utxoView, attachBlock.Transactions); err != nil {
			return err
		}
		txStatus, err := c.GetTransactionStatus(&attachBlock.ID)
		if err != nil {
			return err
		}

		if err := utxoView.ApplyBlock(attachBlock, txStatus); err != nil {
			return err
		}
		log.WithFields(log.Fields{"height": attachBlock.Height, "hash": attachBlock.ID.String()}).Debug("Attach from mainchain")
	}

	return c.setState(block, utxoView)
}

// SaveBlock will validate and save block into storage
func (c *Chain) SaveBlock(block *types.Block) error {
	blockEnts := types.MapBlock(block)
	if err := c.validateBlock(blockEnts); err != nil {
		return errors.Sub(ErrBadBlock, err)
	}

	if err := c.store.SaveBlock(block, blockEnts.TransactionStatus); err != nil {
		return err
	}
	log.WithFields(log.Fields{"height": block.Height, "hash": blockEnts.ID.String()}).Info("Block saved on disk")

	c.orphanManage.Delete(&blockEnts.ID)
	parent := c.index.GetNode(&block.PreviousBlockHash)
	node, err := NewBlockNode(&block.BlockHeader, parent)
	if err != nil {
		return err
	}

	c.index.AddNode(node)
	return nil
}

func (c *Chain) findBestChainTail(block *types.Block) (bestBlock *types.Block) {
	bestBlock = block
	blockHash := block.Hash()
	preorphans, ok := c.orphanManage.preOrphans[blockHash]
	if !ok {
		return
	}

	for _, preorphan := range preorphans {
		orphanBlock, ok := c.orphanManage.Get(preorphan)
		if !ok {
			continue
		}

		if err := c.SaveBlock(orphanBlock); err != nil {
			log.WithFields(log.Fields{
				"height": block.Height,
				"hash":   blockHash.String(),
			}).Errorf("findBestChainTail fail on save block %v", err)
			continue
		}

		if subResult := c.findBestChainTail(orphanBlock); subResult.Height > bestBlock.Height {
			bestBlock = subResult
		}
	}

	return
}

type processBlockResponse struct {
	isOrphan bool
	err      error
}

type processBlockMsg struct {
	block *types.Block
	reply chan processBlockResponse
}

func (c *Chain) ProcessBlock(block *types.Block) (bool, error) {
	reply := make(chan processBlockResponse, 1)
	c.processBlockCh <- &processBlockMsg{block: block, reply: reply}
	response := <-reply
	return response.isOrphan, response.err
}

func (c *Chain) blockProcesser() {
	for msg := range c.processBlockCh {
		isOrphan, err := c.processBlock(msg.block)
		msg.reply <- processBlockResponse{isOrphan: isOrphan, err: err}
	}
}

// ProcessBlock is the entry for handle block insert
func (c *Chain) processBlock(block *types.Block) (bool, error) {
	blockHash := block.Hash()
	if c.BlockExist(&blockHash) {
		log.WithField("hash", blockHash.String()).Debug("Skip process due to block already been handled")
		return c.orphanManage.BlockExist(&blockHash), nil
	}
	if !c.store.BlockExist(&block.PreviousBlockHash) {
		log.WithField("hash", blockHash.String()).Debug("Add block to orphan manage")
		c.orphanManage.Add(block)
		return true, nil
	}
	if err := c.SaveBlock(block); err != nil {
		return false, err
	}

	bestBlock := c.findBestChainTail(block)
	bestMainChain := c.index.BestNode()
	bestBlockHash := bestBlock.Hash()
	bestNode := c.index.GetNode(&bestBlockHash)

	if bestNode.parent == bestMainChain {
		log.WithField("hash", blockHash.String()).Debug("Start to append block to the tail of mainchain")
		return false, c.connectBlock(bestBlock)
	}

	if bestNode.height > bestMainChain.height && bestNode.workSum.Cmp(bestMainChain.workSum) >= 0 {
		log.WithField("hash", blockHash.String()).Debug("Start to reorganize mainchain")
		return false, c.reorganizeChain(bestBlock)
	}

	return false, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 CertiVox UK Ltd

This file is part of The CertiVox MIRACL IOT Crypto SDK (MiotCL)

MiotCL is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

MiotCL is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with MiotCL.  If not, see <http://www.gnu.org/licenses/>.

You can be released from the requirements of the license by purchasing
a commercial license.
*/

/* Finite Field arithmetic */
/* CLINT mod p functions */

package main

//import "fmt"

var p BIG=BIG{w: [NLEN]int64(Modulus)}

type FP struct {
	x *BIG
}

/* Constructors */
func NewFPint(a int) *FP {
	F:=new(FP)
	F.x=NewBIGint(a)
	F.nres()
	return F
}

func NewFPbig(a *BIG) *FP {
	F:=new(FP)
	F.x=NewBIGcopy(a)
	F.nres()
	return F
}

func NewFPcopy(a *FP) *FP {
	F:=new(FP)
	F.x=NewBIGcopy(a.x)
	return F
}

func (F *FP) toString() string {
	return F.redc().toString()
}

/* convert to Montgomery n-residue form */
func (F *FP) nres() {
	if MODTYPE!=PSEUDO_MERSENNE && MODTYPE!=GENERALISED_MERSENNE {
		d:=NewDBIGscopy(F.x)
		d.shl(uint(NLEN)*BASEBITS)
		F.x.copy(d.mod(&p))
	}
}

/* convert back to regular form */
func (F *FP) redc() *BIG {
	if MODTYPE!=PSEUDO_MERSENNE && MODTYPE!=GENERALISED_MERSENNE {
		d:=NewDBIGscopy(F.x)
		return mod(d)
	} else {
		r:=NewBIGcopy(F.x)
		return r
	}
}

/* reduce this mod Modulus */
func (F *FP) reduce() {
	F.x.mod(&p)
}

/* test this=0? */
func (F *FP) iszilch() bool {
	F.reduce()
	return F.x.iszilch()
}

/* copy from FP b */
func (F *FP) copy(b *FP ) {
	F.x.copy(b.x)
}

/* set this=0 */
func (F *FP) zero() {
	F.x.zero()
}

/* set this=1 */
func (F *FP) one() {
	F.x.one(); F.nres()
}

/* normalise this */
func (F *FP) norm() {
	F.x.norm();
}

/* swap FPs depending on d */
func (F *FP) cswap(b *FP,d int32) {
	F.x.cswap(b.x,d);
}

/* copy FPs depending on d */
func (F *FP) cmove(b *FP,d int32) {
	F.x.cmove(b.x,d)
}

/* this*=b mod Modulus */
func (F *FP) mul(b *FP) {

	F.norm();
	b.norm();
	ea:=EXCESS(F.x)
	eb:=EXCESS(b.x)

	if (ea+1)>FEXCESS/(eb+1) {
		F.reduce()

	d:=mul(F.x,b.x)
	F.x.copy(mod(d))
}

/* this = -this mod Modulus */
func (F *FP) neg() {
	m:=NewBIGcopy(&p)

	F.norm()

	ov:=EXCESS(F.x);
	sb:=uint(1); for ov!=0 {sb++;ov>>=1}

	m.fshl(sb)
	F.x.rsub(m)

	if EXCESS(F.x)>=FEXCESS {F.reduce()}
}

/* this*=c mod Modulus, where c is a small int */
func (F *FP) imul(c int) {
	F.norm()
	s:=false
	if (c<0) {
		c=-c
		s=true
	}
	afx:=(EXCESS(F.x)+1)*(int64(c)+1)+1;
	if (c<NEXCESS && afx<FEXCESS) {
		F.x.imul(c);
	} else {
		if (afx<FEXCESS) {
			F.x.pmul(c)
		} else {
			d:=F.x.pxmul(c)
			F.x.copy(d.mod(&p))
		}
	}
	if s {F.neg()}
	F.norm()
}

/* this*=this mod Modulus */
func (F *FP) sqr() {
	F.norm();
	ea:=EXCESS(F.x)
	if (ea+1)>FEXCESS/(ea+1) {
		F.reduce()

	d:=sqr(F.x)
	F.x.copy(mod(d))
}

/* this+=b */
func (F *FP) add(b *FP) {
	F.x.add(b.x)
	if (EXCESS(F.x)+2>=FEXCESS) {F.reduce()}
}

/* this-=b */
func (F *FP) sub(b *FP) {
	n:=NewFPcopy(b)
	n.neg()
	F.add(n)
}

/* this/=2 mod Modulus */
func (F *FP) div2() {
	F.x.norm()
	if (F.x.parity()==0) {
		F.x.fshr(1)
	} else {
		F.x.add(&p)
		F.x.norm()
		F.x.fshr(1)
	}
}

/* this=1/this mod Modulus */
func (F *FP) inverse() {
	r:=F.redc()
	r.invmodp(&p)
	F.x.copy(r)
	F.nres()
}

/* return TRUE if this==a */
func (F *FP) equals(a *FP) bool {
	a.reduce()
	F.reduce()
	if (comp(a.x,F.x)==0) {return true}
	return false
}

/* return this^e mod Modulus */
func (F *FP) pow(e *BIG) *FP {
	r:=NewFPint(1)
	e.norm()
	F.x.norm()
	m:=NewFPcopy(F)
	for true {
		bt:=e.parity();
		e.fshr(1);
		if bt==1 {r.mul(m)}
		if e.iszilch() {break}
		m.sqr();
	}
	r.x.mod(&p);
	return r;
}

/* return sqrt(this) mod Modulus */
func (F *FP) sqrt() *FP {
	F.reduce();
	b:=NewBIGcopy(&p)
	if MOD8==5 {
		b.dec(5); b.norm(); b.shr(3)
		i:=NewFPcopy(F); i.x.shl(1)
		v:=i.pow(b)
		i.mul(v); i.mul(v)
		i.x.dec(1)
		r:=NewFPcopy(F)
		r.mul(v); r.mul(i)
		r.reduce()
		return r
	} else {
		b.inc(1); b.norm(); b.shr(2)
		return F.pow(b);
	}
}

/* return jacobi symbol (this/Modulus) */
func (F *FP) jacobi() int {
	w:=F.redc();
	return w.jacobi(&p)
}
<commit_msg>Bug fix<commit_after>/*
Copyright 2015 CertiVox UK Ltd

This file is part of The CertiVox MIRACL IOT Crypto SDK (MiotCL)

MiotCL is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

MiotCL is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with MiotCL.  If not, see <http://www.gnu.org/licenses/>.

You can be released from the requirements of the license by purchasing
a commercial license.
*/

/* Finite Field arithmetic */
/* CLINT mod p functions */

package main

//import "fmt"

var p BIG=BIG{w: [NLEN]int64(Modulus)}

type FP struct {
	x *BIG
}

/* Constructors */
func NewFPint(a int) *FP {
	F:=new(FP)
	F.x=NewBIGint(a)
	F.nres()
	return F
}

func NewFPbig(a *BIG) *FP {
	F:=new(FP)
	F.x=NewBIGcopy(a)
	F.nres()
	return F
}

func NewFPcopy(a *FP) *FP {
	F:=new(FP)
	F.x=NewBIGcopy(a.x)
	return F
}

func (F *FP) toString() string {
	return F.redc().toString()
}

/* convert to Montgomery n-residue form */
func (F *FP) nres() {
	if MODTYPE!=PSEUDO_MERSENNE && MODTYPE!=GENERALISED_MERSENNE {
		d:=NewDBIGscopy(F.x)
		d.shl(uint(NLEN)*BASEBITS)
		F.x.copy(d.mod(&p))
	}
}

/* convert back to regular form */
func (F *FP) redc() *BIG {
	if MODTYPE!=PSEUDO_MERSENNE && MODTYPE!=GENERALISED_MERSENNE {
		d:=NewDBIGscopy(F.x)
		return mod(d)
	} else {
		r:=NewBIGcopy(F.x)
		return r
	}
}

/* reduce this mod Modulus */
func (F *FP) reduce() {
	F.x.mod(&p)
}

/* test this=0? */
func (F *FP) iszilch() bool {
	F.reduce()
	return F.x.iszilch()
}

/* copy from FP b */
func (F *FP) copy(b *FP ) {
	F.x.copy(b.x)
}

/* set this=0 */
func (F *FP) zero() {
	F.x.zero()
}

/* set this=1 */
func (F *FP) one() {
	F.x.one(); F.nres()
}

/* normalise this */
func (F *FP) norm() {
	F.x.norm();
}

/* swap FPs depending on d */
func (F *FP) cswap(b *FP,d int32) {
	F.x.cswap(b.x,d);
}

/* copy FPs depending on d */
func (F *FP) cmove(b *FP,d int32) {
	F.x.cmove(b.x,d)
}

/* this*=b mod Modulus */
func (F *FP) mul(b *FP) {

	F.norm();
	b.norm();
	ea:=EXCESS(F.x)
	eb:=EXCESS(b.x)

	if (ea+1)>FEXCESS/(eb+1) {
		F.reduce()
	}
	d:=mul(F.x,b.x)
	F.x.copy(mod(d))
}

/* this = -this mod Modulus */
func (F *FP) neg() {
	m:=NewBIGcopy(&p)

	F.norm()

	ov:=EXCESS(F.x);
	sb:=uint(1); for ov!=0 {sb++;ov>>=1}

	m.fshl(sb)
	F.x.rsub(m)

	if EXCESS(F.x)>=FEXCESS {F.reduce()}
}

/* this*=c mod Modulus, where c is a small int */
func (F *FP) imul(c int) {
	F.norm()
	s:=false
	if (c<0) {
		c=-c
		s=true
	}
	afx:=(EXCESS(F.x)+1)*(int64(c)+1)+1;
	if (c<NEXCESS && afx<FEXCESS) {
		F.x.imul(c);
	} else {
		if (afx<FEXCESS) {
			F.x.pmul(c)
		} else {
			d:=F.x.pxmul(c)
			F.x.copy(d.mod(&p))
		}
	}
	if s {F.neg()}
	F.norm()
}

/* this*=this mod Modulus */
func (F *FP) sqr() {
	F.norm();
	ea:=EXCESS(F.x)
	if (ea+1)>FEXCESS/(ea+1) {
		F.reduce()
	}
	d:=sqr(F.x)
	F.x.copy(mod(d))
}

/* this+=b */
func (F *FP) add(b *FP) {
	F.x.add(b.x)
	if (EXCESS(F.x)+2>=FEXCESS) {F.reduce()}
}

/* this-=b */
func (F *FP) sub(b *FP) {
	n:=NewFPcopy(b)
	n.neg()
	F.add(n)
}

/* this/=2 mod Modulus */
func (F *FP) div2() {
	F.x.norm()
	if (F.x.parity()==0) {
		F.x.fshr(1)
	} else {
		F.x.add(&p)
		F.x.norm()
		F.x.fshr(1)
	}
}

/* this=1/this mod Modulus */
func (F *FP) inverse() {
	r:=F.redc()
	r.invmodp(&p)
	F.x.copy(r)
	F.nres()
}

/* return TRUE if this==a */
func (F *FP) equals(a *FP) bool {
	a.reduce()
	F.reduce()
	if (comp(a.x,F.x)==0) {return true}
	return false
}

/* return this^e mod Modulus */
func (F *FP) pow(e *BIG) *FP {
	r:=NewFPint(1)
	e.norm()
	F.x.norm()
	m:=NewFPcopy(F)
	for true {
		bt:=e.parity();
		e.fshr(1);
		if bt==1 {r.mul(m)}
		if e.iszilch() {break}
		m.sqr();
	}
	r.x.mod(&p);
	return r;
}

/* return sqrt(this) mod Modulus */
func (F *FP) sqrt() *FP {
	F.reduce();
	b:=NewBIGcopy(&p)
	if MOD8==5 {
		b.dec(5); b.norm(); b.shr(3)
		i:=NewFPcopy(F); i.x.shl(1)
		v:=i.pow(b)
		i.mul(v); i.mul(v)
		i.x.dec(1)
		r:=NewFPcopy(F)
		r.mul(v); r.mul(i)
		r.reduce()
		return r
	} else {
		b.inc(1); b.norm(); b.shr(2)
		return F.pow(b);
	}
}

/* return jacobi symbol (this/Modulus) */
func (F *FP) jacobi() int {
	w:=F.redc();
	return w.jacobi(&p)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package httputil

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"

	"github.com/prometheus/prometheus/promql"
)

// GetQueryParams calls r.ParseForm and returns r.Form.
func GetQueryParams(r *http.Request) url.Values {
	r.ParseForm()
	return r.Form
}

var jsonFormatVersion = 1

// ErrorJSON writes the given error JSON-formatted to w.
func ErrorJSON(w io.Writer, err error) error {
	data := struct {
		Type    string `json:"type"`
		Value   string `json:"value"`
		Version int    `json:"version"`
	}{
		Type:    "error",
		Value:   err.Error(),
		Version: jsonFormatVersion,
	}
	enc := json.NewEncoder(w)
	return enc.Encode(data)
}

// RespondJSON converts the given data value to JSON and writes it to w.
func RespondJSON(w io.Writer, val promql.Value) error {
	data := struct {
		Type    string      `json:"type"`
		Value   interface{} `json:"value"`
		Version int         `json:"version"`
	}{
		Type:    val.Type().String(),
		Value:   val,
		Version: jsonFormatVersion,
	}
	// TODO(fabxc): Adding MarshalJSON to promql.Values might be a good idea.
	if sc, ok := val.(*promql.Scalar); ok {
		data.Value = sc.Value
	}
	enc := json.NewEncoder(w)
	return enc.Encode(data)
}
<commit_msg>httputil: remove non-general helper functions<commit_after><|endoftext|>


<commit_before>package main

import (
  "encoding/json"
  "io/ioutil"
  "log"
  "net/http"
  "os"
  "strings"

  "github.com/urfave/negroni"
  "github.com/unrolled/render"
  "github.com/chromatixau/gomiddleware"
)

func main() {
  logfilename := os.Getenv("GO_LOGFILE")
  if logfilename == "" {
    logfilename = "log/goapp.log"
  }
  errorLog, err := os.OpenFile(logfilename, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
  if err != nil {
    log.Fatal("error writing to log: " + logfilename)
  }
  defer errorLog.Close()

  rend := render.New(render.Options{IsDevelopment: true, Directory: "theme/templates" })
  mux := http.NewServeMux()
  n := negroni.New()
  l := chromatixau.NewLoggerWithStream( errorLog )
  r := negroni.NewRecovery()
  r.Logger = l
  r.PrintStack = false
  baseRoute := os.Getenv("GOBASEROUTE")

  handleRender(mux, rend, l, baseRoute)
  s := chromatixau.NewStatic(http.Dir("public"))
  if baseRoute != "" {
    s.Prefix = "/" + baseRoute
  }

  n.Use(r)
  n.Use(l)
  n.UseHandler(mux)

  n.Use(s)

  port := ":" + os.Getenv("PORT")
  if port == ":" {
    port = ":8080"
  }
  addr := os.Getenv("SERVER_ADDR")

  l.Println("Starting Goapp Service")
  l.Println("----------------------")
  http.ListenAndServe( addr + port, n )
}

func handleRender(mux *http.ServeMux, rend *render.Render, logger chromatixau.ALogger, base string) {
  mux.HandleFunc( "/", func(w http.ResponseWriter, req *http.Request) {
    logger.Println( "start" )
    baseURI, prefix := getBaseURI(req, base, logger)

    templateName, hasTemplate, isPublicFile := getTemplate(req, prefix, logger)
    if isPublicFile == true {
      logger.Println( "public file" )
      return
    }
    if false == hasTemplate {
      rend.HTML(w, http.StatusServiceUnavailable, "default/templateUnavailable", "")
      return
    }
    data := loadData(req, templateName + ".json", baseURI, prefix, logger)

    rend.HTML(w, http.StatusOK, templateName, data)
  })
}

func getBaseURI(req *http.Request, baseRoute string, logger chromatixau.ALogger) (string, string) {
  scheme, host, prefix, _ := getRequestVars(req, baseRoute, logger)
  baseURI := scheme + "://" + host
  if prefix != "" {
    baseURI = baseURI + "/" + prefix
  }
  return baseURI, prefix
}

func getRequestVars(req *http.Request, baseRoute string, logger chromatixau.ALogger) (string, string, string, string) {
  proto := req.URL.Scheme
  if proto == "" {
    proto = "http"
  }

  forwardedProto := req.Header.Get( "X-Forwarded-Proto" )
  if forwardedProto != "" {
    proto = forwardedProto
  }

  forwardedHost := req.Header.Get( "X-Forwarded-Host" )
  host := req.Host
  if forwardedHost != "" {
    host = forwardedHost
  }

  forwardedPrefix := req.Header.Get( "X-Forwarded-Prefix" )
  prefix := baseRoute
  if forwardedPrefix != "" {
    prefix = forwardedPrefix
  }

  forwardedPath := req.Header.Get( "X-Forwarded-Path" )
  path := req.URL.Path
  if forwardedPath != "" {
    path = forwardedPath
  }
	return proto, host, prefix, path
}

func getTemplate(req *http.Request, baseURI string, logger chromatixau.ALogger) (string, bool, bool) {
  logger.Println( "RequestURI Template: " + req.RequestURI )
  logger.Println( "BaseURI Template: " + baseURI )
	templateName := strings.TrimSuffix(strings.TrimPrefix(req.RequestURI, "/" + baseURI), "/")
  isPublicFile := false
  hasTemplate := false

  if strings.HasPrefix(templateName, "/") {
    templateName = strings.TrimPrefix( templateName, "/" )
  }

  logger.Println( "Template Name: [" + templateName + "]" )
	if templateName == "" {
		templateName = "index"
    hasTemplate = Exists( templateName, ".tmpl", "theme/templates", logger )
    isPublicFile = false
    return templateName, hasTemplate, isPublicFile
	}

  isPublicFile = Exists( templateName, "", "public", logger )
  if false == isPublicFile {
    hasTemplate = Exists( templateName, ".tmpl", "theme/templates", logger )
  }
  return templateName, hasTemplate, isPublicFile
}

func Exists(name string, extension string, folder string, logger chromatixau.ALogger) bool {
  filename := folder + "/" + name + extension
  logger.Println( "file exists?: " + filename )
  _, err := os.Stat( filename )
  if os.IsNotExist(err) {
    logger.Println( "No" )
    return false
  }
  logger.Println( "Yes" )
  return true
}

func loadData(req *http.Request, filename string, baseURI string, prefix string, logger chromatixau.ALogger) interface{} {
  var raw []byte
  var data map[string]interface{}
  logger.Println( "datafile: " + filename )
  raw, err := ioutil.ReadFile("theme/data/" + filename)
  if err != nil {
    data = make( map[string]interface{} )
  } else {
    err = json.Unmarshal(raw, &data)
    if err != nil {
      logger.Println( "Unmarshalling JSON failed" )
    }
  }
  data["BaseURI"] = baseURI + "/"
  basePath := ""
  if prefix != "" {
    basePath = "/" + prefix
  }
  path := strings.TrimPrefix(req.RequestURI, basePath)
  data["CanonicalURI"] = strings.TrimSuffix(baseURI + path, "/")
  return data
}
<commit_msg>use gomiddleware repo<commit_after>package main

import (
  "encoding/json"
  "io/ioutil"
  "log"
  "net/http"
  "os"
  "strings"

  "github.com/urfave/negroni"
  "github.com/unrolled/render"
  "github.com/chromatixau/gomiddleware"
)

func main() {
  logfilename := os.Getenv("GO_LOGFILE")
  if logfilename == "" {
    logfilename = "log/goapp.log"
  }
  errorLog, err := os.OpenFile(logfilename, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
  if err != nil {
    log.Fatal("error writing to log: " + logfilename)
  }
  defer errorLog.Close()

  rend := render.New(render.Options{IsDevelopment: true, Directory: "theme/templates" })
  mux := http.NewServeMux()
  n := negroni.New()
  l := gomiddleware.NewLoggerWithStream( errorLog )
  r := negroni.NewRecovery()
  r.Logger = l
  r.PrintStack = false
  baseRoute := os.Getenv("GOBASEROUTE")

  handleRender(mux, rend, l, baseRoute)
  s := gomiddleware.NewStatic(http.Dir("public"))
  if baseRoute != "" {
    s.Prefix = "/" + baseRoute
  }

  n.Use(r)
  n.Use(l)
  n.UseHandler(mux)

  n.Use(s)

  port := ":" + os.Getenv("PORT")
  if port == ":" {
    port = ":8080"
  }
  addr := os.Getenv("SERVER_ADDR")

  l.Println("Starting Goapp Service")
  l.Println("----------------------")
  http.ListenAndServe( addr + port, n )
}

func handleRender(mux *http.ServeMux, rend *render.Render, logger gomiddleware.ALogger, base string) {
  mux.HandleFunc( "/", func(w http.ResponseWriter, req *http.Request) {
    logger.Println( "start" )
    baseURI, prefix := getBaseURI(req, base, logger)

    templateName, hasTemplate, isPublicFile := getTemplate(req, prefix, logger)
    if isPublicFile == true {
      logger.Println( "public file" )
      return
    }
    if false == hasTemplate {
      rend.HTML(w, http.StatusServiceUnavailable, "default/templateUnavailable", "")
      return
    }
    data := loadData(req, templateName + ".json", baseURI, prefix, logger)

    rend.HTML(w, http.StatusOK, templateName, data)
  })
}

func getBaseURI(req *http.Request, baseRoute string, logger gomiddleware.ALogger) (string, string) {
  scheme, host, prefix, _ := getRequestVars(req, baseRoute, logger)
  baseURI := scheme + "://" + host
  if prefix != "" {
    baseURI = baseURI + "/" + prefix
  }
  return baseURI, prefix
}

func getRequestVars(req *http.Request, baseRoute string, logger gomiddleware.ALogger) (string, string, string, string) {
  proto := req.URL.Scheme
  if proto == "" {
    proto = "http"
  }

  forwardedProto := req.Header.Get( "X-Forwarded-Proto" )
  if forwardedProto != "" {
    proto = forwardedProto
  }

  forwardedHost := req.Header.Get( "X-Forwarded-Host" )
  host := req.Host
  if forwardedHost != "" {
    host = forwardedHost
  }

  forwardedPrefix := req.Header.Get( "X-Forwarded-Prefix" )
  prefix := baseRoute
  if forwardedPrefix != "" {
    prefix = forwardedPrefix
  }

  forwardedPath := req.Header.Get( "X-Forwarded-Path" )
  path := req.URL.Path
  if forwardedPath != "" {
    path = forwardedPath
  }
	return proto, host, prefix, path
}

func getTemplate(req *http.Request, baseURI string, logger gomiddleware.ALogger) (string, bool, bool) {
  logger.Println( "RequestURI Template: " + req.RequestURI )
  logger.Println( "BaseURI Template: " + baseURI )
	templateName := strings.TrimSuffix(strings.TrimPrefix(req.RequestURI, "/" + baseURI), "/")
  isPublicFile := false
  hasTemplate := false

  if strings.HasPrefix(templateName, "/") {
    templateName = strings.TrimPrefix( templateName, "/" )
  }

  logger.Println( "Template Name: [" + templateName + "]" )
	if templateName == "" {
		templateName = "index"
    hasTemplate = Exists( templateName, ".tmpl", "theme/templates", logger )
    isPublicFile = false
    return templateName, hasTemplate, isPublicFile
	}

  isPublicFile = Exists( templateName, "", "public", logger )
  if false == isPublicFile {
    hasTemplate = Exists( templateName, ".tmpl", "theme/templates", logger )
  }
  return templateName, hasTemplate, isPublicFile
}

func Exists(name string, extension string, folder string, logger gomiddleware.ALogger) bool {
  filename := folder + "/" + name + extension
  logger.Println( "file exists?: " + filename )
  _, err := os.Stat( filename )
  if os.IsNotExist(err) {
    logger.Println( "No" )
    return false
  }
  logger.Println( "Yes" )
  return true
}

func loadData(req *http.Request, filename string, baseURI string, prefix string, logger gomiddleware.ALogger) interface{} {
  var raw []byte
  var data map[string]interface{}
  logger.Println( "datafile: " + filename )
  raw, err := ioutil.ReadFile("theme/data/" + filename)
  if err != nil {
    data = make( map[string]interface{} )
  } else {
    err = json.Unmarshal(raw, &data)
    if err != nil {
      logger.Println( "Unmarshalling JSON failed" )
    }
  }
  data["BaseURI"] = baseURI + "/"
  basePath := ""
  if prefix != "" {
    basePath = "/" + prefix
  }
  path := strings.TrimPrefix(req.RequestURI, basePath)
  data["CanonicalURI"] = strings.TrimSuffix(baseURI + path, "/")
  return data
}
<|endoftext|>


<commit_before>package pull

import (
	"context"
	"sync"
	"sync/atomic"
	"time"

	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/remotes"
	"github.com/containerd/containerd/remotes/docker"
	distreference "github.com/docker/distribution/reference"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/resolver"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

var cache *resolverCache

func init() {
	cache = newResolverCache()
}

type ResolverOpt struct {
	Hosts      docker.RegistryHosts
	Auth       *resolver.SessionAuthenticator
	ImageStore images.Store
	Mode       source.ResolveMode
	Ref        string
}

func NewResolver(g session.Group, opt ResolverOpt) remotes.Resolver {
	if res := cache.Get(opt.Ref, g); res != nil {
		return withLocal(res, opt.ImageStore, opt.Mode)
	}

	r := resolver.New(opt.Hosts, opt.Auth)
	r = cache.Add(opt.Ref, r, opt.Auth, g)

	return withLocal(r, opt.ImageStore, opt.Mode)
}

func EnsureManifestRequested(ctx context.Context, res remotes.Resolver, ref string) {
	rr := res
	lr, ok := res.(withLocalResolver)
	if ok {
		if atomic.LoadInt64(&lr.counter) > 0 {
			return
		}
		rr = lr.Resolver
	}
	cr, ok := rr.(*cachedResolver)
	if !ok {
		return
	}
	if atomic.LoadInt64(&cr.counter) == 0 {
		res.Resolve(ctx, ref)
	}
}

func withLocal(r remotes.Resolver, imageStore images.Store, mode source.ResolveMode) remotes.Resolver {
	if imageStore == nil || mode == source.ResolveModeForcePull {
		return r
	}

	return withLocalResolver{Resolver: r, is: imageStore, mode: mode}
}

// A remotes.Resolver which checks the local image store if the real
// resolver cannot find the image, essentially falling back to a local
// image if one is present.
//
// We do not override the Fetcher or Pusher methods:
//
// - Fetcher is called by github.com/containerd/containerd/remotes/:fetch()
//   only after it has checked for the content locally, so avoid the
//   hassle of interposing a local-fetch proxy and simply pass on the
//   request.
// - Pusher wouldn't make sense to push locally, so just forward.

type withLocalResolver struct {
	counter int64 // needs to be 64bit aligned for 32bit systems
	remotes.Resolver
	is   images.Store
	mode source.ResolveMode
}

func (r withLocalResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) {
	if r.mode == source.ResolveModePreferLocal {
		if img, err := r.is.Get(ctx, ref); err == nil {
			atomic.AddInt64(&r.counter, 1)
			return ref, img.Target, nil
		}
	}

	n, desc, err := r.Resolver.Resolve(ctx, ref)
	if err == nil {
		return n, desc, err
	}

	if r.mode == source.ResolveModeDefault {
		if img, err := r.is.Get(ctx, ref); err == nil {
			return ref, img.Target, nil
		}
	}

	return "", ocispec.Descriptor{}, err
}

type resolverCache struct {
	mu sync.Mutex
	m  map[string]cachedResolver
}

type cachedResolver struct {
	counter int64
	timeout time.Time
	remotes.Resolver
	auth *resolver.SessionAuthenticator
}

func (cr *cachedResolver) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
	atomic.AddInt64(&cr.counter, 1)
	return cr.Resolver.Resolve(ctx, ref)
}

func (r *resolverCache) Add(ref string, resolver remotes.Resolver, auth *resolver.SessionAuthenticator, g session.Group) *cachedResolver {
	r.mu.Lock()
	defer r.mu.Unlock()

	ref = r.repo(ref)

	cr, ok := r.m[ref]
	cr.timeout = time.Now().Add(time.Minute)
	if ok {
		cr.auth.AddSession(g)
		return &cr
	}

	cr.Resolver = resolver
	cr.auth = auth
	r.m[ref] = cr
	return &cr
}

func (r *resolverCache) repo(refStr string) string {
	ref, err := distreference.ParseNormalizedNamed(refStr)
	if err != nil {
		return refStr
	}
	return ref.Name()
}

func (r *resolverCache) Get(ref string, g session.Group) *cachedResolver {
	r.mu.Lock()
	defer r.mu.Unlock()

	ref = r.repo(ref)

	cr, ok := r.m[ref]
	if ok {
		cr.auth.AddSession(g)
		return &cr
	}
	return nil
}

func (r *resolverCache) clean(now time.Time) {
	r.mu.Lock()
	for k, cr := range r.m {
		if now.After(cr.timeout) {
			delete(r.m, k)
		}
	}
	r.mu.Unlock()
}

func newResolverCache() *resolverCache {
	rc := &resolverCache{
		m: map[string]cachedResolver{},
	}
	t := time.NewTicker(time.Minute)
	go func() {
		for {
			rc.clean(<-t.C)
		}
	}()
	return rc
}
<commit_msg>pull: fix counter for ensuremanifestloaded check<commit_after>package pull

import (
	"context"
	"sync"
	"sync/atomic"
	"time"

	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/remotes"
	"github.com/containerd/containerd/remotes/docker"
	distreference "github.com/docker/distribution/reference"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/resolver"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

var cache *resolverCache

func init() {
	cache = newResolverCache()
}

type ResolverOpt struct {
	Hosts      docker.RegistryHosts
	Auth       *resolver.SessionAuthenticator
	ImageStore images.Store
	Mode       source.ResolveMode
	Ref        string
}

func NewResolver(g session.Group, opt ResolverOpt) remotes.Resolver {
	if res := cache.Get(opt.Ref, g); res != nil {
		return withLocal(res, opt.ImageStore, opt.Mode)
	}

	r := resolver.New(opt.Hosts, opt.Auth)
	r = cache.Add(opt.Ref, r, opt.Auth, g)

	return withLocal(r, opt.ImageStore, opt.Mode)
}

func EnsureManifestRequested(ctx context.Context, res remotes.Resolver, ref string) {
	rr := res
	lr, ok := res.(withLocalResolver)
	if ok {
		if atomic.LoadInt64(&lr.counter) > 0 {
			return
		}
		rr = lr.Resolver
	}
	cr, ok := rr.(*cachedResolver)
	if !ok {
		return
	}
	if atomic.LoadInt64(cr.counter) == 0 {
		res.Resolve(ctx, ref)
	}
}

func withLocal(r remotes.Resolver, imageStore images.Store, mode source.ResolveMode) remotes.Resolver {
	if imageStore == nil || mode == source.ResolveModeForcePull {
		return r
	}

	return withLocalResolver{Resolver: r, is: imageStore, mode: mode}
}

// A remotes.Resolver which checks the local image store if the real
// resolver cannot find the image, essentially falling back to a local
// image if one is present.
//
// We do not override the Fetcher or Pusher methods:
//
// - Fetcher is called by github.com/containerd/containerd/remotes/:fetch()
//   only after it has checked for the content locally, so avoid the
//   hassle of interposing a local-fetch proxy and simply pass on the
//   request.
// - Pusher wouldn't make sense to push locally, so just forward.

type withLocalResolver struct {
	counter int64 // needs to be 64bit aligned for 32bit systems
	remotes.Resolver
	is   images.Store
	mode source.ResolveMode
}

func (r withLocalResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) {
	if r.mode == source.ResolveModePreferLocal {
		if img, err := r.is.Get(ctx, ref); err == nil {
			atomic.AddInt64(&r.counter, 1)
			return ref, img.Target, nil
		}
	}

	n, desc, err := r.Resolver.Resolve(ctx, ref)
	if err == nil {
		return n, desc, err
	}

	if r.mode == source.ResolveModeDefault {
		if img, err := r.is.Get(ctx, ref); err == nil {
			return ref, img.Target, nil
		}
	}

	return "", ocispec.Descriptor{}, err
}

type resolverCache struct {
	mu sync.Mutex
	m  map[string]cachedResolver
}

type cachedResolver struct {
	counter *int64
	timeout time.Time
	remotes.Resolver
	auth *resolver.SessionAuthenticator
}

func (cr *cachedResolver) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
	atomic.AddInt64(cr.counter, 1)
	return cr.Resolver.Resolve(ctx, ref)
}

func (r *resolverCache) Add(ref string, resolver remotes.Resolver, auth *resolver.SessionAuthenticator, g session.Group) *cachedResolver {
	r.mu.Lock()
	defer r.mu.Unlock()

	ref = r.repo(ref)

	cr, ok := r.m[ref]
	cr.timeout = time.Now().Add(time.Minute)
	if ok {
		cr.auth.AddSession(g)
		return &cr
	}

	var counter int64
	cr.counter = &counter
	cr.Resolver = resolver
	cr.auth = auth
	r.m[ref] = cr
	return &cr
}

func (r *resolverCache) repo(refStr string) string {
	ref, err := distreference.ParseNormalizedNamed(refStr)
	if err != nil {
		return refStr
	}
	return ref.Name()
}

func (r *resolverCache) Get(ref string, g session.Group) *cachedResolver {
	r.mu.Lock()
	defer r.mu.Unlock()

	ref = r.repo(ref)

	cr, ok := r.m[ref]
	if ok {
		cr.auth.AddSession(g)
		return &cr
	}
	return nil
}

func (r *resolverCache) clean(now time.Time) {
	r.mu.Lock()
	for k, cr := range r.m {
		if now.After(cr.timeout) {
			delete(r.m, k)
		}
	}
	r.mu.Unlock()
}

func newResolverCache() *resolverCache {
	rc := &resolverCache{
		m: map[string]cachedResolver{},
	}
	t := time.NewTicker(time.Minute)
	go func() {
		for {
			rc.clean(<-t.C)
		}
	}()
	return rc
}
<|endoftext|>